From ca5d297661b0fc91268059f72b98a7d4cb690890 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 16 Jul 2025 17:59:12 +0000 Subject: [PATCH 001/798] try to test btrfs integration using github actions --- .github/workflows/make-and-test.yml | 8 ++------ src/package.json | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/make-and-test.yml b/.github/workflows/make-and-test.yml index b233bf2421..933bd6e328 100644 --- a/.github/workflows/make-and-test.yml +++ b/.github/workflows/make-and-test.yml @@ -91,12 +91,8 @@ jobs: # cache: "pnpm" # cache-dependency-path: "src/packages/pnpm-lock.yaml" - - name: Download and install Valkey - run: | - VALKEY_VERSION=8.1.2 - curl -LO https://download.valkey.io/releases/valkey-${VALKEY_VERSION}-jammy-x86_64.tar.gz - tar -xzf valkey-${VALKEY_VERSION}-jammy-x86_64.tar.gz - sudo cp valkey-${VALKEY_VERSION}-jammy-x86_64/bin/valkey-server /usr/local/bin/ + - name: Install btrfs-progs + run: sudo apt-get update && sudo apt-get install -y btrfs-progs - name: Set up Python venv and Jupyter kernel run: | diff --git a/src/package.json b/src/package.json index d0f0701277..efa416dcd0 100644 --- a/src/package.json +++ b/src/package.json @@ -18,7 +18,7 @@ "version-check": "pip3 install typing_extensions mypy || pip3 install --break-system-packages typing_extensions mypy && ./workspaces.py version-check && mypy scripts/check_npm_packages.py", "test-parallel": "unset DEBUG && pnpm run version-check && cd packages && pnpm run -r --parallel test", "test": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test", - "test-github-ci": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test --exclude=jupyter,file-server --retries=1", + "test-github-ci": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test --exclude=jupyter --retries=1", "depcheck": "cd packages && pnpm run -r --parallel depcheck", "prettier-all": "cd packages/", "local-ci": "./scripts/ci.sh", From f4ad609ff0bd5ad0990cae2efa3521f049f8ff35 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 16 Jul 2025 18:15:01 +0000 Subject: [PATCH 002/798] also need bup for file-server --- .github/workflows/make-and-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/make-and-test.yml b/.github/workflows/make-and-test.yml index 933bd6e328..19b2feeb82 100644 --- a/.github/workflows/make-and-test.yml +++ b/.github/workflows/make-and-test.yml @@ -91,8 +91,8 @@ jobs: # cache: "pnpm" # cache-dependency-path: "src/packages/pnpm-lock.yaml" - - name: Install btrfs-progs - run: sudo apt-get update && sudo apt-get install -y btrfs-progs + - name: Install btrfs-progs and bup for @cocalc/file-server + run: sudo apt-get update && sudo apt-get install -y btrfs-progs bup - name: Set up Python venv and Jupyter kernel run: | From adbb4ebcc9acf8b2cd76566897286ce808ea59d4 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 16 Jul 2025 18:15:09 +0000 Subject: [PATCH 003/798] fix a run script --- src/scripts/g-tmux.sh | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/scripts/g-tmux.sh b/src/scripts/g-tmux.sh index f75aaf865a..386479673a 100755 --- a/src/scripts/g-tmux.sh +++ b/src/scripts/g-tmux.sh @@ -12,14 +12,8 @@ sleep 2 tmux send-keys -t mysession:1 'pnpm database' C-m if [ -n "$NO_RSPACK_DEV_SERVER" ]; then -sleep 2 -tmux send-keys -t mysession:2 'pnpm rspack' C-m - -else - -# no longer needed, due to using rspack for nextjs -#sleep 2 -#tmux send-keys -t mysession:2 '$PWD/scripts/memory_monitor.py' C-m + sleep 2 + tmux send-keys -t mysession:2 'pnpm rspack' C-m fi tmux attach -t mysession:1 From 4cd8f3b44f1e47aed94272adf3f58dda48c6c936 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 16 Jul 2025 23:48:49 +0000 Subject: [PATCH 004/798] file-server: refactor fs sandbox module --- src/packages/file-server/btrfs/subvolume.ts | 6 +- .../{btrfs/subvolume-fs.ts => fs/index.ts} | 153 ++++++++++-------- 2 files changed, 89 insertions(+), 70 deletions(-) rename src/packages/file-server/{btrfs/subvolume-fs.ts => fs/index.ts} (51%) diff --git a/src/packages/file-server/btrfs/subvolume.ts b/src/packages/file-server/btrfs/subvolume.ts index 3f77abd9c4..6926fc8e57 100644 --- a/src/packages/file-server/btrfs/subvolume.ts +++ b/src/packages/file-server/btrfs/subvolume.ts @@ -6,10 +6,10 @@ import { type Filesystem, DEFAULT_SUBVOLUME_SIZE } from "./filesystem"; import refCache from "@cocalc/util/refcache"; import { sudo } from "./util"; import { join, normalize } from "path"; -import { SubvolumeFilesystem } from "./subvolume-fs"; import { SubvolumeBup } from "./subvolume-bup"; import { SubvolumeSnapshots } from "./subvolume-snapshots"; import { SubvolumeQuota } from "./subvolume-quota"; +import { SandboxedFilesystem } from "../fs"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import getLogger from "@cocalc/backend/logger"; @@ -26,7 +26,7 @@ export class Subvolume { public readonly filesystem: Filesystem; public readonly path: string; - public readonly fs: SubvolumeFilesystem; + public readonly fs: SandboxedFilesystem; public readonly bup: SubvolumeBup; public readonly snapshots: SubvolumeSnapshots; public readonly quota: SubvolumeQuota; @@ -35,7 +35,7 @@ export class Subvolume { this.filesystem = filesystem; this.name = name; this.path = join(filesystem.opts.mount, name); - this.fs = new SubvolumeFilesystem(this); + this.fs = new SandboxedFilesystem(this.path); this.bup = new SubvolumeBup(this); this.snapshots = new SubvolumeSnapshots(this); this.quota = new SubvolumeQuota(this); diff --git a/src/packages/file-server/btrfs/subvolume-fs.ts b/src/packages/file-server/fs/index.ts similarity index 51% rename from src/packages/file-server/btrfs/subvolume-fs.ts rename to src/packages/file-server/fs/index.ts index f1f2dd3677..487327c17e 100644 --- a/src/packages/file-server/btrfs/subvolume-fs.ts +++ b/src/packages/file-server/fs/index.ts @@ -1,3 +1,9 @@ +/* +Given a path to a folder on the filesystem, this provides +a wrapper class with an API very similar to the fs/promises modules, +but which only allows access to files in that folder. +*/ + import { appendFile, chmod, @@ -21,111 +27,100 @@ import { import { exists } from "@cocalc/backend/misc/async-utils-node"; import { type DirectoryListingEntry } from "@cocalc/util/types"; import getListing from "@cocalc/backend/get-listing"; -import { type Subvolume } from "./subvolume"; -import { isdir, sudo } from "./util"; - -export class SubvolumeFilesystem { - constructor(private subvolume: Subvolume) {} - - private normalize = this.subvolume.normalize; +import { isdir, sudo } from "../btrfs/util"; +import { join, resolve } from "path"; - ls = async ( - path: string, - { hidden, limit }: { hidden?: boolean; limit?: number } = {}, - ): Promise => { - return await getListing(this.normalize(path), hidden, { - limit, - home: "/", - }); - }; +export class SandboxedFilesystem { + // path should be the path to a FOLDER on the filesystem (not a file) + constructor(public readonly path: string) {} - readFile = async (path: string, encoding?: any): Promise => { - return await readFile(this.normalize(path), encoding); + private safeAbsPath = (path: string) => { + if (typeof path != "string") { + throw Error(`path must be a string but is of type ${typeof path}`); + } + return join(this.path, resolve("/", path)); }; - writeFile = async (path: string, data: string | Buffer) => { - return await writeFile(this.normalize(path), data); + appendFile = async (path: string, data: string | Buffer, encoding?) => { + return await appendFile(this.safeAbsPath(path), data, encoding); }; - appendFile = async (path: string, data: string | Buffer, encoding?) => { - return await appendFile(this.normalize(path), data, encoding); + chmod = async (path: string, mode: string | number) => { + await chmod(this.safeAbsPath(path), mode); }; - unlink = async (path: string) => { - await unlink(this.normalize(path)); + copyFile = async (src: string, dest: string) => { + await copyFile(this.safeAbsPath(src), this.safeAbsPath(dest)); }; - stat = async (path: string) => { - return await stat(this.normalize(path)); + cp = async (src: string, dest: string, options?) => { + await cp(this.safeAbsPath(src), this.safeAbsPath(dest), options); }; exists = async (path: string) => { - return await exists(this.normalize(path)); + return await exists(this.safeAbsPath(path)); }; // hard link link = async (existingPath: string, newPath: string) => { - return await link(this.normalize(existingPath), this.normalize(newPath)); - }; - - symlink = async (target: string, path: string) => { - return await symlink(this.normalize(target), this.normalize(path)); - }; - - realpath = async (path: string) => { - const x = await realpath(this.normalize(path)); - return x.slice(this.subvolume.path.length + 1); - }; - - rename = async (oldPath: string, newPath: string) => { - await rename(this.normalize(oldPath), this.normalize(newPath)); + return await link( + this.safeAbsPath(existingPath), + this.safeAbsPath(newPath), + ); }; - utimes = async ( + ls = async ( path: string, - atime: number | string | Date, - mtime: number | string | Date, - ) => { - await utimes(this.normalize(path), atime, mtime); + { hidden, limit }: { hidden?: boolean; limit?: number } = {}, + ): Promise => { + return await getListing(this.safeAbsPath(path), hidden, { + limit, + home: "/", + }); }; - watch = (filename: string, options?) => { - return watch(this.normalize(filename), options); + mkdir = async (path: string, options?) => { + await mkdir(this.safeAbsPath(path), options); }; - truncate = async (path: string, len?: number) => { - await truncate(this.normalize(path), len); + readFile = async (path: string, encoding?: any): Promise => { + return await readFile(this.safeAbsPath(path), encoding); }; - copyFile = async (src: string, dest: string) => { - await copyFile(this.normalize(src), this.normalize(dest)); + realpath = async (path: string) => { + const x = await realpath(this.safeAbsPath(path)); + return x.slice(this.path.length + 1); }; - cp = async (src: string, dest: string, options?) => { - await cp(this.normalize(src), this.normalize(dest), options); + rename = async (oldPath: string, newPath: string) => { + await rename(this.safeAbsPath(oldPath), this.safeAbsPath(newPath)); }; - chmod = async (path: string, mode: string | number) => { - await chmod(this.normalize(path), mode); + rm = async (path: string, options?) => { + await rm(this.safeAbsPath(path), options); }; - mkdir = async (path: string, options?) => { - await mkdir(this.normalize(path), options); + rmdir = async (path: string, options?) => { + await rmdir(this.safeAbsPath(path), options); }; rsync = async ({ src, target, - args = ["-axH"], timeout = 5 * 60 * 1000, }: { src: string; target: string; - args?: string[]; timeout?: number; }): Promise<{ stdout: string; stderr: string; exit_code: number }> => { - let srcPath = this.normalize(src); - let targetPath = this.normalize(target); + let srcPath = this.safeAbsPath(src); + let targetPath = this.safeAbsPath(target); + if (src.endsWith("/")) { + srcPath += "/"; + } + if (target.endsWith("/")) { + targetPath += "/"; + } if (!srcPath.endsWith("/") && (await isdir(srcPath))) { srcPath += "/"; if (!targetPath.endsWith("/")) { @@ -134,17 +129,41 @@ export class SubvolumeFilesystem { } return await sudo({ command: "rsync", - args: [...args, srcPath, targetPath], + args: [srcPath, targetPath], err_on_exit: false, timeout: timeout / 1000, }); }; - rmdir = async (path: string, options?) => { - await rmdir(this.normalize(path), options); + stat = async (path: string) => { + return await stat(this.safeAbsPath(path)); }; - rm = async (path: string, options?) => { - await rm(this.normalize(path), options); + symlink = async (target: string, path: string) => { + return await symlink(this.safeAbsPath(target), this.safeAbsPath(path)); + }; + + truncate = async (path: string, len?: number) => { + await truncate(this.safeAbsPath(path), len); + }; + + unlink = async (path: string) => { + await unlink(this.safeAbsPath(path)); + }; + + utimes = async ( + path: string, + atime: number | string | Date, + mtime: number | string | Date, + ) => { + await utimes(this.safeAbsPath(path), atime, mtime); + }; + + watch = (filename: string, options?) => { + return watch(this.safeAbsPath(filename), options); + }; + + writeFile = async (path: string, data: string | Buffer) => { + return await writeFile(this.safeAbsPath(path), data); }; } From d03d6f9be68773dfcffbd564fc26e15cfde7f16c Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 17 Jul 2025 01:56:45 +0000 Subject: [PATCH 005/798] working on adding conat fileserver support --- src/packages/conat/files/fs.ts | 134 ++++++++++++++++++ .../file-server/btrfs/subvolume-bup.ts | 6 +- src/packages/file-server/btrfs/subvolume.ts | 40 ++++-- src/packages/file-server/conat/local-path.ts | 44 ++++++ .../file-server/conat/test/local-path.test.ts | 3 + src/packages/file-server/fs/sandbox.test.ts | 59 ++++++++ .../file-server/fs/{index.ts => sandbox.ts} | 50 ++----- src/packages/file-server/package.json | 17 +-- src/packages/pnpm-lock.yaml | 7 +- 9 files changed, 301 insertions(+), 59 deletions(-) create mode 100644 src/packages/conat/files/fs.ts create mode 100644 src/packages/file-server/conat/local-path.ts create mode 100644 src/packages/file-server/conat/test/local-path.test.ts create mode 100644 src/packages/file-server/fs/sandbox.test.ts rename src/packages/file-server/fs/{index.ts => sandbox.ts} (78%) diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts new file mode 100644 index 0000000000..048a491f91 --- /dev/null +++ b/src/packages/conat/files/fs.ts @@ -0,0 +1,134 @@ +import { type Client } from "@cocalc/conat/core/client"; +import { conat } from "@cocalc/conat/client"; + +export interface Filesystem { + appendFile: (path: string, data: string | Buffer, encoding?) => Promise; + chmod: (path: string, mode: string | number) => Promise; + copyFile: (src: string, dest: string) => Promise; + cp: (src: string, dest: string, options?) => Promise; + exists: (path: string) => Promise; + link: (existingPath: string, newPath: string) => Promise; + mkdir: (path: string, options?) => Promise; + readFile: (path: string, encoding?: any) => Promise; + readdir: (path: string) => Promise; + realpath: (path: string) => Promise; + rename: (oldPath: string, newPath: string) => Promise; + rm: (path: string, options?) => Promise; + rmdir: (path: string, options?) => Promise; + stat: (path: string) => Promise; + symlink: (target: string, path: string) => Promise; + truncate: (path: string, len?: number) => Promise; + unlink: (path: string) => Promise; + utimes: ( + path: string, + atime: number | string | Date, + mtime: number | string | Date, + ) => Promise; + writeFile: (path: string, data: string | Buffer) => Promise; +} + +export interface Stats { + dev: number; + ino: number; + mode: number; + nlink: number; + uid: number; + gid: number; + rdev: number; + size: number; + blksize: number; + blocks: number; + atimeMs: number; + mtimeMs: number; + ctimeMs: number; + birthtimeMs: number; + atime: Date; + mtime: Date; + ctime: Date; + birthtime: Date; +} + +interface Options { + service: string; + client?: Client; + fs: (subject?: string) => Promise; +} + +export async function fsServer({ service, fs, client }: Options) { + return await (client ?? conat()).service( + `${service}.*`, + { + async appendFile(path: string, data: string | Buffer, encoding?) { + await (await fs(this.subject)).appendFile(path, data, encoding); + }, + async chmod(path: string, mode: string | number) { + await (await fs(this.subject)).chmod(path, mode); + }, + async copyFile(src: string, dest: string) { + await (await fs(this.subject)).copyFile(src, dest); + }, + async cp(src: string, dest: string, options?) { + await (await fs(this.subject)).cp(src, dest, options); + }, + async exists(path: string) { + await (await fs(this.subject)).exists(path); + }, + async link(existingPath: string, newPath: string) { + await (await fs(this.subject)).link(existingPath, newPath); + }, + async mkdir(path: string, options?) { + await (await fs(this.subject)).mkdir(path, options); + }, + async readFile(path: string, encoding?) { + return await (await fs(this.subject)).readFile(path, encoding); + }, + async readdir(path: string) { + return await (await fs(this.subject)).readdir(path); + }, + async realpath(path: string) { + return await (await fs(this.subject)).realpath(path); + }, + async rename(oldPath: string, newPath: string) { + return await (await fs(this.subject)).rename(oldPath, newPath); + }, + async rm(path: string, options?) { + return await (await fs(this.subject)).rm(path, options); + }, + async rmdir(path: string, options?) { + return await (await fs(this.subject)).rmdir(path, options); + }, + async stat(path: string): Promise { + return await (await fs(this.subject)).stat(path); + }, + async symlink(target: string, path: string) { + return await (await fs(this.subject)).symlink(target, path); + }, + async truncate(path: string, len?: number) { + return await (await fs(this.subject)).truncate(path, len); + }, + async unlink(path: string) { + return await (await fs(this.subject)).unlink(path); + }, + async utimes( + path: string, + atime: number | string | Date, + mtime: number | string | Date, + ) { + return await (await fs(this.subject)).utimes(path, atime, mtime); + }, + async writeFile(path: string, data: string | Buffer) { + return await (await fs(this.subject)).writeFile(path, data); + }, + }, + ); +} + +export function fsClient({ + client, + subject, +}: { + client?: Client; + subject: string; +}) { + return (client ?? conat()).call(subject); +} diff --git a/src/packages/file-server/btrfs/subvolume-bup.ts b/src/packages/file-server/btrfs/subvolume-bup.ts index 21cbbff364..3849b49379 100644 --- a/src/packages/file-server/btrfs/subvolume-bup.ts +++ b/src/packages/file-server/btrfs/subvolume-bup.ts @@ -46,7 +46,7 @@ export class SubvolumeBup { `createBackup: creating ${BUP_SNAPSHOT} to get a consistent backup`, ); await this.subvolume.snapshots.create(BUP_SNAPSHOT); - const target = this.subvolume.normalize( + const target = this.subvolume.fs.safeAbsPath( this.subvolume.snapshots.path(BUP_SNAPSHOT), ); @@ -133,7 +133,9 @@ export class SubvolumeBup { return v; } - path = normalize(path); + path = this.subvolume.fs + .safeAbsPath(path) + .slice(this.subvolume.path.length); const { stdout } = await sudo({ command: "bup", args: [ diff --git a/src/packages/file-server/btrfs/subvolume.ts b/src/packages/file-server/btrfs/subvolume.ts index 6926fc8e57..72c3dd7d54 100644 --- a/src/packages/file-server/btrfs/subvolume.ts +++ b/src/packages/file-server/btrfs/subvolume.ts @@ -4,12 +4,12 @@ A subvolume import { type Filesystem, DEFAULT_SUBVOLUME_SIZE } from "./filesystem"; import refCache from "@cocalc/util/refcache"; -import { sudo } from "./util"; -import { join, normalize } from "path"; +import { isdir, sudo } from "./util"; +import { join } from "path"; import { SubvolumeBup } from "./subvolume-bup"; import { SubvolumeSnapshots } from "./subvolume-snapshots"; import { SubvolumeQuota } from "./subvolume-quota"; -import { SandboxedFilesystem } from "../fs"; +import { SandboxedFilesystem } from "../fs/sandbox"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import getLogger from "@cocalc/backend/logger"; @@ -77,11 +77,35 @@ export class Subvolume { }); }; - // this should provide a path that is guaranteed to be - // inside this.path on the filesystem or throw error - // [ ] TODO: not sure if the code here is sufficient!! - normalize = (path: string) => { - return join(this.path, normalize(path)); + rsync = async ({ + src, + target, + timeout = 5 * 60 * 1000, + }: { + src: string; + target: string; + timeout?: number; + }): Promise<{ stdout: string; stderr: string; exit_code: number }> => { + let srcPath = this.fs.safeAbsPath(src); + let targetPath = this.fs.safeAbsPath(target); + if (src.endsWith("/")) { + srcPath += "/"; + } + if (target.endsWith("/")) { + targetPath += "/"; + } + if (!srcPath.endsWith("/") && (await isdir(srcPath))) { + srcPath += "/"; + if (!targetPath.endsWith("/")) { + targetPath += "/"; + } + } + return await sudo({ + command: "rsync", + args: [srcPath, targetPath], + err_on_exit: false, + timeout: timeout / 1000, + }); }; } diff --git a/src/packages/file-server/conat/local-path.ts b/src/packages/file-server/conat/local-path.ts new file mode 100644 index 0000000000..e8450f3adf --- /dev/null +++ b/src/packages/file-server/conat/local-path.ts @@ -0,0 +1,44 @@ +import { fsServer } from "@cocalc/conat/files/fs"; +import { conat } from "@cocalc/backend/conat"; +import { SandboxedFilesystem } from "@cocalc/file-server/fs/sandbox"; +import { mkdir } from "fs/promises"; +import { join } from "path"; +import { isValidUUID } from "@cocalc/util/misc"; + +export function localPathFileserver({ + service, + path, +}: { + service: string; + path: string; +}) { + const client = conat(); + const server = fsServer({ + service, + client, + fs: async (subject: string) => { + const project_id = getProjectId(subject); + const p = join(path, project_id); + try { + await mkdir(p); + } catch {} + return new SandboxedFilesystem(p); + }, + }); + return server; +} + +function getProjectId(subject: string) { + const v = subject.split("."); + if (v.length != 2) { + throw Error("subject must have 2 segments"); + } + if (!v[1].startsWith("project-")) { + throw Error("second segment of subject must start with 'project-'"); + } + const project_id = v[1].slice("project-".length); + if (!isValidUUID(project_id)) { + throw Error("not a valid project id"); + } + return project_id; +} diff --git a/src/packages/file-server/conat/test/local-path.test.ts b/src/packages/file-server/conat/test/local-path.test.ts new file mode 100644 index 0000000000..4aa561af08 --- /dev/null +++ b/src/packages/file-server/conat/test/local-path.test.ts @@ -0,0 +1,3 @@ +describe("use the simple fileserver", () => { + it("does nothing", async () => {}); +}); diff --git a/src/packages/file-server/fs/sandbox.test.ts b/src/packages/file-server/fs/sandbox.test.ts new file mode 100644 index 0000000000..73bd45b171 --- /dev/null +++ b/src/packages/file-server/fs/sandbox.test.ts @@ -0,0 +1,59 @@ +import { SandboxedFilesystem } from "./sandbox"; +import { mkdtemp, mkdir, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "path"; + +let tempDir; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); +}); + +describe("test using the filesystem sandbox to do a few standard things", () => { + let fs; + it("creates and reads file", async () => { + await mkdir(join(tempDir, "test-1")); + fs = new SandboxedFilesystem(join(tempDir, "test-1")); + await fs.writeFile("a", "hi"); + const r = await fs.readFile("a", "utf8"); + expect(r).toEqual("hi"); + }); + + it("truncate file", async () => { + await fs.writeFile("b", "hello"); + await fs.truncate("b", 4); + const r = await fs.readFile("b", "utf8"); + expect(r).toEqual("hell"); + }); +}); + +describe("make various attempts to break out of the sandbox", () => { + let fs; + it("creates sandbox", async () => { + await mkdir(join(tempDir, "test-2")); + fs = new SandboxedFilesystem(join(tempDir, "test-2")); + await fs.writeFile("x", "hi"); + }); + + it("obvious first attempt to escape fails", async () => { + const v = await fs.readdir(".."); + expect(v).toEqual(["x"]); + }); + + it("obvious first attempt to escape fails", async () => { + const v = await fs.readdir("a/../.."); + expect(v).toEqual(["x"]); + }); + + it("another attempt", async () => { + await fs.copyFile("/x", "/tmp"); + const v = await fs.readdir("a/../.."); + expect(v).toEqual(["tmp", "x"]); + + const r = await fs.readFile("tmp", "utf8"); + expect(r).toEqual("hi"); + }); +}); + +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); diff --git a/src/packages/file-server/fs/index.ts b/src/packages/file-server/fs/sandbox.ts similarity index 78% rename from src/packages/file-server/fs/index.ts rename to src/packages/file-server/fs/sandbox.ts index 487327c17e..dabd277ec2 100644 --- a/src/packages/file-server/fs/index.ts +++ b/src/packages/file-server/fs/sandbox.ts @@ -1,7 +1,14 @@ /* Given a path to a folder on the filesystem, this provides -a wrapper class with an API very similar to the fs/promises modules, +a wrapper class with an API similar to the fs/promises modules, but which only allows access to files in that folder. +It's a bit simpler with return data that is always +serializable. + +Absolute and relative paths are considered as relative to the input folder path. + +REFERENCE: We don't use https://github.com/metarhia/sandboxed-fs, but did +look at the code. */ import { @@ -10,6 +17,7 @@ import { cp, copyFile, link, + readdir, readFile, realpath, rename, @@ -27,14 +35,13 @@ import { import { exists } from "@cocalc/backend/misc/async-utils-node"; import { type DirectoryListingEntry } from "@cocalc/util/types"; import getListing from "@cocalc/backend/get-listing"; -import { isdir, sudo } from "../btrfs/util"; import { join, resolve } from "path"; export class SandboxedFilesystem { // path should be the path to a FOLDER on the filesystem (not a file) constructor(public readonly path: string) {} - private safeAbsPath = (path: string) => { + safeAbsPath = (path: string) => { if (typeof path != "string") { throw Error(`path must be a string but is of type ${typeof path}`); } @@ -87,7 +94,11 @@ export class SandboxedFilesystem { return await readFile(this.safeAbsPath(path), encoding); }; - realpath = async (path: string) => { + readdir = async (path: string): Promise => { + return await readdir(this.safeAbsPath(path)); + }; + + realpath = async (path: string): Promise => { const x = await realpath(this.safeAbsPath(path)); return x.slice(this.path.length + 1); }; @@ -104,37 +115,6 @@ export class SandboxedFilesystem { await rmdir(this.safeAbsPath(path), options); }; - rsync = async ({ - src, - target, - timeout = 5 * 60 * 1000, - }: { - src: string; - target: string; - timeout?: number; - }): Promise<{ stdout: string; stderr: string; exit_code: number }> => { - let srcPath = this.safeAbsPath(src); - let targetPath = this.safeAbsPath(target); - if (src.endsWith("/")) { - srcPath += "/"; - } - if (target.endsWith("/")) { - targetPath += "/"; - } - if (!srcPath.endsWith("/") && (await isdir(srcPath))) { - srcPath += "/"; - if (!targetPath.endsWith("/")) { - targetPath += "/"; - } - } - return await sudo({ - command: "rsync", - args: [srcPath, targetPath], - err_on_exit: false, - timeout: timeout / 1000, - }); - }; - stat = async (path: string) => { return await stat(this.safeAbsPath(path)); }; diff --git a/src/packages/file-server/package.json b/src/packages/file-server/package.json index 58285c4bf6..c72ed98314 100644 --- a/src/packages/file-server/package.json +++ b/src/packages/file-server/package.json @@ -4,7 +4,9 @@ "description": "CoCalc File Server", "exports": { "./btrfs": "./dist/btrfs/index.js", - "./btrfs/*": "./dist/btrfs/*.js" + "./btrfs/*": "./dist/btrfs/*.js", + "./conat/*": "./dist/conat/*.js", + "./fs/*": "./dist/fs/*.js" }, "scripts": { "preinstall": "npx only-allow pnpm", @@ -14,20 +16,13 @@ "depcheck": "pnpx depcheck", "clean": "rm -rf node_modules dist" }, - "files": [ - "dist/**", - "README.md", - "package.json" - ], + "files": ["dist/**", "README.md", "package.json"], "author": "SageMath, Inc.", - "keywords": [ - "utilities", - "btrfs", - "cocalc" - ], + "keywords": ["utilities", "btrfs", "cocalc"], "license": "SEE LICENSE.md", "dependencies": { "@cocalc/backend": "workspace:*", + "@cocalc/conat": "workspace:*", "@cocalc/file-server": "workspace:*", "@cocalc/util": "workspace:*", "awaiting": "^3.0.0" diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 1fe4499fe0..ac022d9c8f 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -289,6 +289,9 @@ importers: '@cocalc/backend': specifier: workspace:* version: link:../backend + '@cocalc/conat': + specifier: workspace:* + version: link:../conat '@cocalc/file-server': specifier: workspace:* version: 'link:' @@ -15321,7 +15324,7 @@ snapshots: axios@1.10.0: dependencies: - follow-redirects: 1.15.9 + follow-redirects: 1.15.9(debug@4.4.1) form-data: 4.0.3 proxy-from-env: 1.1.0 transitivePeerDependencies: @@ -17457,8 +17460,6 @@ snapshots: dependencies: dtype: 2.0.0 - follow-redirects@1.15.9: {} - follow-redirects@1.15.9(debug@4.4.1): optionalDependencies: debug: 4.4.1 From cbe88607c4a5351d0b2f62941a3cc520e9ba3b56 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 17 Jul 2025 03:27:37 +0000 Subject: [PATCH 006/798] start writing unit tests for local-path fs --- .../file-server/conat/test/local-path.test.ts | 42 ++++++++++++++++++- src/packages/file-server/package.json | 2 +- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/src/packages/file-server/conat/test/local-path.test.ts b/src/packages/file-server/conat/test/local-path.test.ts index 4aa561af08..682cec7719 100644 --- a/src/packages/file-server/conat/test/local-path.test.ts +++ b/src/packages/file-server/conat/test/local-path.test.ts @@ -1,3 +1,43 @@ +import { localPathFileserver } from "../local-path"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "path"; +import { fsClient } from "@cocalc/conat/files/fs"; + +let tempDir; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc-local-path")); +}); + describe("use the simple fileserver", () => { - it("does nothing", async () => {}); + let service; + it("creates the simple fileserver service", async () => { + service = await localPathFileserver({ service: "fs", path: tempDir }); + }); + + const project_id = "6b851643-360e-435e-b87e-f9a6ab64a8b1"; + let fs; + it("create a client", () => { + fs = fsClient({ subject: `fs.project-${project_id}` }); + }); + + it("checks appendFile works", async () => { + await fs.appendFile("a", "foo"); + expect(await fs.readFile("a", "utf8")).toEqual("foo"); + }); + + it("checks chmod works", async () => { + await fs.writeFile("b", "hi"); + await fs.chmod("b", 0o755); + const s = await fs.stat("b"); + expect(s.mode.toString(8)).toBe("100755"); + }); + + it("closes the service", () => { + service.close(); + }); +}); + +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); }); diff --git a/src/packages/file-server/package.json b/src/packages/file-server/package.json index c72ed98314..55d9d9980a 100644 --- a/src/packages/file-server/package.json +++ b/src/packages/file-server/package.json @@ -12,7 +12,7 @@ "preinstall": "npx only-allow pnpm", "build": "pnpm exec tsc --build", "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", - "test": "pnpm exec jest", + "test": "pnpm exec jest --forceExit", "depcheck": "pnpx depcheck", "clean": "rm -rf node_modules dist" }, From 3e57982ebebf9fef87c44b4f0c55d9efd3e3b6e2 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 17 Jul 2025 04:41:39 +0000 Subject: [PATCH 007/798] file-server: implement more of the fs api and unit tests, and even stats.isDirectory(), etc. --- src/packages/conat/core/client.ts | 6 +- src/packages/conat/files/fs.ts | 101 +++++++++++++++--- .../file-server/conat/test/local-path.test.ts | 76 +++++++++++-- src/packages/file-server/fs/sandbox.ts | 10 ++ src/packages/frontend/conat/client.ts | 3 + 5 files changed, 176 insertions(+), 20 deletions(-) diff --git a/src/packages/conat/core/client.ts b/src/packages/conat/core/client.ts index 27e8f0bb4e..c0f541c91f 100644 --- a/src/packages/conat/core/client.ts +++ b/src/packages/conat/core/client.ts @@ -1133,7 +1133,11 @@ export class Client extends EventEmitter { return new Proxy( {}, { - get: (_, name) => { + get: (target, name) => { + const s = target[String(name)]; + if (s !== undefined) { + return s; + } if (typeof name !== "string") { return undefined; } diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index 048a491f91..e33d745dbe 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -4,10 +4,12 @@ import { conat } from "@cocalc/conat/client"; export interface Filesystem { appendFile: (path: string, data: string | Buffer, encoding?) => Promise; chmod: (path: string, mode: string | number) => Promise; + constants: () => Promise<{ [key: string]: number }>; copyFile: (src: string, dest: string) => Promise; cp: (src: string, dest: string, options?) => Promise; exists: (path: string) => Promise; link: (existingPath: string, newPath: string) => Promise; + lstat: (path: string) => Promise; mkdir: (path: string, options?) => Promise; readFile: (path: string, encoding?: any) => Promise; readdir: (path: string) => Promise; @@ -15,7 +17,7 @@ export interface Filesystem { rename: (oldPath: string, newPath: string) => Promise; rm: (path: string, options?) => Promise; rmdir: (path: string, options?) => Promise; - stat: (path: string) => Promise; + stat: (path: string) => Promise; symlink: (target: string, path: string) => Promise; truncate: (path: string, len?: number) => Promise; unlink: (path: string) => Promise; @@ -27,7 +29,7 @@ export interface Filesystem { writeFile: (path: string, data: string | Buffer) => Promise; } -export interface Stats { +interface IStats { dev: number; ino: number; mode: number; @@ -48,6 +50,48 @@ export interface Stats { birthtime: Date; } +class Stats { + dev: number; + ino: number; + mode: number; + nlink: number; + uid: number; + gid: number; + rdev: number; + size: number; + blksize: number; + blocks: number; + atimeMs: number; + mtimeMs: number; + ctimeMs: number; + birthtimeMs: number; + atime: Date; + mtime: Date; + ctime: Date; + birthtime: Date; + + constructor(private constants: { [key: string]: number }) {} + + isSymbolicLink = () => + (this.mode & this.constants.S_IFMT) === this.constants.S_IFLNK; + + isFile = () => (this.mode & this.constants.S_IFMT) === this.constants.S_IFREG; + + isDirectory = () => + (this.mode & this.constants.S_IFMT) === this.constants.S_IFDIR; + + isBlockDevice = () => + (this.mode & this.constants.S_IFMT) === this.constants.S_IFBLK; + + isCharacterDevice = () => + (this.mode & this.constants.S_IFMT) === this.constants.S_IFCHR; + + isFIFO = () => (this.mode & this.constants.S_IFMT) === this.constants.S_IFIFO; + + isSocket = () => + (this.mode & this.constants.S_IFMT) === this.constants.S_IFSOCK; +} + interface Options { service: string; client?: Client; @@ -64,6 +108,9 @@ export async function fsServer({ service, fs, client }: Options) { async chmod(path: string, mode: string | number) { await (await fs(this.subject)).chmod(path, mode); }, + async constants(): Promise<{ [key: string]: number }> { + return await (await fs(this.subject)).constants(); + }, async copyFile(src: string, dest: string) { await (await fs(this.subject)).copyFile(src, dest); }, @@ -71,11 +118,14 @@ export async function fsServer({ service, fs, client }: Options) { await (await fs(this.subject)).cp(src, dest, options); }, async exists(path: string) { - await (await fs(this.subject)).exists(path); + return await (await fs(this.subject)).exists(path); }, async link(existingPath: string, newPath: string) { await (await fs(this.subject)).link(existingPath, newPath); }, + async lstat(path: string): Promise { + return await (await fs(this.subject)).lstat(path); + }, async mkdir(path: string, options?) { await (await fs(this.subject)).mkdir(path, options); }, @@ -89,35 +139,35 @@ export async function fsServer({ service, fs, client }: Options) { return await (await fs(this.subject)).realpath(path); }, async rename(oldPath: string, newPath: string) { - return await (await fs(this.subject)).rename(oldPath, newPath); + await (await fs(this.subject)).rename(oldPath, newPath); }, async rm(path: string, options?) { - return await (await fs(this.subject)).rm(path, options); + await (await fs(this.subject)).rm(path, options); }, async rmdir(path: string, options?) { - return await (await fs(this.subject)).rmdir(path, options); + await (await fs(this.subject)).rmdir(path, options); }, - async stat(path: string): Promise { + async stat(path: string): Promise { return await (await fs(this.subject)).stat(path); }, async symlink(target: string, path: string) { - return await (await fs(this.subject)).symlink(target, path); + await (await fs(this.subject)).symlink(target, path); }, async truncate(path: string, len?: number) { - return await (await fs(this.subject)).truncate(path, len); + await (await fs(this.subject)).truncate(path, len); }, async unlink(path: string) { - return await (await fs(this.subject)).unlink(path); + await (await fs(this.subject)).unlink(path); }, async utimes( path: string, atime: number | string | Date, mtime: number | string | Date, ) { - return await (await fs(this.subject)).utimes(path, atime, mtime); + await (await fs(this.subject)).utimes(path, atime, mtime); }, async writeFile(path: string, data: string | Buffer) { - return await (await fs(this.subject)).writeFile(path, data); + await (await fs(this.subject)).writeFile(path, data); }, }, ); @@ -130,5 +180,30 @@ export function fsClient({ client?: Client; subject: string; }) { - return (client ?? conat()).call(subject); + let call = (client ?? conat()).call(subject); + + let constants: any = null; + const stat0 = call.stat.bind(call); + call.stat = async (path: string) => { + const s = await stat0(path); + constants = constants ?? (await call.constants()); + const stats = new Stats(constants); + for (const k in s) { + stats[k] = s[k]; + } + return stats; + }; + + const lstat0 = call.lstat.bind(call); + call.lstat = async (path: string) => { + const s = await lstat0(path); + constants = constants ?? (await call.constants()); + const stats = new Stats(constants); + for (const k in s) { + stats[k] = s[k]; + } + return stats; + }; + + return call; } diff --git a/src/packages/file-server/conat/test/local-path.test.ts b/src/packages/file-server/conat/test/local-path.test.ts index 682cec7719..875cbc34f0 100644 --- a/src/packages/file-server/conat/test/local-path.test.ts +++ b/src/packages/file-server/conat/test/local-path.test.ts @@ -3,6 +3,7 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "path"; import { fsClient } from "@cocalc/conat/files/fs"; +import { randomId } from "@cocalc/conat/names"; let tempDir; beforeAll(async () => { @@ -10,31 +11,94 @@ beforeAll(async () => { }); describe("use the simple fileserver", () => { - let service; + const service = `fs-${randomId()}`; + let server; it("creates the simple fileserver service", async () => { - service = await localPathFileserver({ service: "fs", path: tempDir }); + server = await localPathFileserver({ service, path: tempDir }); }); const project_id = "6b851643-360e-435e-b87e-f9a6ab64a8b1"; let fs; it("create a client", () => { - fs = fsClient({ subject: `fs.project-${project_id}` }); + fs = fsClient({ subject: `${service}.project-${project_id}` }); }); - it("checks appendFile works", async () => { + it("appendFile works", async () => { + await fs.writeFile("a", ""); await fs.appendFile("a", "foo"); expect(await fs.readFile("a", "utf8")).toEqual("foo"); }); - it("checks chmod works", async () => { + it("chmod works", async () => { await fs.writeFile("b", "hi"); await fs.chmod("b", 0o755); const s = await fs.stat("b"); expect(s.mode.toString(8)).toBe("100755"); }); + it("constants work", async () => { + const constants = await fs.constants(); + expect(constants.O_RDONLY).toBe(0); + expect(constants.O_WRONLY).toBe(1); + expect(constants.O_RDWR).toBe(2); + }); + + it("copyFile works", async () => { + await fs.writeFile("c", "hello"); + await fs.copyFile("c", "d.txt"); + expect(await fs.readFile("d.txt", "utf8")).toEqual("hello"); + }); + + it("cp works on a directory", async () => { + await fs.mkdir("folder"); + await fs.writeFile("folder/a.txt", "hello"); + await fs.cp("folder", "folder2", { recursive: true }); + expect(await fs.readFile("folder2/a.txt", "utf8")).toEqual("hello"); + }); + + it("exists works", async () => { + expect(await fs.exists("does-not-exist")).toBe(false); + await fs.writeFile("does-exist", ""); + expect(await fs.exists("does-exist")).toBe(true); + }); + + it("creating a hard link works", async () => { + await fs.writeFile("source", "the source"); + await fs.link("source", "target"); + expect(await fs.readFile("target", "utf8")).toEqual("the source"); + // hard link, not symlink + expect(await fs.realpath("target")).toBe("target"); + + await fs.appendFile("source", " and more"); + expect(await fs.readFile("target", "utf8")).toEqual("the source and more"); + }); + + it("mkdir works", async () => { + await fs.mkdir("xyz"); + const s = await fs.stat("xyz"); + expect(s.isDirectory()).toBe(true); + }); + + it("creating a symlink works", async () => { + await fs.writeFile("source1", "the source"); + await fs.symlink("source1", "target1"); + expect(await fs.readFile("target1", "utf8")).toEqual("the source"); + // symlink, not hard + expect(await fs.realpath("target1")).toBe("source1"); + await fs.appendFile("source1", " and more"); + expect(await fs.readFile("target1", "utf8")).toEqual("the source and more"); + const stats = await fs.stat("target1"); + expect(stats.isSymbolicLink()).toBe(false); + + const lstats = await fs.lstat("target1"); + expect(lstats.isSymbolicLink()).toBe(true); + + const stats0 = await fs.stat("source1"); + expect(stats0.isSymbolicLink()).toBe(false); + }); + it("closes the service", () => { - service.close(); + server.close(); }); }); diff --git a/src/packages/file-server/fs/sandbox.ts b/src/packages/file-server/fs/sandbox.ts index dabd277ec2..4c5d7de83b 100644 --- a/src/packages/file-server/fs/sandbox.ts +++ b/src/packages/file-server/fs/sandbox.ts @@ -15,8 +15,10 @@ import { appendFile, chmod, cp, + constants, copyFile, link, + lstat, readdir, readFile, realpath, @@ -56,6 +58,10 @@ export class SandboxedFilesystem { await chmod(this.safeAbsPath(path), mode); }; + constants = async (): Promise<{ [key: string]: number }> => { + return constants; + }; + copyFile = async (src: string, dest: string) => { await copyFile(this.safeAbsPath(src), this.safeAbsPath(dest)); }; @@ -86,6 +92,10 @@ export class SandboxedFilesystem { }); }; + lstat = async (path: string) => { + return await lstat(this.safeAbsPath(path)); + }; + mkdir = async (path: string, options?) => { await mkdir(this.safeAbsPath(path), options); }; diff --git a/src/packages/frontend/conat/client.ts b/src/packages/frontend/conat/client.ts index 628522f8d2..bf7af5088d 100644 --- a/src/packages/frontend/conat/client.ts +++ b/src/packages/frontend/conat/client.ts @@ -46,6 +46,7 @@ import { deleteRememberMe, setRememberMe, } from "@cocalc/frontend/misc/remember-me"; +import { fsClient } from "@cocalc/conat/files/fs"; export interface ConatConnectionStatus { state: "connected" | "disconnected"; @@ -515,6 +516,8 @@ export class ConatClient extends EventEmitter { }; refCacheInfo = () => refCacheInfo(); + + fsClient = (subject: string) => fsClient({ subject, client: this.conat() }); } function setDeleted({ project_id, path, deleted }) { From e146dd18579cecbc391f5b1ce128fd0adcadf088 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 17 Jul 2025 18:26:51 +0000 Subject: [PATCH 008/798] btrfs: enable and fix some disabled tests --- src/packages/file-server/btrfs/subvolume-snapshots.ts | 2 +- src/packages/file-server/btrfs/test/subvolume.test.ts | 4 ++-- src/packages/file-server/conat/test/local-path.test.ts | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/packages/file-server/btrfs/subvolume-snapshots.ts b/src/packages/file-server/btrfs/subvolume-snapshots.ts index ffe71fe6fc..9dcd4f30ad 100644 --- a/src/packages/file-server/btrfs/subvolume-snapshots.ts +++ b/src/packages/file-server/btrfs/subvolume-snapshots.ts @@ -27,7 +27,7 @@ export class SubvolumeSnapshots { return; } await this.subvolume.fs.mkdir(SNAPSHOTS); - await this.subvolume.fs.chmod(SNAPSHOTS, "0550"); + await this.subvolume.fs.chmod(SNAPSHOTS, "0700"); }; create = async (name?: string) => { diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index e586cc656b..72f72c4dfd 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -226,7 +226,7 @@ describe("test snapshots", () => { }); }); -describe.only("test bup backups", () => { +describe("test bup backups", () => { let vol: Subvolume; it("creates a volume", async () => { vol = await fs.subvolumes.get("bup-test"); @@ -273,7 +273,7 @@ describe.only("test bup backups", () => { { name: "mydir", size: 0, mtime: x[1].mtime, isdir: true }, ]); expect(Math.abs((x[0].mtime ?? 0) * 1000 - Date.now())).toBeLessThan( - 60_000, + 5 * 60_000, ); }); diff --git a/src/packages/file-server/conat/test/local-path.test.ts b/src/packages/file-server/conat/test/local-path.test.ts index 875cbc34f0..4a66847a60 100644 --- a/src/packages/file-server/conat/test/local-path.test.ts +++ b/src/packages/file-server/conat/test/local-path.test.ts @@ -77,9 +77,10 @@ describe("use the simple fileserver", () => { await fs.mkdir("xyz"); const s = await fs.stat("xyz"); expect(s.isDirectory()).toBe(true); + expect(s.isFile()).toBe(false); }); - it("creating a symlink works", async () => { + it("creating a symlink works (and using lstat)", async () => { await fs.writeFile("source1", "the source"); await fs.symlink("source1", "target1"); expect(await fs.readFile("target1", "utf8")).toEqual("the source"); From a7a0d95c0e960bb0f33928517c1aa2b6a4068c4a Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 18 Jul 2025 14:29:29 +0000 Subject: [PATCH 009/798] fix file-server conat test to work with self-contained testing conat server --- src/packages/file-server/conat/local-path.ts | 6 ++++-- src/packages/file-server/conat/test/local-path.test.ts | 5 ++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/packages/file-server/conat/local-path.ts b/src/packages/file-server/conat/local-path.ts index e8450f3adf..53edc7443b 100644 --- a/src/packages/file-server/conat/local-path.ts +++ b/src/packages/file-server/conat/local-path.ts @@ -1,18 +1,20 @@ import { fsServer } from "@cocalc/conat/files/fs"; -import { conat } from "@cocalc/backend/conat"; import { SandboxedFilesystem } from "@cocalc/file-server/fs/sandbox"; import { mkdir } from "fs/promises"; import { join } from "path"; import { isValidUUID } from "@cocalc/util/misc"; +import { type Client, getClient } from "@cocalc/conat/core/client"; export function localPathFileserver({ service, path, + client, }: { service: string; path: string; + client?: Client; }) { - const client = conat(); + client ??= getClient(); const server = fsServer({ service, client, diff --git a/src/packages/file-server/conat/test/local-path.test.ts b/src/packages/file-server/conat/test/local-path.test.ts index 4a66847a60..0cd4a92718 100644 --- a/src/packages/file-server/conat/test/local-path.test.ts +++ b/src/packages/file-server/conat/test/local-path.test.ts @@ -4,9 +4,11 @@ import { tmpdir } from "node:os"; import { join } from "path"; import { fsClient } from "@cocalc/conat/files/fs"; import { randomId } from "@cocalc/conat/names"; +import { before, after, client } from "@cocalc/backend/conat/test/setup"; let tempDir; beforeAll(async () => { + await before(); tempDir = await mkdtemp(join(tmpdir(), "cocalc-local-path")); }); @@ -14,7 +16,7 @@ describe("use the simple fileserver", () => { const service = `fs-${randomId()}`; let server; it("creates the simple fileserver service", async () => { - server = await localPathFileserver({ service, path: tempDir }); + server = await localPathFileserver({ client, service, path: tempDir }); }); const project_id = "6b851643-360e-435e-b87e-f9a6ab64a8b1"; @@ -104,5 +106,6 @@ describe("use the simple fileserver", () => { }); afterAll(async () => { + await after(); await rm(tempDir, { force: true, recursive: true }); }); From 732ee7289ebda1db64668e3df5ba1101739c1361 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 18 Jul 2025 16:59:50 +0000 Subject: [PATCH 010/798] btrfs: add sync command in hopes of getting tests to pass on gitub CI --- src/packages/file-server/btrfs/filesystem.ts | 4 ++++ .../file-server/btrfs/test/filesystem-stress.test.ts | 8 ++++++++ src/packages/file-server/conat/test/local-path.test.ts | 2 ++ 3 files changed, 14 insertions(+) diff --git a/src/packages/file-server/btrfs/filesystem.ts b/src/packages/file-server/btrfs/filesystem.ts index 927fb23548..194988e036 100644 --- a/src/packages/file-server/btrfs/filesystem.ts +++ b/src/packages/file-server/btrfs/filesystem.ts @@ -71,6 +71,10 @@ export class Filesystem { await this.initBup(); }; + sync = async () => { + await btrfs({ args: ["filesystem", "sync", this.opts.mount] }); + }; + unmount = async () => { await sudo({ command: "umount", diff --git a/src/packages/file-server/btrfs/test/filesystem-stress.test.ts b/src/packages/file-server/btrfs/test/filesystem-stress.test.ts index a2e1de9531..354f261801 100644 --- a/src/packages/file-server/btrfs/test/filesystem-stress.test.ts +++ b/src/packages/file-server/btrfs/test/filesystem-stress.test.ts @@ -48,6 +48,7 @@ describe("stress operations with subvolumes", () => { }); it("clone the first group in serial", async () => { + await fs.sync(); // needed on github actions const t = Date.now(); for (let i = 0; i < count1; i++) { await fs.subvolumes.clone(`${i}`, `clone-of-${i}`); @@ -58,6 +59,7 @@ describe("stress operations with subvolumes", () => { }); it("clone the second group in parallel", async () => { + await fs.sync(); // needed on github actions const t = Date.now(); const v: any[] = []; for (let i = 0; i < count2; i++) { @@ -90,6 +92,12 @@ describe("stress operations with subvolumes", () => { `deleted ${Math.round((count2 / (Date.now() - t)) * 1000)} subvolumes per second in parallel`, ); }); + + it("everything should be gone except the clones", async () => { + await fs.sync(); + const v = await fs.subvolumes.list(); + expect(v.length).toBe(count1 + count2); + }); }); afterAll(after); diff --git a/src/packages/file-server/conat/test/local-path.test.ts b/src/packages/file-server/conat/test/local-path.test.ts index 0cd4a92718..44b717cca8 100644 --- a/src/packages/file-server/conat/test/local-path.test.ts +++ b/src/packages/file-server/conat/test/local-path.test.ts @@ -99,6 +99,8 @@ describe("use the simple fileserver", () => { const stats0 = await fs.stat("source1"); expect(stats0.isSymbolicLink()).toBe(false); }); + + it("closes the service", () => { server.close(); From 4644d6a308dab8c5519014c8bd1b14fa7e1e892a Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 18 Jul 2025 19:02:41 +0000 Subject: [PATCH 011/798] address obvious symlink issue with fs sandbox; also make tests hopefully work on github --- src/package.json | 2 +- src/packages/file-server/btrfs/filesystem.ts | 1 + .../file-server/btrfs/subvolume-bup.ts | 8 +- src/packages/file-server/btrfs/subvolume.ts | 4 +- .../btrfs/test/filesystem-stress.test.ts | 3 - .../file-server/btrfs/test/subvolume.test.ts | 2 +- .../file-server/conat/test/local-path.test.ts | 90 +++++++++++++- src/packages/file-server/fs/sandbox.ts | 113 ++++++++++++++---- src/packages/file-server/package.json | 1 + src/workspaces.py | 27 +++-- 10 files changed, 200 insertions(+), 51 deletions(-) diff --git a/src/package.json b/src/package.json index efa416dcd0..8f4ce63e09 100644 --- a/src/package.json +++ b/src/package.json @@ -18,7 +18,7 @@ "version-check": "pip3 install typing_extensions mypy || pip3 install --break-system-packages typing_extensions mypy && ./workspaces.py version-check && mypy scripts/check_npm_packages.py", "test-parallel": "unset DEBUG && pnpm run version-check && cd packages && pnpm run -r --parallel test", "test": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test", - "test-github-ci": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test --exclude=jupyter --retries=1", + "test-github-ci": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test --test-github-ci --exclude=jupyter --retries=1", "depcheck": "cd packages && pnpm run -r --parallel depcheck", "prettier-all": "cd packages/", "local-ci": "./scripts/ci.sh", diff --git a/src/packages/file-server/btrfs/filesystem.ts b/src/packages/file-server/btrfs/filesystem.ts index 194988e036..248e086630 100644 --- a/src/packages/file-server/btrfs/filesystem.ts +++ b/src/packages/file-server/btrfs/filesystem.ts @@ -69,6 +69,7 @@ export class Filesystem { args: ["quota", "enable", "--simple", this.opts.mount], }); await this.initBup(); + await this.sync(); }; sync = async () => { diff --git a/src/packages/file-server/btrfs/subvolume-bup.ts b/src/packages/file-server/btrfs/subvolume-bup.ts index 3849b49379..b4d64c9b5a 100644 --- a/src/packages/file-server/btrfs/subvolume-bup.ts +++ b/src/packages/file-server/btrfs/subvolume-bup.ts @@ -46,7 +46,7 @@ export class SubvolumeBup { `createBackup: creating ${BUP_SNAPSHOT} to get a consistent backup`, ); await this.subvolume.snapshots.create(BUP_SNAPSHOT); - const target = this.subvolume.fs.safeAbsPath( + const target = await this.subvolume.fs.safeAbsPath( this.subvolume.snapshots.path(BUP_SNAPSHOT), ); @@ -133,9 +133,9 @@ export class SubvolumeBup { return v; } - path = this.subvolume.fs - .safeAbsPath(path) - .slice(this.subvolume.path.length); + path = (await this.subvolume.fs.safeAbsPath(path)).slice( + this.subvolume.path.length, + ); const { stdout } = await sudo({ command: "bup", args: [ diff --git a/src/packages/file-server/btrfs/subvolume.ts b/src/packages/file-server/btrfs/subvolume.ts index 72c3dd7d54..7eb7808518 100644 --- a/src/packages/file-server/btrfs/subvolume.ts +++ b/src/packages/file-server/btrfs/subvolume.ts @@ -86,8 +86,8 @@ export class Subvolume { target: string; timeout?: number; }): Promise<{ stdout: string; stderr: string; exit_code: number }> => { - let srcPath = this.fs.safeAbsPath(src); - let targetPath = this.fs.safeAbsPath(target); + let srcPath = await this.fs.safeAbsPath(src); + let targetPath = await this.fs.safeAbsPath(target); if (src.endsWith("/")) { srcPath += "/"; } diff --git a/src/packages/file-server/btrfs/test/filesystem-stress.test.ts b/src/packages/file-server/btrfs/test/filesystem-stress.test.ts index 354f261801..2302785d14 100644 --- a/src/packages/file-server/btrfs/test/filesystem-stress.test.ts +++ b/src/packages/file-server/btrfs/test/filesystem-stress.test.ts @@ -48,7 +48,6 @@ describe("stress operations with subvolumes", () => { }); it("clone the first group in serial", async () => { - await fs.sync(); // needed on github actions const t = Date.now(); for (let i = 0; i < count1; i++) { await fs.subvolumes.clone(`${i}`, `clone-of-${i}`); @@ -59,7 +58,6 @@ describe("stress operations with subvolumes", () => { }); it("clone the second group in parallel", async () => { - await fs.sync(); // needed on github actions const t = Date.now(); const v: any[] = []; for (let i = 0; i < count2; i++) { @@ -94,7 +92,6 @@ describe("stress operations with subvolumes", () => { }); it("everything should be gone except the clones", async () => { - await fs.sync(); const v = await fs.subvolumes.list(); expect(v.length).toBe(count1 + count2); }); diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index 72f72c4dfd..d7e4bdface 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -143,7 +143,7 @@ describe("the filesystem operations", () => { await vol.fs.writeFile("w.txt", "hi"); const ac = new AbortController(); const { signal } = ac; - const watcher = vol.fs.watch("w.txt", { signal }); + const watcher = await vol.fs.watch("w.txt", { signal }); vol.fs.appendFile("w.txt", " there"); // @ts-ignore const { value, done } = await watcher.next(); diff --git a/src/packages/file-server/conat/test/local-path.test.ts b/src/packages/file-server/conat/test/local-path.test.ts index 44b717cca8..a1a9013a4c 100644 --- a/src/packages/file-server/conat/test/local-path.test.ts +++ b/src/packages/file-server/conat/test/local-path.test.ts @@ -1,25 +1,28 @@ import { localPathFileserver } from "../local-path"; -import { mkdtemp, rm } from "node:fs/promises"; +import { mkdtemp, readFile, rm, symlink } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "path"; import { fsClient } from "@cocalc/conat/files/fs"; import { randomId } from "@cocalc/conat/names"; import { before, after, client } from "@cocalc/backend/conat/test/setup"; +import { uuid } from "@cocalc/util/misc"; let tempDir; +let tempDir2; beforeAll(async () => { await before(); tempDir = await mkdtemp(join(tmpdir(), "cocalc-local-path")); + tempDir2 = await mkdtemp(join(tmpdir(), "cocalc-local-path-2")); }); -describe("use the simple fileserver", () => { +describe("use all the standard api functions of fs", () => { const service = `fs-${randomId()}`; let server; it("creates the simple fileserver service", async () => { server = await localPathFileserver({ client, service, path: tempDir }); }); - const project_id = "6b851643-360e-435e-b87e-f9a6ab64a8b1"; + const project_id = uuid(); let fs; it("create a client", () => { fs = fsClient({ subject: `${service}.project-${project_id}` }); @@ -82,6 +85,27 @@ describe("use the simple fileserver", () => { expect(s.isFile()).toBe(false); }); + it("readFile works", async () => { + await fs.writeFile("a", Buffer.from([1, 2, 3])); + const s = await fs.readFile("a"); + expect(s).toEqual(Buffer.from([1, 2, 3])); + + await fs.writeFile("b.txt", "conat"); + const t = await fs.readFile("b.txt", "utf8"); + expect(t).toEqual("conat"); + }); + + it("readdir works", async () => { + await fs.mkdir("dirtest"); + for (let i = 0; i < 5; i++) { + await fs.writeFile(`dirtest/${i}`, `${i}`); + } + const fire = "🔥.txt"; + await fs.writeFile(join("dirtest", fire), "this is ️‍🔥!"); + const v = await fs.readdir("dirtest"); + expect(v).toEqual(["0", "1", "2", "3", "4", fire]); + }); + it("creating a symlink works (and using lstat)", async () => { await fs.writeFile("source1", "the source"); await fs.symlink("source1", "target1"); @@ -99,15 +123,71 @@ describe("use the simple fileserver", () => { const stats0 = await fs.stat("source1"); expect(stats0.isSymbolicLink()).toBe(false); }); - - it("closes the service", () => { server.close(); }); }); +describe("security: dangerous symlinks can't be followed", () => { + const service = `fs-${randomId()}`; + let server; + it("creates the simple fileserver service", async () => { + server = await localPathFileserver({ client, service, path: tempDir2 }); + }); + + const project_id = uuid(); + const project_id2 = uuid(); + let fs, fs2; + it("create two clients", () => { + fs = fsClient({ subject: `${service}.project-${project_id}` }); + fs2 = fsClient({ subject: `${service}.project-${project_id2}` }); + }); + + it("create a secret in one", async () => { + await fs.writeFile("password", "s3cr3t"); + await fs2.writeFile("a", "init"); + }); + + // This is setup bypassing security and is part of our threat model, due to users + // having full access internally to their sandbox fs. + it("directly create a file that is a symlink outside of the sandbox -- this should work", async () => { + await symlink( + join(tempDir2, project_id, "password"), + join(tempDir2, project_id2, "link"), + ); + const s = await readFile(join(tempDir2, project_id2, "link"), "utf8"); + expect(s).toBe("s3cr3t"); + }); + + it("fails to read the symlink content via the api", async () => { + await expect(async () => { + await fs2.readFile("link", "utf8"); + }).rejects.toThrow("outside of sandbox"); + }); + + it("directly create a relative symlink ", async () => { + await symlink( + join("..", project_id, "password"), + join(tempDir2, project_id2, "link2"), + ); + const s = await readFile(join(tempDir2, project_id2, "link2"), "utf8"); + expect(s).toBe("s3cr3t"); + }); + + it("fails to read the relative symlink content via the api", async () => { + await expect(async () => { + await fs2.readFile("link2", "utf8"); + }).rejects.toThrow("outside of sandbox"); + }); + + it("closes the server", () => { + server.close(); + }); +}); + afterAll(async () => { await after(); await rm(tempDir, { force: true, recursive: true }); + // await rm(tempDir2, { force: true, recursive: true }); }); diff --git a/src/packages/file-server/fs/sandbox.ts b/src/packages/file-server/fs/sandbox.ts index 4c5d7de83b..f3795937c0 100644 --- a/src/packages/file-server/fs/sandbox.ts +++ b/src/packages/file-server/fs/sandbox.ts @@ -9,6 +9,36 @@ Absolute and relative paths are considered as relative to the input folder path. REFERENCE: We don't use https://github.com/metarhia/sandboxed-fs, but did look at the code. + + + +SECURITY: + +The following could be a big problem -- user somehow create or change path to +be a dangerous symlink *after* the realpath check below, but before we do an fs *read* +operation. If they did that, then we would end up reading the target of the +symlink. I.e., if they could somehow create the file *as an unsafe symlink* +right after we confirm that it does not exist and before we read from it. This +would only happen via something not involving this sandbox, e.g., the filesystem +mounted into a container some other way. + +In short, I'm worried about: + +1. Request to read a file named "link" which is just a normal file. We confirm this using realpath + in safeAbsPath. +2. Somehow delete "link" and replace it by a new file that is a symlink to "../{project_id}/.ssh/id_ed25519" +3. Read the file "link" and get the contents of "../{project_id}/.ssh/id_ed25519". + +The problem is that 1 and 3 happen microseconds apart as separate calls to the filesystem. + +**[ ] TODO -- NOT IMPLEMENTED YET: This is why we have to uses file descriptors!** + +1. User requests to read a file named "link" which is just a normal file. +2. We wet file descriptor fd for whatever "link" is. Then confirm this is OK using realpath in safeAbsPath. +3. user somehow deletes "link" and replace it by a new file that is a symlink to "../{project_id}/.ssh/id_ed25519" +4. We read from the file descriptor fd and get the contents of original "link" (or error). + + */ import { @@ -43,19 +73,40 @@ export class SandboxedFilesystem { // path should be the path to a FOLDER on the filesystem (not a file) constructor(public readonly path: string) {} - safeAbsPath = (path: string) => { + safeAbsPath = async (path: string): Promise => { if (typeof path != "string") { throw Error(`path must be a string but is of type ${typeof path}`); } - return join(this.path, resolve("/", path)); + // pathInSandbox is *definitely* a path in the sandbox: + const pathInSandbox = join(this.path, resolve("/", path)); + // However, there is still one threat, which is that it could + // be a path to an existing link that goes out of the sandbox. So + // we resolve to the realpath: + try { + const p = await realpath(pathInSandbox); + if (p != this.path && !p.startsWith(this.path + "/")) { + throw Error( + `realpath of '${path}' resolves to a path outside of sandbox`, + ); + } + // don't return the result of calling realpath -- what's important is + // their path's realpath is in the sandbox. + return pathInSandbox; + } catch (err) { + if (err.code == "ENOENT") { + return pathInSandbox; + } else { + throw err; + } + } }; appendFile = async (path: string, data: string | Buffer, encoding?) => { - return await appendFile(this.safeAbsPath(path), data, encoding); + return await appendFile(await this.safeAbsPath(path), data, encoding); }; chmod = async (path: string, mode: string | number) => { - await chmod(this.safeAbsPath(path), mode); + await chmod(await this.safeAbsPath(path), mode); }; constants = async (): Promise<{ [key: string]: number }> => { @@ -63,22 +114,26 @@ export class SandboxedFilesystem { }; copyFile = async (src: string, dest: string) => { - await copyFile(this.safeAbsPath(src), this.safeAbsPath(dest)); + await copyFile(await this.safeAbsPath(src), await this.safeAbsPath(dest)); }; cp = async (src: string, dest: string, options?) => { - await cp(this.safeAbsPath(src), this.safeAbsPath(dest), options); + await cp( + await this.safeAbsPath(src), + await this.safeAbsPath(dest), + options, + ); }; exists = async (path: string) => { - return await exists(this.safeAbsPath(path)); + return await exists(await this.safeAbsPath(path)); }; // hard link link = async (existingPath: string, newPath: string) => { return await link( - this.safeAbsPath(existingPath), - this.safeAbsPath(newPath), + await this.safeAbsPath(existingPath), + await this.safeAbsPath(newPath), ); }; @@ -86,59 +141,65 @@ export class SandboxedFilesystem { path: string, { hidden, limit }: { hidden?: boolean; limit?: number } = {}, ): Promise => { - return await getListing(this.safeAbsPath(path), hidden, { + return await getListing(await this.safeAbsPath(path), hidden, { limit, home: "/", }); }; lstat = async (path: string) => { - return await lstat(this.safeAbsPath(path)); + return await lstat(await this.safeAbsPath(path)); }; mkdir = async (path: string, options?) => { - await mkdir(this.safeAbsPath(path), options); + await mkdir(await this.safeAbsPath(path), options); }; readFile = async (path: string, encoding?: any): Promise => { - return await readFile(this.safeAbsPath(path), encoding); + return await readFile(await this.safeAbsPath(path), encoding); }; readdir = async (path: string): Promise => { - return await readdir(this.safeAbsPath(path)); + return await readdir(await this.safeAbsPath(path)); }; realpath = async (path: string): Promise => { - const x = await realpath(this.safeAbsPath(path)); + const x = await realpath(await this.safeAbsPath(path)); return x.slice(this.path.length + 1); }; rename = async (oldPath: string, newPath: string) => { - await rename(this.safeAbsPath(oldPath), this.safeAbsPath(newPath)); + await rename( + await this.safeAbsPath(oldPath), + await this.safeAbsPath(newPath), + ); }; rm = async (path: string, options?) => { - await rm(this.safeAbsPath(path), options); + await rm(await this.safeAbsPath(path), options); }; rmdir = async (path: string, options?) => { - await rmdir(this.safeAbsPath(path), options); + await rmdir(await this.safeAbsPath(path), options); }; stat = async (path: string) => { - return await stat(this.safeAbsPath(path)); + return await stat(await this.safeAbsPath(path)); }; symlink = async (target: string, path: string) => { - return await symlink(this.safeAbsPath(target), this.safeAbsPath(path)); + return await symlink( + await this.safeAbsPath(target), + await this.safeAbsPath(path), + ); }; truncate = async (path: string, len?: number) => { - await truncate(this.safeAbsPath(path), len); + await truncate(await this.safeAbsPath(path), len); }; unlink = async (path: string) => { - await unlink(this.safeAbsPath(path)); + await unlink(await this.safeAbsPath(path)); }; utimes = async ( @@ -146,14 +207,14 @@ export class SandboxedFilesystem { atime: number | string | Date, mtime: number | string | Date, ) => { - await utimes(this.safeAbsPath(path), atime, mtime); + await utimes(await this.safeAbsPath(path), atime, mtime); }; - watch = (filename: string, options?) => { - return watch(this.safeAbsPath(filename), options); + watch = async (filename: string, options?) => { + return watch(await this.safeAbsPath(filename), options); }; writeFile = async (path: string, data: string | Buffer) => { - return await writeFile(this.safeAbsPath(path), data); + return await writeFile(await this.safeAbsPath(path), data); }; } diff --git a/src/packages/file-server/package.json b/src/packages/file-server/package.json index 55d9d9980a..e12191d181 100644 --- a/src/packages/file-server/package.json +++ b/src/packages/file-server/package.json @@ -13,6 +13,7 @@ "build": "pnpm exec tsc --build", "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", "test": "pnpm exec jest --forceExit", + "test-github-ci": "pnpm exec jest --maxWorkers=1 --forceExit", "depcheck": "pnpx depcheck", "clean": "rm -rf node_modules dist" }, diff --git a/src/workspaces.py b/src/workspaces.py index da60e90a8d..57574af6d5 100755 --- a/src/workspaces.py +++ b/src/workspaces.py @@ -284,11 +284,7 @@ def test(args) -> None: success = [] def status(): - print("Status: ", { - "flaky": flaky, - "fails": fails, - "success": success - }) + print("Status: ", {"flaky": flaky, "fails": fails, "success": success}) v = packages(args) v.sort() @@ -307,11 +303,15 @@ def f(): print(f"TESTING {n}/{len(v)}: {path}") print("*") print("*" * 40) - cmd("pnpm run --if-present test", package_path) + if args.test_github_ci and 'test-github-ci' in open( + os.path.join(package_path, 'package.json')).read(): + cmd("pnpm run test-github-ci", package_path) + else: + cmd("pnpm run --if-present test", package_path) success.append(path) worked = False - for i in range(args.retries+1): + for i in range(args.retries + 1): try: f() worked = True @@ -325,7 +325,9 @@ def f(): flaky.append(path) print(f"ERROR testing {path}") if args.retries - i >= 1: - print(f"Trying {path} again at most {args.retries - i} more times") + print( + f"Trying {path} again at most {args.retries - i} more times" + ) if not worked: fails.append(path) @@ -577,7 +579,14 @@ def packages_arg(parser): "--retries", type=int, default=2, - help="how many times to retry a failed test suite before giving up; set to 0 to NOT retry") + help= + "how many times to retry a failed test suite before giving up; set to 0 to NOT retry" + ) + subparser.add_argument( + '--test-github-ci', + const=True, + action="store_const", + help="run 'pnpm test-github-ci' if available instead of 'pnpm test'") packages_arg(subparser) subparser.set_defaults(func=test) From 02cc5f2214b17e037410dcb5d10a0c1cdc14d21b Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 18 Jul 2025 22:44:51 +0000 Subject: [PATCH 012/798] finish testing conat fs interface and fix some subtle issues with stat found when testing --- src/packages/conat/files/fs.ts | 11 +- .../file-server/btrfs/test/subvolume.test.ts | 2 + .../file-server/conat/test/local-path.test.ts | 163 ++++++++++++++++-- 3 files changed, 164 insertions(+), 12 deletions(-) diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index e33d745dbe..506f3823b5 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -148,7 +148,16 @@ export async function fsServer({ service, fs, client }: Options) { await (await fs(this.subject)).rmdir(path, options); }, async stat(path: string): Promise { - return await (await fs(this.subject)).stat(path); + const s = await (await fs(this.subject)).stat(path); + return { + ...s, + // for some reason these times get corrupted on transport from the nodejs datastructure, + // so we make them standard Date objects. + atime: new Date(s.atime), + mtime: new Date(s.mtime), + ctime: new Date(s.ctime), + birthtime: new Date(s.birthtime), + }; }, async symlink(target: string, path: string) { await (await fs(this.subject)).symlink(target, path); diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index d7e4bdface..ecc5ed9756 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -126,6 +126,7 @@ describe("the filesystem operations", () => { it("make a file readonly, then change it back", async () => { await vol.fs.writeFile("c.txt", "hi"); await vol.fs.chmod("c.txt", "440"); + await fs.sync(); expect(async () => { await vol.fs.appendFile("c.txt", " there"); }).rejects.toThrow("EACCES"); @@ -219,6 +220,7 @@ describe("test snapshots", () => { }); it("unlock our snapshot and delete it", async () => { + await fs.sync(); await vol.snapshots.unlock("snap1"); await vol.snapshots.delete("snap1"); expect(await vol.snapshots.exists("snap1")).toBe(false); diff --git a/src/packages/file-server/conat/test/local-path.test.ts b/src/packages/file-server/conat/test/local-path.test.ts index a1a9013a4c..d53f72c0f6 100644 --- a/src/packages/file-server/conat/test/local-path.test.ts +++ b/src/packages/file-server/conat/test/local-path.test.ts @@ -1,5 +1,5 @@ import { localPathFileserver } from "../local-path"; -import { mkdtemp, readFile, rm, symlink } from "node:fs/promises"; +import { link, mkdtemp, readFile, rm, symlink } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "path"; import { fsClient } from "@cocalc/conat/files/fs"; @@ -106,7 +106,131 @@ describe("use all the standard api functions of fs", () => { expect(v).toEqual(["0", "1", "2", "3", "4", fire]); }); - it("creating a symlink works (and using lstat)", async () => { + it("realpath works", async () => { + await fs.writeFile("file0", "file0"); + await fs.symlink("file0", "file1"); + expect(await fs.readFile("file1", "utf8")).toBe("file0"); + const r = await fs.realpath("file1"); + expect(r).toBe("file0"); + + await fs.writeFile("file2", "file2"); + await fs.link("file2", "file3"); + expect(await fs.readFile("file3", "utf8")).toBe("file2"); + const r3 = await fs.realpath("file3"); + expect(r3).toBe("file3"); + }); + + it("rename a file", async () => { + await fs.writeFile("bella", "poo"); + await fs.rename("bella", "bells"); + expect(await fs.readFile("bells", "utf8")).toBe("poo"); + await fs.mkdir("x"); + await fs.rename("bells", "x/belltown"); + }); + + it("rm a file", async () => { + await fs.writeFile("bella-to-rm", "poo"); + await fs.rm("bella-to-rm"); + expect(await fs.exists("bella-to-rm")).toBe(false); + }); + + it("rm a directory", async () => { + await fs.mkdir("rm-dir"); + expect(async () => { + await fs.rm("rm-dir"); + }).rejects.toThrow("Path is a directory"); + await fs.rm("rm-dir", { recursive: true }); + expect(await fs.exists("rm-dir")).toBe(false); + }); + + it("rm a nonempty directory", async () => { + await fs.mkdir("rm-dir2"); + await fs.writeFile("rm-dir2/a", "a"); + await fs.rm("rm-dir2", { recursive: true }); + expect(await fs.exists("rm-dir2")).toBe(false); + }); + + it("rmdir empty directory", async () => { + await fs.mkdir("rm-dir3"); + await fs.rmdir("rm-dir3"); + expect(await fs.exists("rm-dir3")).toBe(false); + }); + + it("stat not existing path", async () => { + expect(async () => { + await fs.stat(randomId()); + }).rejects.toThrow("no such file or directory"); + }); + + it("stat a file", async () => { + await fs.writeFile("abc.txt", "hi"); + const stat = await fs.stat("abc.txt"); + expect(stat.size).toBe(2); + expect(stat.isFile()).toBe(true); + expect(stat.isSymbolicLink()).toBe(false); + expect(stat.isDirectory()).toBe(false); + expect(stat.isBlockDevice()).toBe(false); + expect(stat.isCharacterDevice()).toBe(false); + expect(stat.isSymbolicLink()).toBe(false); + expect(stat.isFIFO()).toBe(false); + expect(stat.isSocket()).toBe(false); + }); + + it("stat a directory", async () => { + await fs.mkdir("my-stat-dir"); + const stat = await fs.stat("my-stat-dir"); + expect(stat.isFile()).toBe(false); + expect(stat.isSymbolicLink()).toBe(false); + expect(stat.isDirectory()).toBe(true); + expect(stat.isBlockDevice()).toBe(false); + expect(stat.isCharacterDevice()).toBe(false); + expect(stat.isSymbolicLink()).toBe(false); + expect(stat.isFIFO()).toBe(false); + expect(stat.isSocket()).toBe(false); + }); + + it("stat a symlink", async () => { + await fs.writeFile("sl2", "the source"); + await fs.symlink("sl2", "target-sl2"); + const stat = await fs.stat("target-sl2"); + // this is how stat works! + expect(stat.isFile()).toBe(true); + expect(stat.isSymbolicLink()).toBe(false); + // so use lstat + const lstat = await fs.lstat("target-sl2"); + expect(lstat.isFile()).toBe(false); + expect(lstat.isSymbolicLink()).toBe(true); + }); + + it("truncate a file", async () => { + await fs.writeFile("t", ""); + await fs.truncate("t", 10); + const s = await fs.stat("t"); + expect(s.size).toBe(10); + }); + + it("delete a file with unlink", async () => { + await fs.writeFile("to-unlink", ""); + await fs.unlink("to-unlink"); + expect(await fs.exists("to-unlink")).toBe(false); + }); + + it("sets times of a file", async () => { + await fs.writeFile("my-times", ""); + const statsBefore = await fs.stat("my-times"); + const atime = Date.now() - 100_000; + const mtime = Date.now() - 10_000_000; + // NOTE: fs.utimes in nodejs takes *seconds*, not ms, hence + // dividing by 1000 here: + await fs.utimes("my-times", atime / 1000, mtime / 1000); + const s = await fs.stat("my-times"); + expect(s.atimeMs).toBeCloseTo(atime); + expect(s.mtimeMs).toBeCloseTo(mtime); + expect(s.atime.valueOf()).toBeCloseTo(atime); + expect(s.mtime.valueOf()).toBeCloseTo(mtime); + }); + + it("creating a symlink works (as does using lstat)", async () => { await fs.writeFile("source1", "the source"); await fs.symlink("source1", "target1"); expect(await fs.readFile("target1", "utf8")).toEqual("the source"); @@ -151,36 +275,53 @@ describe("security: dangerous symlinks can't be followed", () => { // This is setup bypassing security and is part of our threat model, due to users // having full access internally to their sandbox fs. - it("directly create a file that is a symlink outside of the sandbox -- this should work", async () => { + it("directly create a dangerous file that is a symlink outside of the sandbox -- this should work", async () => { await symlink( join(tempDir2, project_id, "password"), - join(tempDir2, project_id2, "link"), + join(tempDir2, project_id2, "danger"), ); - const s = await readFile(join(tempDir2, project_id2, "link"), "utf8"); + const s = await readFile(join(tempDir2, project_id2, "danger"), "utf8"); expect(s).toBe("s3cr3t"); }); it("fails to read the symlink content via the api", async () => { await expect(async () => { - await fs2.readFile("link", "utf8"); + await fs2.readFile("danger", "utf8"); }).rejects.toThrow("outside of sandbox"); }); - it("directly create a relative symlink ", async () => { + it("directly create a dangerous relative symlink ", async () => { await symlink( join("..", project_id, "password"), - join(tempDir2, project_id2, "link2"), + join(tempDir2, project_id2, "danger2"), ); - const s = await readFile(join(tempDir2, project_id2, "link2"), "utf8"); + const s = await readFile(join(tempDir2, project_id2, "danger2"), "utf8"); expect(s).toBe("s3cr3t"); }); it("fails to read the relative symlink content via the api", async () => { await expect(async () => { - await fs2.readFile("link2", "utf8"); + await fs2.readFile("danger2", "utf8"); }).rejects.toThrow("outside of sandbox"); }); + // This is not a vulnerability, because there's no way for the user + // to create a hard link like this from within an nfs mount (say) + // of their own folder. + it("directly create a hard link", async () => { + await link( + join(tempDir2, project_id, "password"), + join(tempDir2, project_id2, "danger3"), + ); + const s = await readFile(join(tempDir2, project_id2, "danger3"), "utf8"); + expect(s).toBe("s3cr3t"); + }); + + it("a hardlink *can* get outside the sandbox", async () => { + const s = await fs2.readFile("danger3", "utf8"); + expect(s).toBe("s3cr3t"); + }); + it("closes the server", () => { server.close(); }); @@ -189,5 +330,5 @@ describe("security: dangerous symlinks can't be followed", () => { afterAll(async () => { await after(); await rm(tempDir, { force: true, recursive: true }); - // await rm(tempDir2, { force: true, recursive: true }); + await rm(tempDir2, { force: true, recursive: true }); }); From 5901190a18af1dd7615d7ea0560873e97aeb4def Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 18 Jul 2025 23:39:05 +0000 Subject: [PATCH 013/798] fix a typescript error --- src/packages/file-server/conat/test/local-path.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/packages/file-server/conat/test/local-path.test.ts b/src/packages/file-server/conat/test/local-path.test.ts index d53f72c0f6..8ccb4b293b 100644 --- a/src/packages/file-server/conat/test/local-path.test.ts +++ b/src/packages/file-server/conat/test/local-path.test.ts @@ -217,7 +217,6 @@ describe("use all the standard api functions of fs", () => { it("sets times of a file", async () => { await fs.writeFile("my-times", ""); - const statsBefore = await fs.stat("my-times"); const atime = Date.now() - 100_000; const mtime = Date.now() - 10_000_000; // NOTE: fs.utimes in nodejs takes *seconds*, not ms, hence From 11a21fa9bc60a4bfb94b22d8dbe3fa392292c17d Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 00:21:39 +0000 Subject: [PATCH 014/798] tiny steps to make sync-doc more flexible in various ways --- src/packages/backend/conat/sync-doc/client.ts | 185 ++++++++++++++++++ .../backend/conat/sync-doc/syncstring.ts | 21 ++ .../conat/sync-doc/test/syncstring.test.ts | 35 ++++ src/packages/backend/package.json | 2 + src/packages/pnpm-lock.yaml | 3 + src/packages/sync/client/sync-client.ts | 3 + src/packages/sync/editor/generic/sync-doc.ts | 8 + 7 files changed, 257 insertions(+) create mode 100644 src/packages/backend/conat/sync-doc/client.ts create mode 100644 src/packages/backend/conat/sync-doc/syncstring.ts create mode 100644 src/packages/backend/conat/sync-doc/test/syncstring.test.ts diff --git a/src/packages/backend/conat/sync-doc/client.ts b/src/packages/backend/conat/sync-doc/client.ts new file mode 100644 index 0000000000..bf6742a550 --- /dev/null +++ b/src/packages/backend/conat/sync-doc/client.ts @@ -0,0 +1,185 @@ +/* + * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. + * License: MS-RSL – see LICENSE.md for details + */ + +import { EventEmitter } from "events"; +import { bind_methods, keys } from "@cocalc/util/misc"; +import { + Client as Client0, + FileWatcher as FileWatcher0, +} from "@cocalc/sync/editor/generic/types"; +import { SyncTable } from "@cocalc/sync/table/synctable"; +import { ExecuteCodeOptionsWithCallback } from "@cocalc/util/types/execute-code"; +import { once } from "@cocalc/util/async-utils"; + +export class FileWatcher extends EventEmitter implements FileWatcher0 { + private path: string; + constructor(path: string) { + super(); + this.path = path; + console.log("FileWatcher", this.path); + } + public close(): void {} +} + +export class Client extends EventEmitter implements Client0 { + private _client_id: string; + private initial_get_query: { [table: string]: any[] }; + public set_queries: any[] = []; + + constructor( + initial_get_query: { [table: string]: any[] }, + client_id: string, + ) { + super(); + this._client_id = client_id; + this.initial_get_query = initial_get_query; + bind_methods(this, ["query", "dbg", "query_cancel"]); + } + + public server_time(): Date { + return new Date(); + } + + isTestClient = () => { + return true; + }; + + public is_project(): boolean { + return false; + } + + public is_browser(): boolean { + return true; + } + + public is_compute_server(): boolean { + return false; + } + + public dbg(_f: string): Function { + // return (...args) => { + // console.log(_f, ...args); + // }; + return (..._) => {}; + } + + public mark_file(_opts: { + project_id: string; + path: string; + action: string; + ttl: number; + }): void { + //console.log("mark_file", opts); + } + + public log_error(opts: { + project_id: string; + path: string; + string_id: string; + error: any; + }): void { + console.log("log_error", opts); + } + + public query(opts): void { + if (opts.options && opts.options.length === 1 && opts.options[0].set) { + // set query + this.set_queries.push(opts); + opts.cb(); + } else { + // get query -- returns predetermined result + const table = keys(opts.query)[0]; + let result = this.initial_get_query[table]; + if (result == null) { + result = []; + } + //console.log("GET QUERY ", table, result); + opts.cb(undefined, { query: { [table]: result } }); + } + } + + path_access(opts: { path: string; mode: string; cb: Function }): void { + console.log("path_access", opts.path, opts.mode); + opts.cb(true); + } + path_exists(opts: { path: string; cb: Function }): void { + console.log("path_access", opts.path); + opts.cb(true); + } + path_stat(opts: { path: string; cb: Function }): void { + console.log("path_state", opts.path); + opts.cb(true); + } + async path_read(opts: { + path: string; + maxsize_MB?: number; + cb: Function; + }): Promise { + console.log("path_ready", opts.path); + opts.cb(true); + } + async write_file(opts: { + path: string; + data: string; + cb: Function; + }): Promise { + console.log("write_file", opts.path, opts.data); + opts.cb(true); + } + watch_file(opts: { path: string }): FileWatcher { + return new FileWatcher(opts.path); + } + + public is_connected(): boolean { + return true; + } + + public is_signed_in(): boolean { + return true; + } + + public touch_project(_): void {} + + public query_cancel(_): void {} + + public alert_message(_): void {} + + public is_deleted(_filename: string, _project_id?: string): boolean { + return false; + } + + public set_deleted(_filename: string, _project_id?: string): void {} + + async synctable_ephemeral( + _project_id: string, + query: any, + options: any, + throttle_changes?: number, + ): Promise { + const s = new SyncTable(query, options, this, throttle_changes); + await once(s, "connected"); + return s; + } + + async synctable_conat(_query: any): Promise { + throw Error("synctable_conat: not implemented"); + } + async pubsub_conat(_query: any): Promise { + throw Error("pubsub_conat: not implemented"); + } + + // account_id or project_id + public client_id(): string { + return this._client_id; + } + + public sage_session({ path }): void { + console.log(`sage_session: path=${path}`); + } + + public shell(opts: ExecuteCodeOptionsWithCallback): void { + console.log(`shell: opts=${JSON.stringify(opts)}`); + } +} diff --git a/src/packages/backend/conat/sync-doc/syncstring.ts b/src/packages/backend/conat/sync-doc/syncstring.ts new file mode 100644 index 0000000000..f0454b7fdf --- /dev/null +++ b/src/packages/backend/conat/sync-doc/syncstring.ts @@ -0,0 +1,21 @@ +import { Client } from "./client"; +import { SyncString } from "@cocalc/sync/editor/string/sync"; +import { a_txt } from "@cocalc/sync/editor/string/test/data"; +import { once } from "@cocalc/util/async-utils"; + +export default async function ephemeralSyncstring() { + const { client_id, project_id, path, init_queries } = a_txt(); + const client = new Client(init_queries, client_id); + const syncstring = new SyncString({ + project_id, + path, + client, + ephemeral: true, + }); + // replace save to disk, since otherwise unless string is empty, + // this will hang forever... and it is called on close. + // @ts-ignore + syncstring.save_to_disk = async () => Promise; + await once(syncstring, "ready"); + return syncstring; +} diff --git a/src/packages/backend/conat/sync-doc/test/syncstring.test.ts b/src/packages/backend/conat/sync-doc/test/syncstring.test.ts new file mode 100644 index 0000000000..62b9875bb5 --- /dev/null +++ b/src/packages/backend/conat/sync-doc/test/syncstring.test.ts @@ -0,0 +1,35 @@ +import syncstring from "@cocalc/backend/conat/sync-doc/syncstring"; + +describe("basic tests of a syncstring", () => { + let s; + + it("creates a syncstring", async () => { + s = await syncstring(); + }); + + it("initially it is empty", () => { + expect(s.to_str()).toBe(""); + expect(s.versions().length).toBe(0); + }); + + it("set the value", () => { + s.from_str("test"); + expect(s.to_str()).toBe("test"); + expect(s.versions().length).toBe(0); + }); + + it("commit the value", () => { + s.commit(); + expect(s.versions().length).toBe(1); + }); + + it("change the value and commit a second time", () => { + s.from_str("bar"); + s.commit(); + expect(s.versions().length).toBe(2); + }); + + it("get first version", () => { + expect(s.version(s.versions()[0]).to_str()).toBe("test"); + }); +}); diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index ebcc660430..f190097f05 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -6,6 +6,7 @@ "./*": "./dist/*.js", "./database": "./dist/database/index.js", "./conat": "./dist/conat/index.js", + "./conat/sync/*": "./dist/conat/sync/*.js", "./server-settings": "./dist/server-settings/index.js", "./auth/*": "./dist/auth/*.js", "./auth/tokens/*": "./dist/auth/tokens/*.js" @@ -34,6 +35,7 @@ "dependencies": { "@cocalc/backend": "workspace:*", "@cocalc/conat": "workspace:*", + "@cocalc/sync": "workspace:*", "@cocalc/util": "workspace:*", "@types/debug": "^4.1.12", "@types/jest": "^29.5.14", diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index ac022d9c8f..930789121f 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -81,6 +81,9 @@ importers: '@cocalc/conat': specifier: workspace:* version: link:../conat + '@cocalc/sync': + specifier: workspace:* + version: link:../sync '@cocalc/util': specifier: workspace:* version: link:../util diff --git a/src/packages/sync/client/sync-client.ts b/src/packages/sync/client/sync-client.ts index 56db603ed7..d3287e9295 100644 --- a/src/packages/sync/client/sync-client.ts +++ b/src/packages/sync/client/sync-client.ts @@ -99,6 +99,7 @@ export class SyncClient { data_server: undefined, client: this.client, ephemeral: false, + fs: undefined, }); return new SyncString(opts0); } @@ -122,6 +123,8 @@ export class SyncClient { client: this.client, ephemeral: false, + + fs: undefined, }); return new SyncDB(opts0); } diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 213775e8d3..9466089037 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -125,6 +125,11 @@ const DEBUG = false; export type State = "init" | "ready" | "closed"; export type DataServer = "project" | "database"; +export interface SyncDocFilesystem { + readFile: (path: string, encoding?: any) => Promise; + writeFile: (path: string, data: string | Buffer) => Promise; +} + export interface SyncOpts0 { project_id: string; path: string; @@ -151,6 +156,9 @@ export interface SyncOpts0 { // which data/changefeed server to use data_server?: DataServer; + + // optional filesystem interface. + fs?: SyncDocFilesystem; } export interface SyncOpts extends SyncOpts0 { From 1f9060f4d118cb1e9b8aa7ec6d3fa23841adb9b6 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 00:40:00 +0000 Subject: [PATCH 015/798] move fs sandbox code to @cocalc/backend, since it fits with other things there and is very lightweight --- .../{file-server/conat => backend/conat/files}/local-path.ts | 2 +- .../conat => backend/conat/files}/test/local-path.test.ts | 0 src/packages/backend/package.json | 1 + .../{file-server/fs/sandbox.ts => backend/sandbox/index.ts} | 0 .../{file-server/fs => backend/sandbox}/sandbox.test.ts | 2 +- 5 files changed, 3 insertions(+), 2 deletions(-) rename src/packages/{file-server/conat => backend/conat/files}/local-path.ts (94%) rename src/packages/{file-server/conat => backend/conat/files}/test/local-path.test.ts (100%) rename src/packages/{file-server/fs/sandbox.ts => backend/sandbox/index.ts} (100%) rename src/packages/{file-server/fs => backend/sandbox}/sandbox.test.ts (96%) diff --git a/src/packages/file-server/conat/local-path.ts b/src/packages/backend/conat/files/local-path.ts similarity index 94% rename from src/packages/file-server/conat/local-path.ts rename to src/packages/backend/conat/files/local-path.ts index 53edc7443b..1c50dcdc80 100644 --- a/src/packages/file-server/conat/local-path.ts +++ b/src/packages/backend/conat/files/local-path.ts @@ -1,5 +1,5 @@ import { fsServer } from "@cocalc/conat/files/fs"; -import { SandboxedFilesystem } from "@cocalc/file-server/fs/sandbox"; +import { SandboxedFilesystem } from "@cocalc/backend/sandbox"; import { mkdir } from "fs/promises"; import { join } from "path"; import { isValidUUID } from "@cocalc/util/misc"; diff --git a/src/packages/file-server/conat/test/local-path.test.ts b/src/packages/backend/conat/files/test/local-path.test.ts similarity index 100% rename from src/packages/file-server/conat/test/local-path.test.ts rename to src/packages/backend/conat/files/test/local-path.test.ts diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index f190097f05..e31ec57b16 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -6,6 +6,7 @@ "./*": "./dist/*.js", "./database": "./dist/database/index.js", "./conat": "./dist/conat/index.js", + "./sandbox": "./dist/sandbox/index.js", "./conat/sync/*": "./dist/conat/sync/*.js", "./server-settings": "./dist/server-settings/index.js", "./auth/*": "./dist/auth/*.js", diff --git a/src/packages/file-server/fs/sandbox.ts b/src/packages/backend/sandbox/index.ts similarity index 100% rename from src/packages/file-server/fs/sandbox.ts rename to src/packages/backend/sandbox/index.ts diff --git a/src/packages/file-server/fs/sandbox.test.ts b/src/packages/backend/sandbox/sandbox.test.ts similarity index 96% rename from src/packages/file-server/fs/sandbox.test.ts rename to src/packages/backend/sandbox/sandbox.test.ts index 73bd45b171..882676e76e 100644 --- a/src/packages/file-server/fs/sandbox.test.ts +++ b/src/packages/backend/sandbox/sandbox.test.ts @@ -1,4 +1,4 @@ -import { SandboxedFilesystem } from "./sandbox"; +import { SandboxedFilesystem } from "@cocalc/backend/sandbox"; import { mkdtemp, mkdir, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "path"; From 4b37a56046ecd0a4b86acca5b0fb3ac441dcf5ee Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 01:05:16 +0000 Subject: [PATCH 016/798] file sandbox: refactor some unit testing code --- .../backend/conat/files/local-path.ts | 6 +-- .../conat/files/test/local-path.test.ts | 52 +++++++------------ src/packages/backend/conat/files/test/util.ts | 30 +++++++++++ 3 files changed, 53 insertions(+), 35 deletions(-) create mode 100644 src/packages/backend/conat/files/test/util.ts diff --git a/src/packages/backend/conat/files/local-path.ts b/src/packages/backend/conat/files/local-path.ts index 1c50dcdc80..fe52d3aedb 100644 --- a/src/packages/backend/conat/files/local-path.ts +++ b/src/packages/backend/conat/files/local-path.ts @@ -5,7 +5,7 @@ import { join } from "path"; import { isValidUUID } from "@cocalc/util/misc"; import { type Client, getClient } from "@cocalc/conat/core/client"; -export function localPathFileserver({ +export async function localPathFileserver({ service, path, client, @@ -15,7 +15,7 @@ export function localPathFileserver({ client?: Client; }) { client ??= getClient(); - const server = fsServer({ + const server = await fsServer({ service, client, fs: async (subject: string) => { @@ -27,7 +27,7 @@ export function localPathFileserver({ return new SandboxedFilesystem(p); }, }); - return server; + return { server, client, path, service, close: () => server.end() }; } function getProjectId(subject: string) { diff --git a/src/packages/backend/conat/files/test/local-path.test.ts b/src/packages/backend/conat/files/test/local-path.test.ts index 8ccb4b293b..65430a8c2a 100644 --- a/src/packages/backend/conat/files/test/local-path.test.ts +++ b/src/packages/backend/conat/files/test/local-path.test.ts @@ -1,31 +1,23 @@ -import { localPathFileserver } from "../local-path"; -import { link, mkdtemp, readFile, rm, symlink } from "node:fs/promises"; -import { tmpdir } from "node:os"; +import { link, readFile, symlink } from "node:fs/promises"; import { join } from "path"; import { fsClient } from "@cocalc/conat/files/fs"; import { randomId } from "@cocalc/conat/names"; -import { before, after, client } from "@cocalc/backend/conat/test/setup"; +import { before, after } from "@cocalc/backend/conat/test/setup"; import { uuid } from "@cocalc/util/misc"; +import { createPathFileserver, cleanupFileservers } from "./util"; -let tempDir; -let tempDir2; -beforeAll(async () => { - await before(); - tempDir = await mkdtemp(join(tmpdir(), "cocalc-local-path")); - tempDir2 = await mkdtemp(join(tmpdir(), "cocalc-local-path-2")); -}); +beforeAll(before); describe("use all the standard api functions of fs", () => { - const service = `fs-${randomId()}`; let server; it("creates the simple fileserver service", async () => { - server = await localPathFileserver({ client, service, path: tempDir }); + server = await createPathFileserver(); }); const project_id = uuid(); let fs; it("create a client", () => { - fs = fsClient({ subject: `${service}.project-${project_id}` }); + fs = fsClient({ subject: `${server.service}.project-${project_id}` }); }); it("appendFile works", async () => { @@ -246,25 +238,22 @@ describe("use all the standard api functions of fs", () => { const stats0 = await fs.stat("source1"); expect(stats0.isSymbolicLink()).toBe(false); }); - - it("closes the service", () => { - server.close(); - }); }); describe("security: dangerous symlinks can't be followed", () => { - const service = `fs-${randomId()}`; let server; + let tempDir; it("creates the simple fileserver service", async () => { - server = await localPathFileserver({ client, service, path: tempDir2 }); + server = await createPathFileserver(); + tempDir = server.path; }); const project_id = uuid(); const project_id2 = uuid(); let fs, fs2; it("create two clients", () => { - fs = fsClient({ subject: `${service}.project-${project_id}` }); - fs2 = fsClient({ subject: `${service}.project-${project_id2}` }); + fs = fsClient({ subject: `${server.service}.project-${project_id}` }); + fs2 = fsClient({ subject: `${server.service}.project-${project_id2}` }); }); it("create a secret in one", async () => { @@ -276,10 +265,10 @@ describe("security: dangerous symlinks can't be followed", () => { // having full access internally to their sandbox fs. it("directly create a dangerous file that is a symlink outside of the sandbox -- this should work", async () => { await symlink( - join(tempDir2, project_id, "password"), - join(tempDir2, project_id2, "danger"), + join(tempDir, project_id, "password"), + join(tempDir, project_id2, "danger"), ); - const s = await readFile(join(tempDir2, project_id2, "danger"), "utf8"); + const s = await readFile(join(tempDir, project_id2, "danger"), "utf8"); expect(s).toBe("s3cr3t"); }); @@ -292,9 +281,9 @@ describe("security: dangerous symlinks can't be followed", () => { it("directly create a dangerous relative symlink ", async () => { await symlink( join("..", project_id, "password"), - join(tempDir2, project_id2, "danger2"), + join(tempDir, project_id2, "danger2"), ); - const s = await readFile(join(tempDir2, project_id2, "danger2"), "utf8"); + const s = await readFile(join(tempDir, project_id2, "danger2"), "utf8"); expect(s).toBe("s3cr3t"); }); @@ -309,10 +298,10 @@ describe("security: dangerous symlinks can't be followed", () => { // of their own folder. it("directly create a hard link", async () => { await link( - join(tempDir2, project_id, "password"), - join(tempDir2, project_id2, "danger3"), + join(tempDir, project_id, "password"), + join(tempDir, project_id2, "danger3"), ); - const s = await readFile(join(tempDir2, project_id2, "danger3"), "utf8"); + const s = await readFile(join(tempDir, project_id2, "danger3"), "utf8"); expect(s).toBe("s3cr3t"); }); @@ -328,6 +317,5 @@ describe("security: dangerous symlinks can't be followed", () => { afterAll(async () => { await after(); - await rm(tempDir, { force: true, recursive: true }); - await rm(tempDir2, { force: true, recursive: true }); + await cleanupFileservers(); }); diff --git a/src/packages/backend/conat/files/test/util.ts b/src/packages/backend/conat/files/test/util.ts new file mode 100644 index 0000000000..2de57463ea --- /dev/null +++ b/src/packages/backend/conat/files/test/util.ts @@ -0,0 +1,30 @@ +import { localPathFileserver } from "../local-path"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "path"; +import { client } from "@cocalc/backend/conat/test/setup"; +import { randomId } from "@cocalc/conat/names"; + +const tempDirs: string[] = []; +const servers: any[] = []; +export async function createPathFileserver({ + service = `fs-${randomId()}`, +}: { service?: string } = {}) { + const tempDir = await mkdtemp(join(tmpdir(), `cocalc-${randomId()}`)); + tempDirs.push(tempDir); + const server = await localPathFileserver({ client, service, path: tempDir }); + servers.push(server); + return server; +} + +// clean up any +export async function cleanupFileservers() { + for (const server of servers) { + server.close(); + } + for (const tempDir of tempDirs) { + try { + await rm(tempDir, { force: true, recursive: true }); + } catch {} + } +} From e6c6c1e0e07a3ec03ed830d82910530340fdd1f6 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 05:12:10 +0000 Subject: [PATCH 017/798] sync-doc: start using the new fs interface --- .../conat/files/test/local-path.test.ts | 16 +++ .../backend/conat/sync-doc/syncstring.ts | 14 +- .../backend/conat/sync-doc/test/setup.ts | 27 ++++ .../conat/sync-doc/test/syncstring.test.ts | 35 ++++- src/packages/backend/sandbox/index.ts | 24 +++- src/packages/conat/core/client.ts | 23 +++- src/packages/conat/files/fs.ts | 2 +- src/packages/sync/editor/generic/sync-doc.ts | 120 ++++++++++++++---- 8 files changed, 222 insertions(+), 39 deletions(-) create mode 100644 src/packages/backend/conat/sync-doc/test/setup.ts diff --git a/src/packages/backend/conat/files/test/local-path.test.ts b/src/packages/backend/conat/files/test/local-path.test.ts index 65430a8c2a..5eb9e9fadd 100644 --- a/src/packages/backend/conat/files/test/local-path.test.ts +++ b/src/packages/backend/conat/files/test/local-path.test.ts @@ -87,6 +87,22 @@ describe("use all the standard api functions of fs", () => { expect(t).toEqual("conat"); }); + it("the full error message structure is preserved exactly as in the nodejs library", async () => { + const path = randomId(); + try { + await fs.readFile(path); + } catch (err) { + expect(err.message).toEqual( + `ENOENT: no such file or directory, open '${path}'`, + ); + expect(err.message).toContain(path); + expect(err.code).toEqual("ENOENT"); + expect(err.errno).toEqual(-2); + expect(err.path).toEqual(path); + expect(err.syscall).toEqual("open"); + } + }); + it("readdir works", async () => { await fs.mkdir("dirtest"); for (let i = 0; i < 5; i++) { diff --git a/src/packages/backend/conat/sync-doc/syncstring.ts b/src/packages/backend/conat/sync-doc/syncstring.ts index f0454b7fdf..c7cee121b5 100644 --- a/src/packages/backend/conat/sync-doc/syncstring.ts +++ b/src/packages/backend/conat/sync-doc/syncstring.ts @@ -2,15 +2,25 @@ import { Client } from "./client"; import { SyncString } from "@cocalc/sync/editor/string/sync"; import { a_txt } from "@cocalc/sync/editor/string/test/data"; import { once } from "@cocalc/util/async-utils"; +import { type SyncDocFilesystem } from "@cocalc/sync/editor/generic/sync-doc"; -export default async function ephemeralSyncstring() { - const { client_id, project_id, path, init_queries } = a_txt(); +export default async function syncstring({ + fs, + project_id, + path, +}: { + fs: SyncDocFilesystem; + project_id: string; + path: string; +}) { + const { client_id, init_queries } = a_txt(); const client = new Client(init_queries, client_id); const syncstring = new SyncString({ project_id, path, client, ephemeral: true, + fs, }); // replace save to disk, since otherwise unless string is empty, // this will hang forever... and it is called on close. diff --git a/src/packages/backend/conat/sync-doc/test/setup.ts b/src/packages/backend/conat/sync-doc/test/setup.ts new file mode 100644 index 0000000000..bcef527ac6 --- /dev/null +++ b/src/packages/backend/conat/sync-doc/test/setup.ts @@ -0,0 +1,27 @@ +import { + before as before0, + after as after0, +} from "@cocalc/backend/conat/test/setup"; +import { + createPathFileserver, + cleanupFileservers, +} from "@cocalc/backend/conat/files/test/util"; +import { type Filesystem } from "@cocalc/conat/files/fs"; +export { uuid } from "@cocalc/util/misc"; +import { fsClient } from "@cocalc/conat/files/fs"; + +export let server; + +export async function before() { + await before0(); + server = await createPathFileserver(); +} + +export function getFS(project_id: string): Filesystem { + return fsClient({ subject: `${server.service}.project-${project_id}` }); +} + +export async function after() { + await cleanupFileservers(); + await after0(); +} diff --git a/src/packages/backend/conat/sync-doc/test/syncstring.test.ts b/src/packages/backend/conat/sync-doc/test/syncstring.test.ts index 62b9875bb5..96bf2d1643 100644 --- a/src/packages/backend/conat/sync-doc/test/syncstring.test.ts +++ b/src/packages/backend/conat/sync-doc/test/syncstring.test.ts @@ -1,35 +1,56 @@ import syncstring from "@cocalc/backend/conat/sync-doc/syncstring"; +import { before, after, getFS, uuid } from "./setup"; + +beforeAll(before); +afterAll(after); describe("basic tests of a syncstring", () => { let s; + const project_id = uuid(); + let fs; - it("creates a syncstring", async () => { - s = await syncstring(); + it("creates the fs client", () => { + fs = getFS(project_id); }); - it("initially it is empty", () => { + it("a syncstring associated to a file that does not exist on disk is initialized to the empty string", async () => { + s = await syncstring({ fs, project_id, path: "new.txt" }); expect(s.to_str()).toBe(""); expect(s.versions().length).toBe(0); + s.close(); + }); + + it("a syncstring for editing a file that already exists on disk is initialized to that file", async () => { + fs = getFS(project_id); + await fs.writeFile("a.txt", "hello"); + s = await syncstring({ fs, project_id, path: "a.txt" }); + expect(s.fs).not.toEqual(undefined); + }); + + it("initially it is 'hello'", () => { + expect(s.to_str()).toBe("hello"); + expect(s.versions().length).toBe(1); }); it("set the value", () => { s.from_str("test"); expect(s.to_str()).toBe("test"); - expect(s.versions().length).toBe(0); + expect(s.versions().length).toBe(1); }); it("commit the value", () => { s.commit(); - expect(s.versions().length).toBe(1); + expect(s.versions().length).toBe(2); }); it("change the value and commit a second time", () => { s.from_str("bar"); s.commit(); - expect(s.versions().length).toBe(2); + expect(s.versions().length).toBe(3); }); it("get first version", () => { - expect(s.version(s.versions()[0]).to_str()).toBe("test"); + expect(s.version(s.versions()[0]).to_str()).toBe("hello"); + expect(s.version(s.versions()[1]).to_str()).toBe("test"); }); }); diff --git a/src/packages/backend/sandbox/index.ts b/src/packages/backend/sandbox/index.ts index f3795937c0..09146f7883 100644 --- a/src/packages/backend/sandbox/index.ts +++ b/src/packages/backend/sandbox/index.ts @@ -38,7 +38,6 @@ The problem is that 1 and 3 happen microseconds apart as separate calls to the f 3. user somehow deletes "link" and replace it by a new file that is a symlink to "../{project_id}/.ssh/id_ed25519" 4. We read from the file descriptor fd and get the contents of original "link" (or error). - */ import { @@ -68,10 +67,31 @@ import { exists } from "@cocalc/backend/misc/async-utils-node"; import { type DirectoryListingEntry } from "@cocalc/util/types"; import getListing from "@cocalc/backend/get-listing"; import { join, resolve } from "path"; +import { replace_all } from "@cocalc/util/misc"; export class SandboxedFilesystem { // path should be the path to a FOLDER on the filesystem (not a file) - constructor(public readonly path: string) {} + constructor(public readonly path: string) { + for (const f in this) { + if (f == "safeAbsPath" || f == "constructor" || f == "path") { + continue; + } + const orig = this[f]; + // @ts-ignore + this[f] = async (...args) => { + try { + // @ts-ignore + return await orig(...args); + } catch (err) { + if (err.path) { + err.path = err.path.slice(this.path.length + 1); + } + err.message = replace_all(err.message, this.path + "/", ""); + throw err; + } + }; + } + } safeAbsPath = async (path: string): Promise => { if (typeof path != "string") { diff --git a/src/packages/conat/core/client.ts b/src/packages/conat/core/client.ts index 4ba04b56b1..e09c94cd85 100644 --- a/src/packages/conat/core/client.ts +++ b/src/packages/conat/core/client.ts @@ -1106,9 +1106,16 @@ export class Client extends EventEmitter { // good for services. await mesg.respond(result); } catch (err) { + let error = err.message; + if (!error) { + error = `${err}`.slice("Error: ".length); + } await mesg.respond(null, { - noThrow: true, // we're not catching this one - headers: { error: `${err}` }, + noThrow: true, // we're not catching this respond + headers: { + error, + error_attrs: JSON.parse(JSON.stringify(err)), + }, }); } }; @@ -1127,7 +1134,7 @@ export class Client extends EventEmitter { const call = async (name: string, args: any[]) => { const resp = await this.request(subject, [name, args], opts); if (resp.headers?.error) { - throw Error(`${resp.headers.error}`); + throw headerToError(resp.headers); } else { return resp.data; } @@ -1944,3 +1951,13 @@ function toConatError(socketIoError) { }); } } + +export function headerToError(headers) { + const err = Error(headers.error); + if (headers.error_attrs) { + for (const field in headers.error_attrs) { + err[field] = headers.error_attrs[field]; + } + } + return err; +} diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index 506f3823b5..f0ce9e1f8e 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -188,7 +188,7 @@ export function fsClient({ }: { client?: Client; subject: string; -}) { +}): Filesystem { let call = (client ?? conat()).call(subject); let constants: any = null; diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 9466089037..498f319c14 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -128,6 +128,7 @@ export type DataServer = "project" | "database"; export interface SyncDocFilesystem { readFile: (path: string, encoding?: any) => Promise; writeFile: (path: string, data: string | Buffer) => Promise; + stat: (path: string) => Promise; // todo } export interface SyncOpts0 { @@ -272,6 +273,8 @@ export class SyncDoc extends EventEmitter { private useConat: boolean; legacy: LegacyHistory; + private fs?: SyncDocFilesystem; + constructor(opts: SyncOpts) { super(); if (opts.string_id === undefined) { @@ -293,6 +296,7 @@ export class SyncDoc extends EventEmitter { "persistent", "data_server", "ephemeral", + "fs", ]) { if (opts[field] != undefined) { this[field] = opts[field]; @@ -1505,31 +1509,34 @@ export class SyncDoc extends EventEmitter { log("file_use_interval"); this.init_file_use_interval(); - - if (await this.isFileServer()) { - log("load_from_disk"); - // This sets initialized, which is needed to be fully ready. - // We keep trying this load from disk until sync-doc is closed - // or it succeeds. It may fail if, e.g., the file is too - // large or is not readable by the user. They are informed to - // fix the problem... and once they do (and wait up to 10s), - // this will finish. - // if (!this.client.is_browser() && !this.client.is_project()) { - // // FAKE DELAY!!! Just to simulate flakiness / slow network!!!! - // await delay(3000); - // } - await retry_until_success({ - f: this.init_load_from_disk, - max_delay: 10000, - desc: "syncdoc -- load_from_disk", - }); - log("done loading from disk"); + if (this.fs != null) { + await this.fsLoadFromDisk(); } else { - if (this.patch_list!.count() == 0) { - await Promise.race([ - this.waitUntilFullyReady(), - once(this.patch_list!, "change"), - ]); + if (await this.isFileServer()) { + log("load_from_disk"); + // This sets initialized, which is needed to be fully ready. + // We keep trying this load from disk until sync-doc is closed + // or it succeeds. It may fail if, e.g., the file is too + // large or is not readable by the user. They are informed to + // fix the problem... and once they do (and wait up to 10s), + // this will finish. + // if (!this.client.is_browser() && !this.client.is_project()) { + // // FAKE DELAY!!! Just to simulate flakiness / slow network!!!! + // await delay(3000); + // } + await retry_until_success({ + f: this.init_load_from_disk, + max_delay: 10000, + desc: "syncdoc -- load_from_disk", + }); + log("done loading from disk"); + } else { + if (this.patch_list!.count() == 0) { + await Promise.race([ + this.waitUntilFullyReady(), + once(this.patch_list!, "change"), + ]); + } } } this.assert_not_closed("initAll -- load from disk"); @@ -1757,7 +1764,46 @@ export class SyncDoc extends EventEmitter { } }; + private fsLoadFromDiskIfNewer = async (): Promise => { + // [ ] TODO: readonly handling... + if (this.fs == null) throw Error("bug"); + const dbg = this.dbg("fsLoadFromDiskIfNewer"); + let stats; + try { + stats = await this.fs.stat(this.path); + } catch (err) { + if (err.code == "ENOENT") { + // path does not exist -- nothing further to do + return false; + } else { + // no clue + return true; + } + } + dbg("path exists"); + const lastChanged = new Date(this.last_changed()); + const firstLoad = this.versions().length == 0; + if (firstLoad || stats.ctime > lastChanged) { + dbg( + `disk file changed more recently than edits, so loading ${stats.ctime} > ${lastChanged}; firstLoad=${firstLoad}`, + ); + await this.fsLoadFromDisk(); + if (firstLoad) { + dbg("emitting first-load event"); + // this event is emited the first time the document is ever loaded from disk. + this.emit("first-load"); + } + dbg("loaded"); + } else { + dbg("stick with sync version"); + } + return false; + }; + private load_from_disk_if_newer = async (): Promise => { + if (this.fs != null) { + return await this.fsLoadFromDiskIfNewer(); + } const last_changed = new Date(this.last_changed()); const firstLoad = this.versions().length == 0; const dbg = this.dbg("load_from_disk_if_newer"); @@ -2938,6 +2984,32 @@ export class SyncDoc extends EventEmitter { this.close(); }; + private fsLoadFromDisk = async (): Promise => { + if (this.fs == null) throw Error("bug"); + const dbg = this.dbg("fsLoadFromDisk"); + + let size: number; + let contents; + try { + contents = await this.fs.readFile(this.path, "utf8"); + dbg("file exists"); + size = contents.length; + this.from_str(contents); + } catch (err) { + if (err.code == "ENOENT") { + dbg("file no longer exists -- setting to blank"); + size = 0; + this.from_str(""); + } else { + throw err; + } + } + // save new version to stream, which we just set via from_str + this.commit(); + await this.save(); + return size; + }; + private load_from_disk = async (): Promise => { const path = this.path; const dbg = this.dbg("load_from_disk"); From d872a972fec9da9109a4985f32f86dde5893d2d3 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 05:20:43 +0000 Subject: [PATCH 018/798] sync-doc: use fs interface to save to disk --- src/packages/backend/conat/sync-doc/syncstring.ts | 4 ---- .../backend/conat/sync-doc/test/syncstring.test.ts | 6 ++++++ src/packages/sync/editor/generic/sync-doc.ts | 14 ++++++++++++++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/packages/backend/conat/sync-doc/syncstring.ts b/src/packages/backend/conat/sync-doc/syncstring.ts index c7cee121b5..64990e42c9 100644 --- a/src/packages/backend/conat/sync-doc/syncstring.ts +++ b/src/packages/backend/conat/sync-doc/syncstring.ts @@ -22,10 +22,6 @@ export default async function syncstring({ ephemeral: true, fs, }); - // replace save to disk, since otherwise unless string is empty, - // this will hang forever... and it is called on close. - // @ts-ignore - syncstring.save_to_disk = async () => Promise; await once(syncstring, "ready"); return syncstring; } diff --git a/src/packages/backend/conat/sync-doc/test/syncstring.test.ts b/src/packages/backend/conat/sync-doc/test/syncstring.test.ts index 96bf2d1643..fa637f28d0 100644 --- a/src/packages/backend/conat/sync-doc/test/syncstring.test.ts +++ b/src/packages/backend/conat/sync-doc/test/syncstring.test.ts @@ -38,6 +38,12 @@ describe("basic tests of a syncstring", () => { expect(s.versions().length).toBe(1); }); + it("save value to disk", async () => { + await s.save_to_disk(); + const disk = await fs.readFile("a.txt", "utf8"); + expect(disk).toEqual("test"); + }); + it("commit the value", () => { s.commit(); expect(s.versions().length).toBe(2); diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 498f319c14..b4c019f57c 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -3188,6 +3188,17 @@ export class SyncDoc extends EventEmitter { return true; }; + fsSaveToDisk = async () => { + const dbg = this.dbg("fsSaveToDisk"); + if (this.client.is_deleted(this.path, this.project_id)) { + dbg("not saving to disk because deleted"); + return; + } + dbg(); + if (this.fs == null) throw Error("bug"); + await this.fs.writeFile(this.path, this.to_str()); + }; + /* Initiates a save of file to disk, then waits for the state to change. */ save_to_disk = async (): Promise => { @@ -3201,6 +3212,9 @@ export class SyncDoc extends EventEmitter { // properly. return; } + if (this.fs != null) { + return await this.fsSaveToDisk(); + } const dbg = this.dbg("save_to_disk"); if (this.client.is_deleted(this.path, this.project_id)) { dbg("not saving to disk because deleted"); From 89dc29087545999a71776cf6bbd1cf21a36a102a Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 05:32:32 +0000 Subject: [PATCH 019/798] sync-doc: start marking fs related client functions as only required for legacy clients --- src/packages/backend/conat/sync-doc/client.ts | 4 ---- src/packages/sync/editor/generic/sync-doc.ts | 19 +++++++++++++++++++ src/packages/sync/editor/generic/types.ts | 2 +- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/packages/backend/conat/sync-doc/client.ts b/src/packages/backend/conat/sync-doc/client.ts index bf6742a550..19813b772d 100644 --- a/src/packages/backend/conat/sync-doc/client.ts +++ b/src/packages/backend/conat/sync-doc/client.ts @@ -104,10 +104,6 @@ export class Client extends EventEmitter implements Client0 { console.log("path_access", opts.path, opts.mode); opts.cb(true); } - path_exists(opts: { path: string; cb: Function }): void { - console.log("path_access", opts.path); - opts.cb(true); - } path_stat(opts: { path: string; cb: Function }): void { console.log("path_state", opts.path); opts.cb(true); diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index b4c019f57c..cb971876cd 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -1714,6 +1714,13 @@ export class SyncDoc extends EventEmitter { }; private pathExistsAndIsReadOnly = async (path): Promise => { + if (this.client.path_exists == null) { + throw Error("legacy clients must define path_exists"); + } + if (this.client.path_access == null) { + throw Error("legacy clients must define path_access"); + } + try { await callback2(this.client.path_access, { path, @@ -1804,6 +1811,9 @@ export class SyncDoc extends EventEmitter { if (this.fs != null) { return await this.fsLoadFromDiskIfNewer(); } + if (this.client.path_exists == null) { + throw Error("legacy clients must define path_exists"); + } const last_changed = new Date(this.last_changed()); const firstLoad = this.versions().length == 0; const dbg = this.dbg("load_from_disk_if_newer"); @@ -2890,6 +2900,12 @@ export class SyncDoc extends EventEmitter { }; private update_watch_path = async (path?: string): Promise => { + if (this.fs != null) { + return; + } + if (this.client.path_exists == null) { + throw Error("legacy clients must define path_exists"); + } const dbg = this.dbg("update_watch_path"); if (this.file_watcher != null) { // clean up @@ -3011,6 +3027,9 @@ export class SyncDoc extends EventEmitter { }; private load_from_disk = async (): Promise => { + if (this.client.path_exists == null) { + throw Error("legacy clients must define path_exists"); + } const path = this.path; const dbg = this.dbg("load_from_disk"); dbg(); diff --git a/src/packages/sync/editor/generic/types.ts b/src/packages/sync/editor/generic/types.ts index 4cf3519e94..9222636e92 100644 --- a/src/packages/sync/editor/generic/types.ts +++ b/src/packages/sync/editor/generic/types.ts @@ -102,7 +102,7 @@ export interface ProjectClient extends EventEmitter { // Only required to work on project client. path_access: (opts: { path: string; mode: string; cb: Function }) => void; - path_exists: (opts: { path: string; cb: Function }) => void; + path_exists?: (opts: { path: string; cb: Function }) => void; path_stat: (opts: { path: string; cb: Function }) => void; From 45e094348ad665eac4ed2060bd0470d357ed0405 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 14:23:01 +0000 Subject: [PATCH 020/798] fix some tests --- src/packages/backend/conat/files/local-path.ts | 2 +- src/packages/backend/{ => files}/sandbox/index.ts | 0 src/packages/backend/{ => files}/sandbox/sandbox.test.ts | 2 +- src/packages/backend/package.json | 3 ++- src/packages/file-server/btrfs/subvolume.ts | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) rename src/packages/backend/{ => files}/sandbox/index.ts (100%) rename src/packages/backend/{ => files}/sandbox/sandbox.test.ts (95%) diff --git a/src/packages/backend/conat/files/local-path.ts b/src/packages/backend/conat/files/local-path.ts index fe52d3aedb..1d0aa43a8e 100644 --- a/src/packages/backend/conat/files/local-path.ts +++ b/src/packages/backend/conat/files/local-path.ts @@ -1,5 +1,5 @@ import { fsServer } from "@cocalc/conat/files/fs"; -import { SandboxedFilesystem } from "@cocalc/backend/sandbox"; +import { SandboxedFilesystem } from "@cocalc/backend/files/sandbox"; import { mkdir } from "fs/promises"; import { join } from "path"; import { isValidUUID } from "@cocalc/util/misc"; diff --git a/src/packages/backend/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts similarity index 100% rename from src/packages/backend/sandbox/index.ts rename to src/packages/backend/files/sandbox/index.ts diff --git a/src/packages/backend/sandbox/sandbox.test.ts b/src/packages/backend/files/sandbox/sandbox.test.ts similarity index 95% rename from src/packages/backend/sandbox/sandbox.test.ts rename to src/packages/backend/files/sandbox/sandbox.test.ts index 882676e76e..5e445b42c0 100644 --- a/src/packages/backend/sandbox/sandbox.test.ts +++ b/src/packages/backend/files/sandbox/sandbox.test.ts @@ -1,4 +1,4 @@ -import { SandboxedFilesystem } from "@cocalc/backend/sandbox"; +import { SandboxedFilesystem } from "@cocalc/backend/files/sandbox"; import { mkdtemp, mkdir, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "path"; diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index e31ec57b16..1ceb59e9c6 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -6,7 +6,8 @@ "./*": "./dist/*.js", "./database": "./dist/database/index.js", "./conat": "./dist/conat/index.js", - "./sandbox": "./dist/sandbox/index.js", + "./files/*": "./dist/files/*.js", + "./files/sandbox": "./dist/files/sandbox/index.js", "./conat/sync/*": "./dist/conat/sync/*.js", "./server-settings": "./dist/server-settings/index.js", "./auth/*": "./dist/auth/*.js", diff --git a/src/packages/file-server/btrfs/subvolume.ts b/src/packages/file-server/btrfs/subvolume.ts index 7eb7808518..f10e9f6182 100644 --- a/src/packages/file-server/btrfs/subvolume.ts +++ b/src/packages/file-server/btrfs/subvolume.ts @@ -9,7 +9,7 @@ import { join } from "path"; import { SubvolumeBup } from "./subvolume-bup"; import { SubvolumeSnapshots } from "./subvolume-snapshots"; import { SubvolumeQuota } from "./subvolume-quota"; -import { SandboxedFilesystem } from "../fs/sandbox"; +import { SandboxedFilesystem } from "@cocalc/backend/files/sandbox"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import getLogger from "@cocalc/backend/logger"; From 6921d7e226046a46570a90bccb17a093289be51d Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 15:35:46 +0000 Subject: [PATCH 021/798] rewrite nodejs watch async iterator since it is broken until node v24 and we don't want to require node24 yet --- src/packages/backend/files/sandbox/index.ts | 14 ++++++++++++-- .../backend/files/sandbox/sandbox.test.ts | 19 +++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index 09146f7883..446a9e0813 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -61,13 +61,14 @@ import { writeFile, unlink, utimes, - watch, } from "node:fs/promises"; +import { watch } from "node:fs"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import { type DirectoryListingEntry } from "@cocalc/util/types"; import getListing from "@cocalc/backend/get-listing"; import { join, resolve } from "path"; import { replace_all } from "@cocalc/util/misc"; +import { EventIterator } from "@cocalc/util/event-iterator"; export class SandboxedFilesystem { // path should be the path to a FOLDER on the filesystem (not a file) @@ -231,7 +232,16 @@ export class SandboxedFilesystem { }; watch = async (filename: string, options?) => { - return watch(await this.safeAbsPath(filename), options); + // NOTE: in node v24 they fixed the fs/promises watch to have a queue, but previous + // versions were clearly badly implemented so we reimplement it from scratch + // using the non-promise watch. + const watcher = watch(await this.safeAbsPath(filename), options); + return new EventIterator(watcher, "change", { + map: (args) => { + // exact same api as new fs/promises watch + return { eventType: args[0], filename: args[1] }; + }, + }); }; writeFile = async (path: string, data: string | Buffer) => { diff --git a/src/packages/backend/files/sandbox/sandbox.test.ts b/src/packages/backend/files/sandbox/sandbox.test.ts index 5e445b42c0..44bcfc9114 100644 --- a/src/packages/backend/files/sandbox/sandbox.test.ts +++ b/src/packages/backend/files/sandbox/sandbox.test.ts @@ -54,6 +54,25 @@ describe("make various attempts to break out of the sandbox", () => { }); }); +describe.only("test watching a file and a folder in the sandbox", () => { + let fs; + it("creates sandbox", async () => { + await mkdir(join(tempDir, "test-watch")); + fs = new SandboxedFilesystem(join(tempDir, "test-watch")); + await fs.writeFile("x", "hi"); + }); + + it("watches the file x for changes", async () => { + const w = await fs.watch("x"); + await fs.appendFile("x", " there"); + const x = await w.next(); + expect(x).toEqual({ + value: { eventType: "change", filename: "x" }, + done: false, + }); + }); +}); + afterAll(async () => { await rm(tempDir, { force: true, recursive: true }); }); From 1363b5365969c7911f2d44554ea8c9e5e77f8673 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 16:00:11 +0000 Subject: [PATCH 022/798] add maxQueue support to watch (and our EventIterator) --- src/packages/backend/files/sandbox/index.ts | 4 +++ .../backend/files/sandbox/sandbox.test.ts | 28 ++++++++++++++++++- src/packages/util/event-iterator.ts | 17 +++++++++-- 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index 446a9e0813..1cbb53616a 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -237,10 +237,14 @@ export class SandboxedFilesystem { // using the non-promise watch. const watcher = watch(await this.safeAbsPath(filename), options); return new EventIterator(watcher, "change", { + maxQueue: options?.maxQueue ?? 2048, map: (args) => { // exact same api as new fs/promises watch return { eventType: args[0], filename: args[1] }; }, + onEnd: () => { + watcher.close(); + }, }); }; diff --git a/src/packages/backend/files/sandbox/sandbox.test.ts b/src/packages/backend/files/sandbox/sandbox.test.ts index 44bcfc9114..b541dae579 100644 --- a/src/packages/backend/files/sandbox/sandbox.test.ts +++ b/src/packages/backend/files/sandbox/sandbox.test.ts @@ -54,7 +54,7 @@ describe("make various attempts to break out of the sandbox", () => { }); }); -describe.only("test watching a file and a folder in the sandbox", () => { +describe("test watching a file and a folder in the sandbox", () => { let fs; it("creates sandbox", async () => { await mkdir(join(tempDir, "test-watch")); @@ -70,6 +70,32 @@ describe.only("test watching a file and a folder in the sandbox", () => { value: { eventType: "change", filename: "x" }, done: false, }); + w.end(); + }); + + it("the maxQueue parameter limits the number of queue events", async () => { + const w = await fs.watch("x", { maxQueue: 2 }); + expect(w.queueSize()).toBe(0); + // make many changes + await fs.appendFile("x", "0"); + await fs.appendFile("x", "0"); + await fs.appendFile("x", "0"); + await fs.appendFile("x", "0"); + // there will only be 2 available: + expect(w.queueSize()).toBe(2); + const x0 = await w.next(); + expect(x0).toEqual({ + value: { eventType: "change", filename: "x" }, + done: false, + }); + const x1 = await w.next(); + expect(x1).toEqual({ + value: { eventType: "change", filename: "x" }, + done: false, + }); + // one more next would hang... + expect(w.queueSize()).toBe(0); + w.end(); }); }); diff --git a/src/packages/util/event-iterator.ts b/src/packages/util/event-iterator.ts index c3ca053301..7353d2991c 100644 --- a/src/packages/util/event-iterator.ts +++ b/src/packages/util/event-iterator.ts @@ -1,7 +1,7 @@ /* LICENSE: MIT -This is a slight fork of +This is a slight fork of https://github.com/sapphiredev/utilities/tree/main/packages/event-iterator @@ -10,7 +10,7 @@ agree with the docs. I can see why. Upstream would capture ['arg1','arg2']] for an event emitter doing this emitter.emit('foo', 'arg1', 'arg2') - + But for our application we only want 'arg1'. I thus added a map option, which makes it easy to do what we want. */ @@ -46,6 +46,9 @@ export interface EventIteratorOptions { // called when iterator ends -- use to do cleanup. onEnd?: (iter?: EventIterator) => void; + + // Specifies the number of events to queue between iterations of the returned. + maxQueue?: number; } /** @@ -100,6 +103,8 @@ export class EventIterator */ readonly #limit: number; + readonly #maxQueue: number; + /** * The timer to track when this will idle out. */ @@ -124,6 +129,7 @@ export class EventIterator this.event = event; this.map = options.map ?? ((args) => args); this.#limit = options.limit ?? Infinity; + this.#maxQueue = options.maxQueue ?? Infinity; this.#idle = options.idle; this.filter = options.filter ?? ((): boolean => true); this.onEnd = options.onEnd; @@ -263,10 +269,17 @@ export class EventIterator try { const value = this.map(args); this.#queue.push(value); + while (this.#queue.length > this.#maxQueue && this.#queue.length > 0) { + this.#queue.shift(); + } } catch (err) { this.err = err; // fake event to trigger handling of err this.emitter.emit(this.event); } } + + public queueSize(): number { + return this.#queue.length; + } } From f3e40430e174afc5223a2b2e07bedb360a925382 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 17:00:16 +0000 Subject: [PATCH 023/798] support EventIterator overflow options (like node fs watch); add ability to EventIterator next to handle if it is ended while waiting (which prevents a major deadlock/race that upstream had and could hang things); rewrite llm server to ensure messages get through, which avoids race condition if client makes llm request immediately on login (highly unlikely, but let's do it right). --- src/packages/backend/conat/test/llm.test.ts | 4 +- src/packages/backend/files/sandbox/index.ts | 22 ++++++++-- .../backend/files/sandbox/sandbox.test.ts | 29 +++++++++++++ src/packages/conat/llm/server.ts | 34 +++++++++++---- src/packages/util/event-iterator.ts | 41 ++++++++++++++----- 5 files changed, 105 insertions(+), 25 deletions(-) diff --git a/src/packages/backend/conat/test/llm.test.ts b/src/packages/backend/conat/test/llm.test.ts index 9ac69af07f..ab0eeb8ece 100644 --- a/src/packages/backend/conat/test/llm.test.ts +++ b/src/packages/backend/conat/test/llm.test.ts @@ -18,7 +18,7 @@ beforeAll(before); describe("create an llm server, client, and stub evaluator, and run an evaluation", () => { // define trivial evaluate - const OUTPUT = "Thanks for asing about "; + const OUTPUT = "Thanks for asking about "; async function evaluate({ input, stream }) { stream(OUTPUT); stream(input); @@ -52,7 +52,7 @@ describe("create an llm server, client, and stub evaluator, and run an evaluatio describe("test an evaluate that throws an error half way through", () => { // define trivial evaluate - const OUTPUT = "Thanks for asing about "; + const OUTPUT = "Thanks for asking about "; const ERROR = "I give up"; async function evaluate({ stream }) { stream(OUTPUT); diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index 1cbb53616a..2888e3a480 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -231,13 +231,24 @@ export class SandboxedFilesystem { await utimes(await this.safeAbsPath(path), atime, mtime); }; - watch = async (filename: string, options?) => { + watch = async ( + filename: string, + options?: { + persistent?: boolean; + recursive?: boolean; + encoding?: string; + signal?: AbortSignal; + maxQueue?: number; + overflow?: "ignore" | "throw"; + }, + ) => { // NOTE: in node v24 they fixed the fs/promises watch to have a queue, but previous // versions were clearly badly implemented so we reimplement it from scratch // using the non-promise watch. - const watcher = watch(await this.safeAbsPath(filename), options); - return new EventIterator(watcher, "change", { + const watcher = watch(await this.safeAbsPath(filename), options as any); + const iter = new EventIterator(watcher, "change", { maxQueue: options?.maxQueue ?? 2048, + overflow: options?.overflow, map: (args) => { // exact same api as new fs/promises watch return { eventType: args[0], filename: args[1] }; @@ -246,6 +257,11 @@ export class SandboxedFilesystem { watcher.close(); }, }); + // AbortController signal can cause this + watcher.once("close", () => { + iter.end(); + }); + return iter; }; writeFile = async (path: string, data: string | Buffer) => { diff --git a/src/packages/backend/files/sandbox/sandbox.test.ts b/src/packages/backend/files/sandbox/sandbox.test.ts index b541dae579..1f29ae4b53 100644 --- a/src/packages/backend/files/sandbox/sandbox.test.ts +++ b/src/packages/backend/files/sandbox/sandbox.test.ts @@ -63,6 +63,7 @@ describe("test watching a file and a folder in the sandbox", () => { }); it("watches the file x for changes", async () => { + await fs.writeFile("x", "hi"); const w = await fs.watch("x"); await fs.appendFile("x", " there"); const x = await w.next(); @@ -74,6 +75,7 @@ describe("test watching a file and a folder in the sandbox", () => { }); it("the maxQueue parameter limits the number of queue events", async () => { + await fs.writeFile("x", "hi"); const w = await fs.watch("x", { maxQueue: 2 }); expect(w.queueSize()).toBe(0); // make many changes @@ -97,6 +99,33 @@ describe("test watching a file and a folder in the sandbox", () => { expect(w.queueSize()).toBe(0); w.end(); }); + + it("maxQueue with overflow throw", async () => { + await fs.writeFile("x", "hi"); + const w = await fs.watch("x", { maxQueue: 2, overflow: "throw" }); + await fs.appendFile("x", "0"); + await fs.appendFile("x", "0"); + await fs.appendFile("x", "0"); + expect(async () => { + await w.next(); + }).rejects.toThrow("maxQueue overflow"); + w.end(); + }); + + it("AbortController works", async () => { + const ac = new AbortController(); + const { signal } = ac; + await fs.writeFile("x", "hi"); + const w = await fs.watch("x", { signal }); + await fs.appendFile("x", "0"); + const e = await w.next(); + expect(e.done).toBe(false); + + // now abort + ac.abort(); + const { done } = await w.next(); + expect(done).toBe(true); + }); }); afterAll(async () => { diff --git a/src/packages/conat/llm/server.ts b/src/packages/conat/llm/server.ts index 77afd7d86d..e04571cbcc 100644 --- a/src/packages/conat/llm/server.ts +++ b/src/packages/conat/llm/server.ts @@ -14,6 +14,9 @@ how paying for that would work. import { conat } from "@cocalc/conat/client"; import { isValidUUID } from "@cocalc/util/misc"; import type { Subscription } from "@cocalc/conat/core/client"; +import { getLogger } from "@cocalc/conat/client"; + +const logger = getLogger("conat:llm:server"); export const SUBJECT = process.env.COCALC_TEST_MODE ? "llm-test" : "llm"; @@ -61,7 +64,7 @@ export async function close() { if (sub == null) { return; } - sub.drain(); + sub.close(); sub = null; } @@ -77,24 +80,37 @@ async function listen(evaluate) { async function handleMessage(mesg, evaluate) { const options = mesg.data; - let seq = 0; - const respond = ({ text, error }: { text?: string; error?: string }) => { - mesg.respondSync({ text, error, seq }); + let seq = -1; + const respond = async ({ + text, + error, + }: { + text?: string; + error?: string; + }) => { seq += 1; + try { + // mesg.respondSync({ text, error, seq }); + const { count } = await mesg.respond({ text, error, seq }); + } catch (err) { + logger.debug("WARNING: error sending response -- ", err); + end(); + } }; let done = false; - const end = () => { + const end = async () => { if (done) return; done = true; - // end response stream with null payload. - mesg.respondSync(null); + // end response stream with null payload -- send sync, or it could + // get sent before the responses above, which would cancel them out! + await mesg.respond(null, { noThrow: true }); }; - const stream = (text?) => { + const stream = async (text?) => { if (done) return; if (text != null) { - respond({ text }); + await respond({ text }); } else { end(); } diff --git a/src/packages/util/event-iterator.ts b/src/packages/util/event-iterator.ts index 7353d2991c..8dcca43803 100644 --- a/src/packages/util/event-iterator.ts +++ b/src/packages/util/event-iterator.ts @@ -49,6 +49,11 @@ export interface EventIteratorOptions { // Specifies the number of events to queue between iterations of the returned. maxQueue?: number; + + // Either 'ignore' or 'throw' when there are more events to be queued than maxQueue allows. + // 'ignore' means overflow events are dropped and a warning is emitted, while + // 'throw' means to throw an exception. Default: 'ignore'. + overflow?: "ignore" | "throw"; } /** @@ -104,6 +109,9 @@ export class EventIterator readonly #limit: number; readonly #maxQueue: number; + readonly #overflow?: "ignore" | "throw"; + + private resolveNext?: Function; /** * The timer to track when this will idle out. @@ -130,6 +138,7 @@ export class EventIterator this.map = options.map ?? ((args) => args); this.#limit = options.limit ?? Infinity; this.#maxQueue = options.maxQueue ?? Infinity; + this.#overflow = options.overflow ?? "ignore"; this.#idle = options.idle; this.filter = options.filter ?? ((): boolean => true); this.onEnd = options.onEnd; @@ -158,6 +167,7 @@ export class EventIterator */ public end(): void { if (this.#ended) return; + this.resolveNext?.(); this.#ended = true; this.#queue = []; @@ -171,14 +181,9 @@ export class EventIterator // aliases to match usage in NATS and CoCalc. close = this.end; stop = this.end; - - drain(): void { - // just immediately end - this.end(); - // [ ] TODO: for compat. I'm not sure what this should be - // or if it matters... - // console.log("WARNING: TODO -- event-iterator drain not implemented"); - } + // TODO/worry: drain doesn't do anything special to address outstanding + // requests like NATS did. Probably this isn't the place for it... + drain = this.end; /** * The next value that's received from the EventEmitter. @@ -232,10 +237,18 @@ export class EventIterator // Once it has received at least one value, we will clear the timer (if defined), // and resolve with the new value: - this.emitter.once(this.event, () => { - if (idleTimer) clearTimeout(idleTimer); + const handleEvent = () => { + delete this.resolveNext; + if (idleTimer) { + clearTimeout(idleTimer); + } resolve(this.next()); - }); + }; + this.emitter.once(this.event, handleEvent); + this.resolveNext = () => { + this.emitter.removeListener(this.event, handleEvent); + resolve({ done: true, value: undefined }); + }; }); } @@ -266,10 +279,16 @@ export class EventIterator * Pushes a value into the queue. */ protected push(...args): void { + if (this.err) { + return; + } try { const value = this.map(args); this.#queue.push(value); while (this.#queue.length > this.#maxQueue && this.#queue.length > 0) { + if (this.#overflow == "throw") { + throw Error("maxQueue overflow"); + } this.#queue.shift(); } } catch (err) { From 0c48df61f3b05a26eb7f3c30977fe949411c8096 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 17:14:46 +0000 Subject: [PATCH 024/798] more fs.watch unit tests --- .../backend/files/sandbox/sandbox.test.ts | 38 +++++++++++++++++++ src/packages/conat/llm/server.ts | 3 +- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/packages/backend/files/sandbox/sandbox.test.ts b/src/packages/backend/files/sandbox/sandbox.test.ts index 1f29ae4b53..ebabcb5dba 100644 --- a/src/packages/backend/files/sandbox/sandbox.test.ts +++ b/src/packages/backend/files/sandbox/sandbox.test.ts @@ -126,6 +126,44 @@ describe("test watching a file and a folder in the sandbox", () => { const { done } = await w.next(); expect(done).toBe(true); }); + + it("watches a directory", async () => { + await fs.mkdir("folder"); + const w = await fs.watch("folder"); + + await fs.writeFile("folder/x", "hi"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "rename", filename: "x" }, + }); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "x" }, + }); + + await fs.appendFile("folder/x", "xxx"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "x" }, + }); + + await fs.writeFile("folder/z", "there"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "rename", filename: "z" }, + }); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "z" }, + }); + + // this is correct -- from the node docs "On most platforms, 'rename' is emitted whenever a filename appears or disappears in the directory." + await fs.unlink("folder/z"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "rename", filename: "z" }, + }); + }); }); afterAll(async () => { diff --git a/src/packages/conat/llm/server.ts b/src/packages/conat/llm/server.ts index e04571cbcc..c13ef765d2 100644 --- a/src/packages/conat/llm/server.ts +++ b/src/packages/conat/llm/server.ts @@ -90,8 +90,7 @@ async function handleMessage(mesg, evaluate) { }) => { seq += 1; try { - // mesg.respondSync({ text, error, seq }); - const { count } = await mesg.respond({ text, error, seq }); + await mesg.respond({ text, error, seq }); } catch (err) { logger.debug("WARNING: error sending response -- ", err); end(); From 41c3cd44318349a0155080e67bf3a9dd97220957 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 17:49:34 +0000 Subject: [PATCH 025/798] add fs watch core --- .../backend/conat/files/test/watch.test.ts | 61 +++++++++++++ src/packages/conat/files/watch.ts | 90 +++++++++++++++++++ src/packages/conat/persist/server.ts | 2 +- 3 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 src/packages/backend/conat/files/test/watch.test.ts create mode 100644 src/packages/conat/files/watch.ts diff --git a/src/packages/backend/conat/files/test/watch.test.ts b/src/packages/backend/conat/files/test/watch.test.ts new file mode 100644 index 0000000000..860c3f99d1 --- /dev/null +++ b/src/packages/backend/conat/files/test/watch.test.ts @@ -0,0 +1,61 @@ +import { before, after, client, wait } from "@cocalc/backend/conat/test/setup"; +import { watchServer, watchClient } from "@cocalc/conat/files/watch"; +import { SandboxedFilesystem } from "@cocalc/backend/files/sandbox"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "path"; +import { randomId } from "@cocalc/conat/names"; + +let tmp; +beforeAll(async () => { + await before(); + tmp = await mkdtemp(join(tmpdir(), `cocalc-${randomId()}`)); +}); +afterAll(async () => { + await after(); + try { + await rm(tmp, { force: true, recursive: true }); + } catch {} +}); + +describe("basic core of the async path watch functionality", () => { + let fs; + it("creates sandboxed filesystem", () => { + fs = new SandboxedFilesystem(tmp); + }); + + let server; + it("create watch server", () => { + server = watchServer({ client, subject: "foo", watch: fs.watch }); + }); + + it("create a file", async () => { + await fs.writeFile("a.txt", "hi"); + }); + + let w; + it("create a watcher client", async () => { + w = await watchClient({ client, subject: "foo", path: "a.txt" }); + }); + + it("observe watch works", async () => { + await fs.appendFile("a.txt", "foo"); + expect(await w.next()).toEqual({ + done: false, + value: [{ eventType: "change", filename: "a.txt" }, {}], + }); + + await fs.appendFile("a.txt", "bar"); + expect(await w.next()).toEqual({ + done: false, + value: [{ eventType: "change", filename: "a.txt" }, {}], + }); + }); + + it("close the watcher client frees up a server socket", async () => { + expect(Object.keys(server.sockets).length).toEqual(1); + w.close(); + await wait({ until: () => Object.keys(server.sockets).length == 0 }); + expect(Object.keys(server.sockets).length).toEqual(0); + }); +}); diff --git a/src/packages/conat/files/watch.ts b/src/packages/conat/files/watch.ts new file mode 100644 index 0000000000..97d0b3dd4a --- /dev/null +++ b/src/packages/conat/files/watch.ts @@ -0,0 +1,90 @@ +/* +Remotely proxying a fs.watch AsyncIterator over a Conat Socket. +*/ + +import { type Client as ConatClient } from "@cocalc/conat/core/client"; +import { + type ConatSocketServer, + type ServerSocket, +} from "@cocalc/conat/socket"; +import { EventIterator } from "@cocalc/util/event-iterator"; + +import { getLogger } from "@cocalc/conat/client"; + +const logger = getLogger("conat:files:watch"); + +// (path:string, options:WatchOptions) => AsyncIterator +type AsyncWatchFunction = any; +type WatchOptions = any; + +export function watchServer({ + client, + subject, + watch, +}: { + client: ConatClient; + subject: string; + watch: AsyncWatchFunction; +}) { + const server: ConatSocketServer = client.socket.listen(subject); + logger.debug("server: listening on ", { subject }); + + server.on("connection", (socket: ServerSocket) => { + logger.debug("server: got new connection", { + id: socket.id, + subject: socket.subject, + }); + let w: undefined | ReturnType = undefined; + socket.on("closed", () => { + w?.close(); + w = undefined; + }); + + socket.on("request", async (mesg) => { + try { + const { path, options } = mesg.data; + logger.debug("got request", { path, options }); + if (w != null) { + w.close(); + w = undefined; + } + w = await watch(path, options); + await mesg.respond(); + for await (const event of w) { + socket.write(event); + } + } catch (err) { + mesg.respondSync(null, { + headers: { error: `${err}`, code: err.code }, + }); + } + }); + }); + + return server; +} + +export async function watchClient({ + client, + subject, + path, + options, +}: { + client: ConatClient; + subject: string; + path: string; + options: WatchOptions; +}) { + const socket = await client.socket.connect(subject); + const iter = new EventIterator(socket, "data", { + onEnd: () => { + socket.close(); + }, + }); + socket.on("closed", () => { + iter.end(); + }); + // tell it what to watch + await socket.request({ path, options }); + return iter; +} diff --git a/src/packages/conat/persist/server.ts b/src/packages/conat/persist/server.ts index 5a270630e7..8ad17e4715 100644 --- a/src/packages/conat/persist/server.ts +++ b/src/packages/conat/persist/server.ts @@ -59,7 +59,6 @@ import { type ConatSocketServer, type ServerSocket, } from "@cocalc/conat/socket"; -import { getLogger } from "@cocalc/conat/client"; import type { StoredMessage, PersistentStream, @@ -70,6 +69,7 @@ import { throttle } from "lodash"; import { type SetOptions } from "./client"; import { once } from "@cocalc/util/async-utils"; import { UsageMonitor } from "@cocalc/conat/monitor/usage"; +import { getLogger } from "@cocalc/conat/client"; const logger = getLogger("persist:server"); From dd29234b94810491f8161416a45f751dc10c807f Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 18:13:24 +0000 Subject: [PATCH 026/798] conat rpc fs.watch implemented and working here - obviously, still worried about leaks, typing, etc. But this works. --- .../backend/conat/files/local-path.ts | 2 +- .../conat/files/test/local-path.test.ts | 49 +++++ .../backend/conat/files/test/watch.test.ts | 4 +- src/packages/conat/files/fs.ts | 195 ++++++++++-------- src/packages/conat/files/watch.ts | 3 +- 5 files changed, 167 insertions(+), 86 deletions(-) diff --git a/src/packages/backend/conat/files/local-path.ts b/src/packages/backend/conat/files/local-path.ts index 1d0aa43a8e..3ba4fb5dad 100644 --- a/src/packages/backend/conat/files/local-path.ts +++ b/src/packages/backend/conat/files/local-path.ts @@ -27,7 +27,7 @@ export async function localPathFileserver({ return new SandboxedFilesystem(p); }, }); - return { server, client, path, service, close: () => server.end() }; + return { server, client, path, service, close: () => server.close() }; } function getProjectId(subject: string) { diff --git a/src/packages/backend/conat/files/test/local-path.test.ts b/src/packages/backend/conat/files/test/local-path.test.ts index 5eb9e9fadd..30c8d04476 100644 --- a/src/packages/backend/conat/files/test/local-path.test.ts +++ b/src/packages/backend/conat/files/test/local-path.test.ts @@ -254,6 +254,55 @@ describe("use all the standard api functions of fs", () => { const stats0 = await fs.stat("source1"); expect(stats0.isSymbolicLink()).toBe(false); }); + + it("watch a file", async () => { + await fs.writeFile("a.txt", "hi"); + const w = await fs.watch("a.txt"); + await fs.appendFile("a.txt", " there"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "a.txt" }, + }); + }); + + it("watch a directory", async () => { + const FOLDER = randomId(); + await fs.mkdir(FOLDER); + const w = await fs.watch(FOLDER); + + await fs.writeFile(join(FOLDER, "x"), "hi"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "rename", filename: "x" }, + }); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "x" }, + }); + + await fs.appendFile(join(FOLDER, "x"), "xxx"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "x" }, + }); + + await fs.writeFile(join(FOLDER, "z"), "there"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "rename", filename: "z" }, + }); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "z" }, + }); + + // this is correct -- from the node docs "On most platforms, 'rename' is emitted whenever a filename appears or disappears in the directory." + await fs.unlink(join(FOLDER, "z")); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "rename", filename: "z" }, + }); + }); }); describe("security: dangerous symlinks can't be followed", () => { diff --git a/src/packages/backend/conat/files/test/watch.test.ts b/src/packages/backend/conat/files/test/watch.test.ts index 860c3f99d1..fdcd760751 100644 --- a/src/packages/backend/conat/files/test/watch.test.ts +++ b/src/packages/backend/conat/files/test/watch.test.ts @@ -42,13 +42,13 @@ describe("basic core of the async path watch functionality", () => { await fs.appendFile("a.txt", "foo"); expect(await w.next()).toEqual({ done: false, - value: [{ eventType: "change", filename: "a.txt" }, {}], + value: { eventType: "change", filename: "a.txt" }, }); await fs.appendFile("a.txt", "bar"); expect(await w.next()).toEqual({ done: false, - value: [{ eventType: "change", filename: "a.txt" }, {}], + value: { eventType: "change", filename: "a.txt" }, }); }); diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index f0ce9e1f8e..afa7826f66 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -1,5 +1,6 @@ import { type Client } from "@cocalc/conat/core/client"; import { conat } from "@cocalc/conat/client"; +import { watchServer, watchClient } from "@cocalc/conat/files/watch"; export interface Filesystem { appendFile: (path: string, data: string | Buffer, encoding?) => Promise; @@ -27,6 +28,8 @@ export interface Filesystem { mtime: number | string | Date, ) => Promise; writeFile: (path: string, data: string | Buffer) => Promise; + // todo: typing + watch: (path: string, options?) => Promise; } interface IStats { @@ -99,87 +102,108 @@ interface Options { } export async function fsServer({ service, fs, client }: Options) { - return await (client ?? conat()).service( - `${service}.*`, - { - async appendFile(path: string, data: string | Buffer, encoding?) { - await (await fs(this.subject)).appendFile(path, data, encoding); - }, - async chmod(path: string, mode: string | number) { - await (await fs(this.subject)).chmod(path, mode); - }, - async constants(): Promise<{ [key: string]: number }> { - return await (await fs(this.subject)).constants(); - }, - async copyFile(src: string, dest: string) { - await (await fs(this.subject)).copyFile(src, dest); - }, - async cp(src: string, dest: string, options?) { - await (await fs(this.subject)).cp(src, dest, options); - }, - async exists(path: string) { - return await (await fs(this.subject)).exists(path); - }, - async link(existingPath: string, newPath: string) { - await (await fs(this.subject)).link(existingPath, newPath); - }, - async lstat(path: string): Promise { - return await (await fs(this.subject)).lstat(path); - }, - async mkdir(path: string, options?) { - await (await fs(this.subject)).mkdir(path, options); - }, - async readFile(path: string, encoding?) { - return await (await fs(this.subject)).readFile(path, encoding); - }, - async readdir(path: string) { - return await (await fs(this.subject)).readdir(path); - }, - async realpath(path: string) { - return await (await fs(this.subject)).realpath(path); - }, - async rename(oldPath: string, newPath: string) { - await (await fs(this.subject)).rename(oldPath, newPath); - }, - async rm(path: string, options?) { - await (await fs(this.subject)).rm(path, options); - }, - async rmdir(path: string, options?) { - await (await fs(this.subject)).rmdir(path, options); - }, - async stat(path: string): Promise { - const s = await (await fs(this.subject)).stat(path); - return { - ...s, - // for some reason these times get corrupted on transport from the nodejs datastructure, - // so we make them standard Date objects. - atime: new Date(s.atime), - mtime: new Date(s.mtime), - ctime: new Date(s.ctime), - birthtime: new Date(s.birthtime), - }; - }, - async symlink(target: string, path: string) { - await (await fs(this.subject)).symlink(target, path); - }, - async truncate(path: string, len?: number) { - await (await fs(this.subject)).truncate(path, len); - }, - async unlink(path: string) { - await (await fs(this.subject)).unlink(path); - }, - async utimes( - path: string, - atime: number | string | Date, - mtime: number | string | Date, - ) { - await (await fs(this.subject)).utimes(path, atime, mtime); - }, - async writeFile(path: string, data: string | Buffer) { - await (await fs(this.subject)).writeFile(path, data); - }, - }, - ); + client ??= conat(); + const subject = `${service}.*`; + const watches: { [subject: string]: any } = {}; + const sub = await client.service(subject, { + async appendFile(path: string, data: string | Buffer, encoding?) { + await (await fs(this.subject)).appendFile(path, data, encoding); + }, + async chmod(path: string, mode: string | number) { + await (await fs(this.subject)).chmod(path, mode); + }, + async constants(): Promise<{ [key: string]: number }> { + return await (await fs(this.subject)).constants(); + }, + async copyFile(src: string, dest: string) { + await (await fs(this.subject)).copyFile(src, dest); + }, + async cp(src: string, dest: string, options?) { + await (await fs(this.subject)).cp(src, dest, options); + }, + async exists(path: string) { + return await (await fs(this.subject)).exists(path); + }, + async link(existingPath: string, newPath: string) { + await (await fs(this.subject)).link(existingPath, newPath); + }, + async lstat(path: string): Promise { + return await (await fs(this.subject)).lstat(path); + }, + async mkdir(path: string, options?) { + await (await fs(this.subject)).mkdir(path, options); + }, + async readFile(path: string, encoding?) { + return await (await fs(this.subject)).readFile(path, encoding); + }, + async readdir(path: string) { + return await (await fs(this.subject)).readdir(path); + }, + async realpath(path: string) { + return await (await fs(this.subject)).realpath(path); + }, + async rename(oldPath: string, newPath: string) { + await (await fs(this.subject)).rename(oldPath, newPath); + }, + async rm(path: string, options?) { + await (await fs(this.subject)).rm(path, options); + }, + async rmdir(path: string, options?) { + await (await fs(this.subject)).rmdir(path, options); + }, + async stat(path: string): Promise { + const s = await (await fs(this.subject)).stat(path); + return { + ...s, + // for some reason these times get corrupted on transport from the nodejs datastructure, + // so we make them standard Date objects. + atime: new Date(s.atime), + mtime: new Date(s.mtime), + ctime: new Date(s.ctime), + birthtime: new Date(s.birthtime), + }; + }, + async symlink(target: string, path: string) { + await (await fs(this.subject)).symlink(target, path); + }, + async truncate(path: string, len?: number) { + await (await fs(this.subject)).truncate(path, len); + }, + async unlink(path: string) { + await (await fs(this.subject)).unlink(path); + }, + async utimes( + path: string, + atime: number | string | Date, + mtime: number | string | Date, + ) { + await (await fs(this.subject)).utimes(path, atime, mtime); + }, + async writeFile(path: string, data: string | Buffer) { + await (await fs(this.subject)).writeFile(path, data); + }, + async watch() { + const subject = this.subject!; + if (watches[subject] != null) { + return; + } + const f = await fs(subject); + watches[subject] = watchServer({ + client, + subject: subject!, + watch: f.watch, + }); + }, + }); + return { + close: () => { + for (const subject in watches) { + watches[subject].close(); + delete watches[subject]; + } + sub.close(); + }, + }; } export function fsClient({ @@ -189,7 +213,8 @@ export function fsClient({ client?: Client; subject: string; }): Filesystem { - let call = (client ?? conat()).call(subject); + client ??= conat(); + let call = client.call(subject); let constants: any = null; const stat0 = call.stat.bind(call); @@ -214,5 +239,11 @@ export function fsClient({ return stats; }; + const ensureWatchServerExists = call.watch.bind(call); + call.watch = async (path: string, options?) => { + await ensureWatchServerExists(path, options); + return await watchClient({ client, subject, path, options }); + }; + return call; } diff --git a/src/packages/conat/files/watch.ts b/src/packages/conat/files/watch.ts index 97d0b3dd4a..47dd65d939 100644 --- a/src/packages/conat/files/watch.ts +++ b/src/packages/conat/files/watch.ts @@ -73,10 +73,11 @@ export async function watchClient({ client: ConatClient; subject: string; path: string; - options: WatchOptions; + options?: WatchOptions; }) { const socket = await client.socket.connect(subject); const iter = new EventIterator(socket, "data", { + map: (args) => args[0], onEnd: () => { socket.close(); }, From 99656fd9e6257199f9f536c03950c4f9a82baa93 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 20:00:01 +0000 Subject: [PATCH 027/798] broken work in progress creating a more generic and unit testable sync-doc --- src/packages/backend/conat/sync-doc/client.ts | 181 ++++++------------ .../backend/conat/sync-doc/syncstring.ts | 7 +- .../backend/conat/sync-doc/test/setup.ts | 1 + .../conat/sync-doc/test/syncstring.test.ts | 42 +++- src/packages/backend/conat/test/util.ts | 4 +- src/packages/conat/core/client.ts | 4 + 6 files changed, 113 insertions(+), 126 deletions(-) diff --git a/src/packages/backend/conat/sync-doc/client.ts b/src/packages/backend/conat/sync-doc/client.ts index 19813b772d..ee53ab1e4c 100644 --- a/src/packages/backend/conat/sync-doc/client.ts +++ b/src/packages/backend/conat/sync-doc/client.ts @@ -4,14 +4,15 @@ */ import { EventEmitter } from "events"; -import { bind_methods, keys } from "@cocalc/util/misc"; import { Client as Client0, FileWatcher as FileWatcher0, } from "@cocalc/sync/editor/generic/types"; -import { SyncTable } from "@cocalc/sync/table/synctable"; -import { ExecuteCodeOptionsWithCallback } from "@cocalc/util/types/execute-code"; -import { once } from "@cocalc/util/async-utils"; +import { conat as conat0 } from "@cocalc/backend/conat/conat"; +import { parseQueryWithOptions } from "@cocalc/sync/table/util"; +import { PubSub } from "@cocalc/conat/sync/pubsub"; +import { type Client as ConatClient } from "@cocalc/conat/core/client"; +import { type ConatSyncTable } from "@cocalc/conat/sync/synctable"; export class FileWatcher extends EventEmitter implements FileWatcher0 { private path: string; @@ -20,94 +21,78 @@ export class FileWatcher extends EventEmitter implements FileWatcher0 { this.path = path; console.log("FileWatcher", this.path); } - public close(): void {} + close(): void {} } export class Client extends EventEmitter implements Client0 { - private _client_id: string; - private initial_get_query: { [table: string]: any[] }; - public set_queries: any[] = []; - - constructor( - initial_get_query: { [table: string]: any[] }, - client_id: string, - ) { + private conat: ConatClient; + constructor(conat?: ConatClient) { super(); - this._client_id = client_id; - this.initial_get_query = initial_get_query; - bind_methods(this, ["query", "dbg", "query_cancel"]); + this.conat = conat ?? conat0(); } - public server_time(): Date { - return new Date(); - } + is_project = (): boolean => false; + is_browser = (): boolean => true; + is_compute_server = (): boolean => false; - isTestClient = () => { - return true; + dbg = (_f: string) => { + return (..._) => {}; }; - public is_project(): boolean { - return false; - } + is_connected = (): boolean => { + return this.conat.isConnected(); + }; - public is_browser(): boolean { - return true; - } + is_signed_in = (): boolean => { + return this.conat.isSignedIn(); + }; + + touch_project = (_): void => {}; - public is_compute_server(): boolean { + is_deleted = (_filename: string, _project_id?: string): boolean => { return false; - } + }; - public dbg(_f: string): Function { - // return (...args) => { - // console.log(_f, ...args); - // }; - return (..._) => {}; - } + set_deleted = (_filename: string, _project_id?: string): void => {}; - public mark_file(_opts: { - project_id: string; - path: string; - action: string; - ttl: number; - }): void { - //console.log("mark_file", opts); - } + synctable_conat = async (query0, options?): Promise => { + const { query } = parseQueryWithOptions(query0, options); + return await this.conat.sync.synctable({ + ...options, + query, + }); + }; - public log_error(opts: { - project_id: string; - path: string; - string_id: string; - error: any; - }): void { - console.log("log_error", opts); - } + pubsub_conat = async (opts): Promise => { + return new PubSub({ client: this.conat, ...opts }); + }; - public query(opts): void { - if (opts.options && opts.options.length === 1 && opts.options[0].set) { - // set query - this.set_queries.push(opts); - opts.cb(); - } else { - // get query -- returns predetermined result - const table = keys(opts.query)[0]; - let result = this.initial_get_query[table]; - if (result == null) { - result = []; - } - //console.log("GET QUERY ", table, result); - opts.cb(undefined, { query: { [table]: result } }); - } - } + // account_id or project_id + client_id = (): string => this.conat.id; + + server_time = (): Date => { + return new Date(); + }; + + ///////////////////////////////// + // EVERYTHING BELOW: TO REMOVE? + mark_file = (_): void => {}; + + alert_message = (_): void => {}; + + sage_session = (_): void => {}; - path_access(opts: { path: string; mode: string; cb: Function }): void { + shell = (_): void => {}; + + path_access = (opts: { path: string; mode: string; cb: Function }): void => { console.log("path_access", opts.path, opts.mode); opts.cb(true); - } - path_stat(opts: { path: string; cb: Function }): void { + }; + path_stat = (opts: { path: string; cb: Function }): void => { console.log("path_state", opts.path); opts.cb(true); - } + }; + async path_read(opts: { path: string; maxsize_MB?: number; @@ -128,54 +113,10 @@ export class Client extends EventEmitter implements Client0 { return new FileWatcher(opts.path); } - public is_connected(): boolean { - return true; - } - - public is_signed_in(): boolean { - return true; - } - - public touch_project(_): void {} - - public query_cancel(_): void {} + log_error = (_): void => {}; - public alert_message(_): void {} - - public is_deleted(_filename: string, _project_id?: string): boolean { - return false; - } - - public set_deleted(_filename: string, _project_id?: string): void {} - - async synctable_ephemeral( - _project_id: string, - query: any, - options: any, - throttle_changes?: number, - ): Promise { - const s = new SyncTable(query, options, this, throttle_changes); - await once(s, "connected"); - return s; - } - - async synctable_conat(_query: any): Promise { - throw Error("synctable_conat: not implemented"); - } - async pubsub_conat(_query: any): Promise { - throw Error("pubsub_conat: not implemented"); - } - - // account_id or project_id - public client_id(): string { - return this._client_id; - } - - public sage_session({ path }): void { - console.log(`sage_session: path=${path}`); - } - - public shell(opts: ExecuteCodeOptionsWithCallback): void { - console.log(`shell: opts=${JSON.stringify(opts)}`); - } + query = (_): void => { + throw Error("not implemented"); + }; + query_cancel = (_): void => {}; } diff --git a/src/packages/backend/conat/sync-doc/syncstring.ts b/src/packages/backend/conat/sync-doc/syncstring.ts index 64990e42c9..0d5a6e28f2 100644 --- a/src/packages/backend/conat/sync-doc/syncstring.ts +++ b/src/packages/backend/conat/sync-doc/syncstring.ts @@ -1,20 +1,21 @@ import { Client } from "./client"; import { SyncString } from "@cocalc/sync/editor/string/sync"; -import { a_txt } from "@cocalc/sync/editor/string/test/data"; import { once } from "@cocalc/util/async-utils"; import { type SyncDocFilesystem } from "@cocalc/sync/editor/generic/sync-doc"; +import { type Client as ConatClient } from "@cocalc/conat/core/client"; export default async function syncstring({ fs, project_id, path, + conat, }: { fs: SyncDocFilesystem; project_id: string; path: string; + conat?: ConatClient; }) { - const { client_id, init_queries } = a_txt(); - const client = new Client(init_queries, client_id); + const client = new Client(conat); const syncstring = new SyncString({ project_id, path, diff --git a/src/packages/backend/conat/sync-doc/test/setup.ts b/src/packages/backend/conat/sync-doc/test/setup.ts index bcef527ac6..dd976fb36a 100644 --- a/src/packages/backend/conat/sync-doc/test/setup.ts +++ b/src/packages/backend/conat/sync-doc/test/setup.ts @@ -2,6 +2,7 @@ import { before as before0, after as after0, } from "@cocalc/backend/conat/test/setup"; +export { connect, wait } from "@cocalc/backend/conat/test/setup"; import { createPathFileserver, cleanupFileservers, diff --git a/src/packages/backend/conat/sync-doc/test/syncstring.test.ts b/src/packages/backend/conat/sync-doc/test/syncstring.test.ts index fa637f28d0..06d3cb07ff 100644 --- a/src/packages/backend/conat/sync-doc/test/syncstring.test.ts +++ b/src/packages/backend/conat/sync-doc/test/syncstring.test.ts @@ -1,10 +1,10 @@ import syncstring from "@cocalc/backend/conat/sync-doc/syncstring"; -import { before, after, getFS, uuid } from "./setup"; +import { before, after, getFS, uuid, wait, connect } from "./setup"; beforeAll(before); afterAll(after); -describe("basic tests of a syncstring", () => { +describe("loading/saving syncstring to disk and setting values", () => { let s; const project_id = uuid(); let fs; @@ -60,3 +60,41 @@ describe("basic tests of a syncstring", () => { expect(s.version(s.versions()[1]).to_str()).toBe("test"); }); }); + +describe.only("sync with two copies of a syncstring", () => { + const project_id = uuid(); + let s1, s2, fs; + + it("creates the fs client and two copies of a syncstring", async () => { + fs = getFS(project_id); + await fs.writeFile("a.txt", "hello"); + s1 = await syncstring({ fs, project_id, path: "a.txt" }); + s2 = await syncstring({ fs, project_id, path: "a.txt", conat: connect() }); + expect(s1.to_str()).toBe("hello"); + expect(s2.to_str()).toBe("hello"); + expect(s1 === s2).toBe(false); + }); + + it("change one, commit and save, and see change reflected in the other", async () => { + s1.from_str("hello world"); + s1.commit(); + await s1.save(); + await wait({ + until: () => { + console.log(s1.to_str(), s2.to_str()); + console.log(s1.patches_table.dstream?.name, s2.patches_table.dstream?.name); + console.log(s1.patches_table.get(), s2.patches_table.get()); + console.log(s1.patch_list.patches, s2.patch_list.patches); + return s2.to_str() == "hello world"; + }, + min: 2000, + }); + }); + + it.skip("change second and see change reflected in first", async () => { + s2.from_str("hello world!"); + s2.commit(); + await s2.save(); + await wait({ until: () => s1.to_str() == "hello world!" }); + }); +}); diff --git a/src/packages/backend/conat/test/util.ts b/src/packages/backend/conat/test/util.ts index a6a9adb437..d3fa39ac63 100644 --- a/src/packages/backend/conat/test/util.ts +++ b/src/packages/backend/conat/test/util.ts @@ -4,12 +4,14 @@ export async function wait({ until: f, start = 5, decay = 1.2, + min = 5, max = 300, timeout = 10000, }: { until: Function; start?: number; decay?: number; + min?: number; max?: number; timeout?: number; }) { @@ -25,7 +27,7 @@ export async function wait({ start, decay, max, - min: 5, + min, timeout, }, ); diff --git a/src/packages/conat/core/client.ts b/src/packages/conat/core/client.ts index e09c94cd85..6bba37d09b 100644 --- a/src/packages/conat/core/client.ts +++ b/src/packages/conat/core/client.ts @@ -561,6 +561,10 @@ export class Client extends EventEmitter { setTimeout(() => this.conn.io.disconnect(), 1); }; + isConnected = () => this.state == "connected"; + + isSignedIn = () => !!(this.info?.user && !this.info?.user?.error); + // this has NO timeout by default waitUntilSignedIn = reuseInFlight( async ({ timeout }: { timeout?: number } = {}) => { From 346b9ef5099f1be3d2f1c1764632653e3abb5768 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 22:02:43 +0000 Subject: [PATCH 028/798] sync-doc: integrating with conat --- src/packages/backend/conat/files/test/util.ts | 2 +- .../backend/conat/files/test/watch.test.ts | 2 +- src/packages/backend/conat/sync-doc/client.ts | 43 +++----------- .../backend/conat/sync-doc/syncstring.ts | 3 +- .../backend/conat/sync-doc/test/setup.ts | 10 +++- .../conat/sync-doc/test/syncstring.test.ts | 57 ++++++++++++------- src/packages/frontend/conat/client.ts | 5 +- src/packages/sync/table/util.ts | 3 + 8 files changed, 61 insertions(+), 64 deletions(-) diff --git a/src/packages/backend/conat/files/test/util.ts b/src/packages/backend/conat/files/test/util.ts index 2de57463ea..b50a114c5d 100644 --- a/src/packages/backend/conat/files/test/util.ts +++ b/src/packages/backend/conat/files/test/util.ts @@ -10,7 +10,7 @@ const servers: any[] = []; export async function createPathFileserver({ service = `fs-${randomId()}`, }: { service?: string } = {}) { - const tempDir = await mkdtemp(join(tmpdir(), `cocalc-${randomId()}`)); + const tempDir = await mkdtemp(join(tmpdir(), `cocalc-${randomId()}0`)); tempDirs.push(tempDir); const server = await localPathFileserver({ client, service, path: tempDir }); servers.push(server); diff --git a/src/packages/backend/conat/files/test/watch.test.ts b/src/packages/backend/conat/files/test/watch.test.ts index fdcd760751..0c3e61c376 100644 --- a/src/packages/backend/conat/files/test/watch.test.ts +++ b/src/packages/backend/conat/files/test/watch.test.ts @@ -9,7 +9,7 @@ import { randomId } from "@cocalc/conat/names"; let tmp; beforeAll(async () => { await before(); - tmp = await mkdtemp(join(tmpdir(), `cocalc-${randomId()}`)); + tmp = await mkdtemp(join(tmpdir(), `cocalc-${randomId()}0`)); }); afterAll(async () => { await after(); diff --git a/src/packages/backend/conat/sync-doc/client.ts b/src/packages/backend/conat/sync-doc/client.ts index ee53ab1e4c..98fd947a57 100644 --- a/src/packages/backend/conat/sync-doc/client.ts +++ b/src/packages/backend/conat/sync-doc/client.ts @@ -4,31 +4,17 @@ */ import { EventEmitter } from "events"; -import { - Client as Client0, - FileWatcher as FileWatcher0, -} from "@cocalc/sync/editor/generic/types"; -import { conat as conat0 } from "@cocalc/backend/conat/conat"; +import { Client as Client0 } from "@cocalc/sync/editor/generic/types"; import { parseQueryWithOptions } from "@cocalc/sync/table/util"; import { PubSub } from "@cocalc/conat/sync/pubsub"; import { type Client as ConatClient } from "@cocalc/conat/core/client"; import { type ConatSyncTable } from "@cocalc/conat/sync/synctable"; -export class FileWatcher extends EventEmitter implements FileWatcher0 { - private path: string; - constructor(path: string) { - super(); - this.path = path; - console.log("FileWatcher", this.path); - } - close(): void {} -} - export class Client extends EventEmitter implements Client0 { private conat: ConatClient; - constructor(conat?: ConatClient) { + constructor(conat: ConatClient) { super(); - this.conat = conat ?? conat0(); + this.conat = conat; } is_project = (): boolean => false; @@ -84,34 +70,21 @@ export class Client extends EventEmitter implements Client0 { shell = (_): void => {}; - path_access = (opts: { path: string; mode: string; cb: Function }): void => { - console.log("path_access", opts.path, opts.mode); + path_access = (opts): void => { opts.cb(true); }; - path_stat = (opts: { path: string; cb: Function }): void => { + path_stat = (opts): void => { console.log("path_state", opts.path); opts.cb(true); }; - async path_read(opts: { - path: string; - maxsize_MB?: number; - cb: Function; - }): Promise { - console.log("path_ready", opts.path); + async path_read(opts): Promise { opts.cb(true); } - async write_file(opts: { - path: string; - data: string; - cb: Function; - }): Promise { - console.log("write_file", opts.path, opts.data); + async write_file(opts): Promise { opts.cb(true); } - watch_file(opts: { path: string }): FileWatcher { - return new FileWatcher(opts.path); - } + watch_file(_): any {} log_error = (_): void => {}; diff --git a/src/packages/backend/conat/sync-doc/syncstring.ts b/src/packages/backend/conat/sync-doc/syncstring.ts index 0d5a6e28f2..ae1d17b335 100644 --- a/src/packages/backend/conat/sync-doc/syncstring.ts +++ b/src/packages/backend/conat/sync-doc/syncstring.ts @@ -13,14 +13,13 @@ export default async function syncstring({ fs: SyncDocFilesystem; project_id: string; path: string; - conat?: ConatClient; + conat: ConatClient; }) { const client = new Client(conat); const syncstring = new SyncString({ project_id, path, client, - ephemeral: true, fs, }); await once(syncstring, "ready"); diff --git a/src/packages/backend/conat/sync-doc/test/setup.ts b/src/packages/backend/conat/sync-doc/test/setup.ts index dd976fb36a..24c6f1e477 100644 --- a/src/packages/backend/conat/sync-doc/test/setup.ts +++ b/src/packages/backend/conat/sync-doc/test/setup.ts @@ -1,6 +1,7 @@ import { before as before0, after as after0, + client as client0, } from "@cocalc/backend/conat/test/setup"; export { connect, wait } from "@cocalc/backend/conat/test/setup"; import { @@ -11,6 +12,8 @@ import { type Filesystem } from "@cocalc/conat/files/fs"; export { uuid } from "@cocalc/util/misc"; import { fsClient } from "@cocalc/conat/files/fs"; +export { client0 as client }; + export let server; export async function before() { @@ -18,8 +21,11 @@ export async function before() { server = await createPathFileserver(); } -export function getFS(project_id: string): Filesystem { - return fsClient({ subject: `${server.service}.project-${project_id}` }); +export function getFS(project_id: string, client): Filesystem { + return fsClient({ + subject: `${server.service}.project-${project_id}`, + client, + }); } export async function after() { diff --git a/src/packages/backend/conat/sync-doc/test/syncstring.test.ts b/src/packages/backend/conat/sync-doc/test/syncstring.test.ts index 06d3cb07ff..f78a4116f1 100644 --- a/src/packages/backend/conat/sync-doc/test/syncstring.test.ts +++ b/src/packages/backend/conat/sync-doc/test/syncstring.test.ts @@ -7,23 +7,24 @@ afterAll(after); describe("loading/saving syncstring to disk and setting values", () => { let s; const project_id = uuid(); - let fs; + let fs, conat; it("creates the fs client", () => { - fs = getFS(project_id); + conat = connect(); + fs = getFS(project_id, conat); }); it("a syncstring associated to a file that does not exist on disk is initialized to the empty string", async () => { - s = await syncstring({ fs, project_id, path: "new.txt" }); + s = await syncstring({ fs, project_id, path: "new.txt", conat }); expect(s.to_str()).toBe(""); expect(s.versions().length).toBe(0); s.close(); }); it("a syncstring for editing a file that already exists on disk is initialized to that file", async () => { - fs = getFS(project_id); + fs = getFS(project_id, conat); await fs.writeFile("a.txt", "hello"); - s = await syncstring({ fs, project_id, path: "a.txt" }); + s = await syncstring({ fs, project_id, path: "a.txt", conat }); expect(s.fs).not.toEqual(undefined); }); @@ -61,17 +62,29 @@ describe("loading/saving syncstring to disk and setting values", () => { }); }); -describe.only("sync with two copies of a syncstring", () => { +describe("synchronized editing with two copies of a syncstring", () => { const project_id = uuid(); - let s1, s2, fs; + let s1, s2, fs1, fs2, client1, client2; it("creates the fs client and two copies of a syncstring", async () => { - fs = getFS(project_id); - await fs.writeFile("a.txt", "hello"); - s1 = await syncstring({ fs, project_id, path: "a.txt" }); - s2 = await syncstring({ fs, project_id, path: "a.txt", conat: connect() }); - expect(s1.to_str()).toBe("hello"); - expect(s2.to_str()).toBe("hello"); + client1 = connect(); + client2 = connect(); + fs1 = getFS(project_id, client1); + s1 = await syncstring({ + fs: fs1, + project_id, + path: "a.txt", + conat: client1, + }); + fs2 = getFS(project_id, client2); + s2 = await syncstring({ + fs: fs2, + project_id, + path: "a.txt", + conat: client2, + }); + expect(s1.to_str()).toBe(""); + expect(s2.to_str()).toBe(""); expect(s1 === s2).toBe(false); }); @@ -81,20 +94,26 @@ describe.only("sync with two copies of a syncstring", () => { await s1.save(); await wait({ until: () => { - console.log(s1.to_str(), s2.to_str()); - console.log(s1.patches_table.dstream?.name, s2.patches_table.dstream?.name); - console.log(s1.patches_table.get(), s2.patches_table.get()); - console.log(s1.patch_list.patches, s2.patch_list.patches); return s2.to_str() == "hello world"; }, - min: 2000, }); }); - it.skip("change second and see change reflected in first", async () => { + it("change second and see change reflected in first", async () => { s2.from_str("hello world!"); s2.commit(); await s2.save(); await wait({ until: () => s1.to_str() == "hello world!" }); }); + + it("view the history from each", async () => { + expect(s1.versions().length).toEqual(2); + expect(s2.versions().length).toEqual(2); + + const v1: string[] = [], + v2: string[] = []; + s1.show_history({ log: (x) => v1.push(x) }); + s2.show_history({ log: (x) => v2.push(x) }); + expect(v1).toEqual(v2); + }); }); diff --git a/src/packages/frontend/conat/client.ts b/src/packages/frontend/conat/client.ts index bf7af5088d..878ef21973 100644 --- a/src/packages/frontend/conat/client.ts +++ b/src/packages/frontend/conat/client.ts @@ -393,10 +393,7 @@ export class ConatClient extends EventEmitter { query0, options?, ): Promise => { - const { query, table } = parseQueryWithOptions(query0, options); - if (options?.project_id != null && query[table][0]["project_id"] === null) { - query[table][0]["project_id"] = options.project_id; - } + const { query } = parseQueryWithOptions(query0, options); return await this.conat().sync.synctable({ ...options, query, diff --git a/src/packages/sync/table/util.ts b/src/packages/sync/table/util.ts index 6815f3052d..dac8c0befd 100644 --- a/src/packages/sync/table/util.ts +++ b/src/packages/sync/table/util.ts @@ -46,6 +46,9 @@ export function parseQueryWithOptions(query, options) { query[table][0][k] = obj[k]; } } + if (options?.project_id != null && query[table][0]["project_id"] === null) { + query[table][0]["project_id"] = options.project_id; + } return { query, table }; } From ec86176a064b98aec34bc8376fe7ff354268dd56 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 22:15:21 +0000 Subject: [PATCH 029/798] sync-doc conat: minor refactoring --- .../sync-doc/{client.ts => sync-client.ts} | 21 +++++++++------- .../backend/conat/sync-doc/syncstring.ts | 21 ++++++++++------ .../backend/conat/sync-doc/test/setup.ts | 11 +++++--- .../conat/sync-doc/test/syncstring.test.ts | 25 ++++++++----------- 4 files changed, 43 insertions(+), 35 deletions(-) rename src/packages/backend/conat/sync-doc/{client.ts => sync-client.ts} (80%) diff --git a/src/packages/backend/conat/sync-doc/client.ts b/src/packages/backend/conat/sync-doc/sync-client.ts similarity index 80% rename from src/packages/backend/conat/sync-doc/client.ts rename to src/packages/backend/conat/sync-doc/sync-client.ts index 98fd947a57..ebc6377f8e 100644 --- a/src/packages/backend/conat/sync-doc/client.ts +++ b/src/packages/backend/conat/sync-doc/sync-client.ts @@ -10,11 +10,14 @@ import { PubSub } from "@cocalc/conat/sync/pubsub"; import { type Client as ConatClient } from "@cocalc/conat/core/client"; import { type ConatSyncTable } from "@cocalc/conat/sync/synctable"; -export class Client extends EventEmitter implements Client0 { - private conat: ConatClient; - constructor(conat: ConatClient) { +export class SyncClient extends EventEmitter implements Client0 { + private client: ConatClient; + constructor(client: ConatClient) { super(); - this.conat = conat; + if (client == null) { + throw Error("client must be specified"); + } + this.client = client; } is_project = (): boolean => false; @@ -26,11 +29,11 @@ export class Client extends EventEmitter implements Client0 { }; is_connected = (): boolean => { - return this.conat.isConnected(); + return this.client.isConnected(); }; is_signed_in = (): boolean => { - return this.conat.isSignedIn(); + return this.client.isSignedIn(); }; touch_project = (_): void => {}; @@ -43,18 +46,18 @@ export class Client extends EventEmitter implements Client0 { synctable_conat = async (query0, options?): Promise => { const { query } = parseQueryWithOptions(query0, options); - return await this.conat.sync.synctable({ + return await this.client.sync.synctable({ ...options, query, }); }; pubsub_conat = async (opts): Promise => { - return new PubSub({ client: this.conat, ...opts }); + return new PubSub({ client: this.client, ...opts }); }; // account_id or project_id - client_id = (): string => this.conat.id; + client_id = (): string => this.client.id; server_time = (): Date => { return new Date(); diff --git a/src/packages/backend/conat/sync-doc/syncstring.ts b/src/packages/backend/conat/sync-doc/syncstring.ts index ae1d17b335..ff6837b58a 100644 --- a/src/packages/backend/conat/sync-doc/syncstring.ts +++ b/src/packages/backend/conat/sync-doc/syncstring.ts @@ -1,25 +1,30 @@ -import { Client } from "./client"; +import { SyncClient } from "./sync-client"; import { SyncString } from "@cocalc/sync/editor/string/sync"; import { once } from "@cocalc/util/async-utils"; -import { type SyncDocFilesystem } from "@cocalc/sync/editor/generic/sync-doc"; import { type Client as ConatClient } from "@cocalc/conat/core/client"; +import { fsClient } from "@cocalc/conat/files/fs"; export default async function syncstring({ - fs, project_id, path, - conat, + client, + // name of the file server that hosts this document: + service, }: { - fs: SyncDocFilesystem; project_id: string; path: string; - conat: ConatClient; + client: ConatClient; + service?: string; }) { - const client = new Client(conat); + const fs = fsClient({ + subject: `${service}.project-${project_id}`, + client, + }); + const syncClient = new SyncClient(client); const syncstring = new SyncString({ project_id, path, - client, + client: syncClient, fs, }); await once(syncstring, "ready"); diff --git a/src/packages/backend/conat/sync-doc/test/setup.ts b/src/packages/backend/conat/sync-doc/test/setup.ts index 24c6f1e477..122b4200f7 100644 --- a/src/packages/backend/conat/sync-doc/test/setup.ts +++ b/src/packages/backend/conat/sync-doc/test/setup.ts @@ -11,23 +11,28 @@ import { import { type Filesystem } from "@cocalc/conat/files/fs"; export { uuid } from "@cocalc/util/misc"; import { fsClient } from "@cocalc/conat/files/fs"; +import syncstring0 from "@cocalc/backend/conat/sync-doc/syncstring"; export { client0 as client }; -export let server; +export let server, fs; export async function before() { await before0(); server = await createPathFileserver(); } -export function getFS(project_id: string, client): Filesystem { +export function getFS(project_id: string, client?): Filesystem { return fsClient({ subject: `${server.service}.project-${project_id}`, - client, + client: client ?? client0, }); } +export async function syncstring(opts) { + return await syncstring0({ ...opts, service: server.service }); +} + export async function after() { await cleanupFileservers(); await after0(); diff --git a/src/packages/backend/conat/sync-doc/test/syncstring.test.ts b/src/packages/backend/conat/sync-doc/test/syncstring.test.ts index f78a4116f1..20ee35f1c4 100644 --- a/src/packages/backend/conat/sync-doc/test/syncstring.test.ts +++ b/src/packages/backend/conat/sync-doc/test/syncstring.test.ts @@ -1,5 +1,4 @@ -import syncstring from "@cocalc/backend/conat/sync-doc/syncstring"; -import { before, after, getFS, uuid, wait, connect } from "./setup"; +import { before, after, uuid, wait, connect, syncstring, getFS } from "./setup"; beforeAll(before); afterAll(after); @@ -7,24 +6,24 @@ afterAll(after); describe("loading/saving syncstring to disk and setting values", () => { let s; const project_id = uuid(); - let fs, conat; + let client; it("creates the fs client", () => { - conat = connect(); - fs = getFS(project_id, conat); + client = connect(); }); it("a syncstring associated to a file that does not exist on disk is initialized to the empty string", async () => { - s = await syncstring({ fs, project_id, path: "new.txt", conat }); + s = await syncstring({ project_id, path: "new.txt", client }); expect(s.to_str()).toBe(""); expect(s.versions().length).toBe(0); s.close(); }); + let fs; it("a syncstring for editing a file that already exists on disk is initialized to that file", async () => { - fs = getFS(project_id, conat); + fs = getFS(project_id, client); await fs.writeFile("a.txt", "hello"); - s = await syncstring({ fs, project_id, path: "a.txt", conat }); + s = await syncstring({ project_id, path: "a.txt", client }); expect(s.fs).not.toEqual(undefined); }); @@ -64,24 +63,20 @@ describe("loading/saving syncstring to disk and setting values", () => { describe("synchronized editing with two copies of a syncstring", () => { const project_id = uuid(); - let s1, s2, fs1, fs2, client1, client2; + let s1, s2, client1, client2; it("creates the fs client and two copies of a syncstring", async () => { client1 = connect(); client2 = connect(); - fs1 = getFS(project_id, client1); s1 = await syncstring({ - fs: fs1, project_id, path: "a.txt", - conat: client1, + client: client1, }); - fs2 = getFS(project_id, client2); s2 = await syncstring({ - fs: fs2, project_id, path: "a.txt", - conat: client2, + client: client2, }); expect(s1.to_str()).toBe(""); expect(s2.to_str()).toBe(""); From 366ca14731598ba160985dc1a17c321075e1130f Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 23:11:09 +0000 Subject: [PATCH 030/798] make SyncString available from conat core client --- .../backend/conat/files/local-path.ts | 11 ++++---- .../{sync-doc/test => test/sync-doc}/setup.ts | 5 ++-- .../test => test/sync-doc}/syncstring.test.ts | 24 +++++++++++------ src/packages/conat/core/client.ts | 22 ++++++++++++++++ src/packages/conat/files/fs.ts | 1 + src/packages/conat/package.json | 16 ++++-------- .../conat/sync-doc/sync-client.ts | 0 .../conat/sync-doc/syncstring.ts | 26 +++++++++---------- src/packages/frontend/conat/client.ts | 3 --- src/packages/jupyter/package.json | 1 - src/packages/pnpm-lock.yaml | 20 +++----------- src/packages/sync/editor/generic/sync-doc.ts | 6 ++--- src/packages/util/package.json | 1 - 13 files changed, 72 insertions(+), 64 deletions(-) rename src/packages/backend/conat/{sync-doc/test => test/sync-doc}/setup.ts (81%) rename src/packages/backend/conat/{sync-doc/test => test/sync-doc}/syncstring.test.ts (84%) rename src/packages/{backend => }/conat/sync-doc/sync-client.ts (100%) rename src/packages/{backend => }/conat/sync-doc/syncstring.ts (75%) diff --git a/src/packages/backend/conat/files/local-path.ts b/src/packages/backend/conat/files/local-path.ts index 3ba4fb5dad..bc905561a6 100644 --- a/src/packages/backend/conat/files/local-path.ts +++ b/src/packages/backend/conat/files/local-path.ts @@ -1,20 +1,21 @@ -import { fsServer } from "@cocalc/conat/files/fs"; +import { fsServer, DEFAULT_FILE_SERVICE } from "@cocalc/conat/files/fs"; import { SandboxedFilesystem } from "@cocalc/backend/files/sandbox"; import { mkdir } from "fs/promises"; import { join } from "path"; import { isValidUUID } from "@cocalc/util/misc"; -import { type Client, getClient } from "@cocalc/conat/core/client"; +import { type Client } from "@cocalc/conat/core/client"; +import { conat } from "@cocalc/backend/conat/conat"; export async function localPathFileserver({ - service, path, + service = DEFAULT_FILE_SERVICE, client, }: { - service: string; path: string; + service?: string; client?: Client; }) { - client ??= getClient(); + client ??= conat(); const server = await fsServer({ service, client, diff --git a/src/packages/backend/conat/sync-doc/test/setup.ts b/src/packages/backend/conat/test/sync-doc/setup.ts similarity index 81% rename from src/packages/backend/conat/sync-doc/test/setup.ts rename to src/packages/backend/conat/test/sync-doc/setup.ts index 122b4200f7..d89109ff7a 100644 --- a/src/packages/backend/conat/sync-doc/test/setup.ts +++ b/src/packages/backend/conat/test/sync-doc/setup.ts @@ -11,7 +11,8 @@ import { import { type Filesystem } from "@cocalc/conat/files/fs"; export { uuid } from "@cocalc/util/misc"; import { fsClient } from "@cocalc/conat/files/fs"; -import syncstring0 from "@cocalc/backend/conat/sync-doc/syncstring"; +import { syncstring as syncstring0 } from "@cocalc/conat/sync-doc/syncstring"; +import { SyncString } from "@cocalc/sync/editor/string/sync"; export { client0 as client }; @@ -29,7 +30,7 @@ export function getFS(project_id: string, client?): Filesystem { }); } -export async function syncstring(opts) { +export async function syncstring(opts): Promise { return await syncstring0({ ...opts, service: server.service }); } diff --git a/src/packages/backend/conat/sync-doc/test/syncstring.test.ts b/src/packages/backend/conat/test/sync-doc/syncstring.test.ts similarity index 84% rename from src/packages/backend/conat/sync-doc/test/syncstring.test.ts rename to src/packages/backend/conat/test/sync-doc/syncstring.test.ts index 20ee35f1c4..c02c2caed8 100644 --- a/src/packages/backend/conat/sync-doc/test/syncstring.test.ts +++ b/src/packages/backend/conat/test/sync-doc/syncstring.test.ts @@ -1,4 +1,4 @@ -import { before, after, uuid, wait, connect, syncstring, getFS } from "./setup"; +import { before, after, uuid, wait, connect, server } from "./setup"; beforeAll(before); afterAll(after); @@ -13,7 +13,11 @@ describe("loading/saving syncstring to disk and setting values", () => { }); it("a syncstring associated to a file that does not exist on disk is initialized to the empty string", async () => { - s = await syncstring({ project_id, path: "new.txt", client }); + s = await client.sync.string({ + project_id, + path: "new.txt", + service: server.service, + }); expect(s.to_str()).toBe(""); expect(s.versions().length).toBe(0); s.close(); @@ -21,9 +25,13 @@ describe("loading/saving syncstring to disk and setting values", () => { let fs; it("a syncstring for editing a file that already exists on disk is initialized to that file", async () => { - fs = getFS(project_id, client); + fs = client.fs({ project_id, service: server.service }); await fs.writeFile("a.txt", "hello"); - s = await syncstring({ project_id, path: "a.txt", client }); + s = await client.sync.string({ + project_id, + path: "a.txt", + service: server.service, + }); expect(s.fs).not.toEqual(undefined); }); @@ -68,15 +76,15 @@ describe("synchronized editing with two copies of a syncstring", () => { it("creates the fs client and two copies of a syncstring", async () => { client1 = connect(); client2 = connect(); - s1 = await syncstring({ + s1 = await client1.sync.string({ project_id, path: "a.txt", - client: client1, + service: server.service, }); - s2 = await syncstring({ + s2 = await client2.sync.string({ project_id, path: "a.txt", - client: client2, + service: server.service, }); expect(s1.to_str()).toBe(""); expect(s2.to_str()).toBe(""); diff --git a/src/packages/conat/core/client.ts b/src/packages/conat/core/client.ts index 6bba37d09b..eedf71e197 100644 --- a/src/packages/conat/core/client.ts +++ b/src/packages/conat/core/client.ts @@ -244,6 +244,12 @@ import { } from "@cocalc/conat/sync/dstream"; import { akv, type AKV } from "@cocalc/conat/sync/akv"; import { astream, type AStream } from "@cocalc/conat/sync/astream"; +import { + syncstring, + type SyncString, + type SyncStringOptions, +} from "@cocalc/conat/sync-doc/syncstring"; +import { fsClient, DEFAULT_FILE_SERVICE } from "@cocalc/conat/files/fs"; import TTL from "@isaacs/ttlcache"; import { ConatSocketServer, @@ -1462,6 +1468,19 @@ export class Client extends EventEmitter { return sub; }; + fs = ({ + project_id, + service = DEFAULT_FILE_SERVICE, + }: { + project_id: string; + service?: string; + }) => { + return fsClient({ + subject: `${service}.project-${project_id}`, + client: this, + }); + }; + sync = { dkv: async (opts: DKVOptions): Promise> => await dkv({ ...opts, client: this }), @@ -1475,6 +1494,9 @@ export class Client extends EventEmitter { await astream({ ...opts, client: this }), synctable: async (opts: SyncTableOptions): Promise => await createSyncTable({ ...opts, client: this }), + string: async ( + opts: Omit, + ): Promise => await syncstring({ ...opts, client: this }), }; socket = { diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index afa7826f66..a06baa02bb 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -1,6 +1,7 @@ import { type Client } from "@cocalc/conat/core/client"; import { conat } from "@cocalc/conat/client"; import { watchServer, watchClient } from "@cocalc/conat/files/watch"; +export const DEFAULT_FILE_SERVICE = "fs"; export interface Filesystem { appendFile: (path: string, data: string | Buffer, encoding?) => Promise; diff --git a/src/packages/conat/package.json b/src/packages/conat/package.json index 79bfe77778..1d8dd0f6ca 100644 --- a/src/packages/conat/package.json +++ b/src/packages/conat/package.json @@ -7,6 +7,8 @@ "./llm/*": "./dist/llm/*.js", "./socket": "./dist/socket/index.js", "./socket/*": "./dist/socket/*.js", + "./sync-doc": "./dist/sync-doc/index.js", + "./sync-doc/*": "./dist/sync-doc/*.js", "./hub/changefeeds": "./dist/hub/changefeeds/index.js", "./hub/api": "./dist/hub/api/index.js", "./hub/api/*": "./dist/hub/api/*.js", @@ -24,21 +26,14 @@ "test": "pnpm exec jest", "depcheck": "pnpx depcheck --ignores events" }, - "files": [ - "dist/**", - "README.md", - "package.json" - ], + "files": ["dist/**", "README.md", "package.json"], "author": "SageMath, Inc.", - "keywords": [ - "utilities", - "conat", - "cocalc" - ], + "keywords": ["utilities", "conat", "cocalc"], "license": "SEE LICENSE.md", "dependencies": { "@cocalc/comm": "workspace:*", "@cocalc/conat": "workspace:*", + "@cocalc/sync": "workspace:*", "@cocalc/util": "workspace:*", "@isaacs/ttlcache": "^1.4.1", "@msgpack/msgpack": "^3.1.1", @@ -56,7 +51,6 @@ }, "devDependencies": { "@types/better-sqlite3": "^7.6.13", - "@types/json-stable-stringify": "^1.0.32", "@types/lodash": "^4.14.202", "@types/node": "^18.16.14" }, diff --git a/src/packages/backend/conat/sync-doc/sync-client.ts b/src/packages/conat/sync-doc/sync-client.ts similarity index 100% rename from src/packages/backend/conat/sync-doc/sync-client.ts rename to src/packages/conat/sync-doc/sync-client.ts diff --git a/src/packages/backend/conat/sync-doc/syncstring.ts b/src/packages/conat/sync-doc/syncstring.ts similarity index 75% rename from src/packages/backend/conat/sync-doc/syncstring.ts rename to src/packages/conat/sync-doc/syncstring.ts index ff6837b58a..2c7fcb795e 100644 --- a/src/packages/backend/conat/sync-doc/syncstring.ts +++ b/src/packages/conat/sync-doc/syncstring.ts @@ -2,24 +2,24 @@ import { SyncClient } from "./sync-client"; import { SyncString } from "@cocalc/sync/editor/string/sync"; import { once } from "@cocalc/util/async-utils"; import { type Client as ConatClient } from "@cocalc/conat/core/client"; -import { fsClient } from "@cocalc/conat/files/fs"; -export default async function syncstring({ - project_id, - path, - client, - // name of the file server that hosts this document: - service, -}: { +export interface SyncStringOptions { project_id: string; path: string; client: ConatClient; + // name of the file server that hosts this document: service?: string; -}) { - const fs = fsClient({ - subject: `${service}.project-${project_id}`, - client, - }); +} + +export type { SyncString }; + +export async function syncstring({ + project_id, + path, + client, + service, +}: SyncStringOptions): Promise { + const fs = client.fs({ service, project_id }); const syncClient = new SyncClient(client); const syncstring = new SyncString({ project_id, diff --git a/src/packages/frontend/conat/client.ts b/src/packages/frontend/conat/client.ts index 878ef21973..dedb2d4533 100644 --- a/src/packages/frontend/conat/client.ts +++ b/src/packages/frontend/conat/client.ts @@ -46,7 +46,6 @@ import { deleteRememberMe, setRememberMe, } from "@cocalc/frontend/misc/remember-me"; -import { fsClient } from "@cocalc/conat/files/fs"; export interface ConatConnectionStatus { state: "connected" | "disconnected"; @@ -513,8 +512,6 @@ export class ConatClient extends EventEmitter { }; refCacheInfo = () => refCacheInfo(); - - fsClient = (subject: string) => fsClient({ subject, client: this.conat() }); } function setDeleted({ project_id, path, deleted }) { diff --git a/src/packages/jupyter/package.json b/src/packages/jupyter/package.json index d1e3f3ab14..a109440ca8 100644 --- a/src/packages/jupyter/package.json +++ b/src/packages/jupyter/package.json @@ -63,7 +63,6 @@ "zeromq": "^6.4.2" }, "devDependencies": { - "@types/json-stable-stringify": "^1.0.32", "@types/node": "^18.16.14", "@types/node-cleanup": "^2.1.2" }, diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 930789121f..135d788520 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -170,6 +170,9 @@ importers: '@cocalc/conat': specifier: workspace:* version: 'link:' + '@cocalc/sync': + specifier: workspace:* + version: link:../sync '@cocalc/util': specifier: workspace:* version: link:../util @@ -216,9 +219,6 @@ importers: '@types/better-sqlite3': specifier: ^7.6.13 version: 7.6.13 - '@types/json-stable-stringify': - specifier: ^1.0.32 - version: 1.2.0 '@types/lodash': specifier: ^4.14.202 version: 4.17.20 @@ -899,9 +899,6 @@ importers: specifier: ^6.4.2 version: 6.5.0 devDependencies: - '@types/json-stable-stringify': - specifier: ^1.0.32 - version: 1.2.0 '@types/node': specifier: ^18.16.14 version: 18.19.118 @@ -1849,9 +1846,6 @@ importers: specifier: ^1.3.0 version: 1.3.0 devDependencies: - '@types/json-stable-stringify': - specifier: ^1.0.32 - version: 1.2.0 '@types/lodash': specifier: ^4.14.202 version: 4.17.20 @@ -4269,10 +4263,6 @@ packages: '@types/json-schema@7.0.15': resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} - '@types/json-stable-stringify@1.2.0': - resolution: {integrity: sha512-PEHY3ohqolHqAzDyB1+31tFaAMnoLN7x/JgdcGmNZ2uvtEJ6rlFCUYNQc0Xe754xxCYLNGZbLUGydSE6tS4S9A==} - deprecated: This is a stub types definition. json-stable-stringify provides its own type definitions, so you do not need this installed. - '@types/katex@0.16.7': resolution: {integrity: sha512-HMwFiRujE5PjrgwHQ25+bsLJgowjGjm5Z8FVSf0N6PwgJrwxH0QxzHYDcKsTfV3wva0vzrpqMTJS2jXPr5BMEQ==} @@ -14546,10 +14536,6 @@ snapshots: '@types/json-schema@7.0.15': {} - '@types/json-stable-stringify@1.2.0': - dependencies: - json-stable-stringify: 1.3.0 - '@types/katex@0.16.7': {} '@types/keyv@3.1.4': diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index cb971876cd..2fed66dd7a 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -2903,9 +2903,6 @@ export class SyncDoc extends EventEmitter { if (this.fs != null) { return; } - if (this.client.path_exists == null) { - throw Error("legacy clients must define path_exists"); - } const dbg = this.dbg("update_watch_path"); if (this.file_watcher != null) { // clean up @@ -2932,6 +2929,9 @@ export class SyncDoc extends EventEmitter { if (this.state === "closed") { throw Error("must not be closed"); } + if (this.client.path_exists == null) { + throw Error("legacy clients must define path_exists"); + } this.watch_path = path; try { if (!(await callback2(this.client.path_exists, { path }))) { diff --git a/src/packages/util/package.json b/src/packages/util/package.json index e9656ac1e3..62d8367572 100644 --- a/src/packages/util/package.json +++ b/src/packages/util/package.json @@ -71,7 +71,6 @@ }, "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/util", "devDependencies": { - "@types/json-stable-stringify": "^1.0.32", "@types/lodash": "^4.14.202", "@types/node": "^18.16.14", "@types/seedrandom": "^3.0.8", From 205f14db5730889369c479e4b8b4ac998fb9308b Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 23:29:25 +0000 Subject: [PATCH 031/798] conat sync/fileserver: integrate with hub --- src/packages/hub/hub.ts | 11 ++++++++++- src/packages/server/conat/index.ts | 16 +++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/packages/hub/hub.ts b/src/packages/hub/hub.ts index 5e5b84d4f1..500b48ad3d 100644 --- a/src/packages/hub/hub.ts +++ b/src/packages/hub/hub.ts @@ -44,6 +44,7 @@ import { initConatChangefeedServer, initConatApi, initConatPersist, + initConatFileserver, } from "@cocalc/server/conat"; import { initConatServer } from "@cocalc/server/conat/socketio"; @@ -182,6 +183,10 @@ async function startServer(): Promise { await initConatServer({ kucalc: program.mode == "kucalc" }); } + if (program.conatFileserver || program.conatServer) { + await initConatFileserver(); + } + if (program.conatApi || program.conatServer) { await initConatApi(); await initConatChangefeedServer(); @@ -318,12 +323,16 @@ async function main(): Promise { ) .option( "--conat-server", - "run a hub that provides a single-core conat server (i.e., conat-router but integrated with the http server), api, and persistence, along with an http server. This is for dev and small deployments of cocalc (and if given, do not bother with --conat-[core|api|persist] below.)", + "run a hub that provides a single-core conat server (i.e., conat-router but integrated with the http server), api, and persistence, fileserver, along with an http server. This is for dev and small deployments of cocalc (and if given, do not bother with --conat-[core|api|persist] below.)", ) .option( "--conat-router", "run a hub that provides the core conat communication layer server over a websocket (but not http server).", ) + .option( + "--conat-fileserver", + "run a hub that provides a fileserver conat service", + ) .option( "--conat-api", "run a hub that connect to conat-router and provides the standard conat API services, e.g., basic api, LLM's, changefeeds, http file upload/download, etc. There must be at least one of these. You can increase or decrease the number of these servers with no coordination needed.", diff --git a/src/packages/server/conat/index.ts b/src/packages/server/conat/index.ts index 56e08e45b5..49e3d4b142 100644 --- a/src/packages/server/conat/index.ts +++ b/src/packages/server/conat/index.ts @@ -5,7 +5,8 @@ import { init as initLLM } from "./llm"; import { loadConatConfiguration } from "./configuration"; import { createTimeService } from "@cocalc/conat/service/time"; export { initConatPersist } from "./persist"; -import { conatApiCount } from "@cocalc/backend/data"; +import { conatApiCount, projects } from "@cocalc/backend/data"; +import { localPathFileserver } from "@cocalc/backend/conat/files/local-path"; export { loadConatConfiguration }; @@ -30,3 +31,16 @@ export async function initConatApi() { initLLM(); createTimeService(); } + +export async function initConatFileserver() { + await loadConatConfiguration(); + const i = projects.indexOf("/[project_id]"); + if (i == -1) { + throw Error( + `projects must be a template containing /[project_id] -- ${projects}`, + ); + } + const path = projects.slice(0, i); + logger.debug("initFileserver", { path }); + localPathFileserver({ path }); +} From 67a3d1d4cbef72ce06d130d29d49448d48cbb3bd Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 23:46:49 +0000 Subject: [PATCH 032/798] conat sync-doc: make sync.string not async to match existing api usage - i might change this everywhere; not sure --- .../backend/conat/test/sync-doc/setup.ts | 17 +---------------- .../conat/test/sync-doc/syncstring.test.ts | 15 ++++++++++----- src/packages/conat/core/client.ts | 5 ++--- src/packages/conat/sync-doc/syncstring.ts | 9 +++------ .../frontend/frame-editors/generic/client.ts | 1 + 5 files changed, 17 insertions(+), 30 deletions(-) diff --git a/src/packages/backend/conat/test/sync-doc/setup.ts b/src/packages/backend/conat/test/sync-doc/setup.ts index d89109ff7a..0155bb46f4 100644 --- a/src/packages/backend/conat/test/sync-doc/setup.ts +++ b/src/packages/backend/conat/test/sync-doc/setup.ts @@ -3,16 +3,12 @@ import { after as after0, client as client0, } from "@cocalc/backend/conat/test/setup"; -export { connect, wait } from "@cocalc/backend/conat/test/setup"; +export { connect, wait, once } from "@cocalc/backend/conat/test/setup"; import { createPathFileserver, cleanupFileservers, } from "@cocalc/backend/conat/files/test/util"; -import { type Filesystem } from "@cocalc/conat/files/fs"; export { uuid } from "@cocalc/util/misc"; -import { fsClient } from "@cocalc/conat/files/fs"; -import { syncstring as syncstring0 } from "@cocalc/conat/sync-doc/syncstring"; -import { SyncString } from "@cocalc/sync/editor/string/sync"; export { client0 as client }; @@ -23,17 +19,6 @@ export async function before() { server = await createPathFileserver(); } -export function getFS(project_id: string, client?): Filesystem { - return fsClient({ - subject: `${server.service}.project-${project_id}`, - client: client ?? client0, - }); -} - -export async function syncstring(opts): Promise { - return await syncstring0({ ...opts, service: server.service }); -} - export async function after() { await cleanupFileservers(); await after0(); diff --git a/src/packages/backend/conat/test/sync-doc/syncstring.test.ts b/src/packages/backend/conat/test/sync-doc/syncstring.test.ts index c02c2caed8..d1d73332a7 100644 --- a/src/packages/backend/conat/test/sync-doc/syncstring.test.ts +++ b/src/packages/backend/conat/test/sync-doc/syncstring.test.ts @@ -1,4 +1,4 @@ -import { before, after, uuid, wait, connect, server } from "./setup"; +import { before, after, uuid, wait, connect, server, once } from "./setup"; beforeAll(before); afterAll(after); @@ -13,11 +13,12 @@ describe("loading/saving syncstring to disk and setting values", () => { }); it("a syncstring associated to a file that does not exist on disk is initialized to the empty string", async () => { - s = await client.sync.string({ + s = client.sync.string({ project_id, path: "new.txt", service: server.service, }); + await once(s, "ready"); expect(s.to_str()).toBe(""); expect(s.versions().length).toBe(0); s.close(); @@ -27,11 +28,12 @@ describe("loading/saving syncstring to disk and setting values", () => { it("a syncstring for editing a file that already exists on disk is initialized to that file", async () => { fs = client.fs({ project_id, service: server.service }); await fs.writeFile("a.txt", "hello"); - s = await client.sync.string({ + s = client.sync.string({ project_id, path: "a.txt", service: server.service, }); + await once(s, "ready"); expect(s.fs).not.toEqual(undefined); }); @@ -76,16 +78,19 @@ describe("synchronized editing with two copies of a syncstring", () => { it("creates the fs client and two copies of a syncstring", async () => { client1 = connect(); client2 = connect(); - s1 = await client1.sync.string({ + s1 = client1.sync.string({ project_id, path: "a.txt", service: server.service, }); - s2 = await client2.sync.string({ + await once(s1, "ready"); + + s2 = client2.sync.string({ project_id, path: "a.txt", service: server.service, }); + await once(s2, "ready"); expect(s1.to_str()).toBe(""); expect(s2.to_str()).toBe(""); expect(s1 === s2).toBe(false); diff --git a/src/packages/conat/core/client.ts b/src/packages/conat/core/client.ts index eedf71e197..9a3d8d3dc4 100644 --- a/src/packages/conat/core/client.ts +++ b/src/packages/conat/core/client.ts @@ -1494,9 +1494,8 @@ export class Client extends EventEmitter { await astream({ ...opts, client: this }), synctable: async (opts: SyncTableOptions): Promise => await createSyncTable({ ...opts, client: this }), - string: async ( - opts: Omit, - ): Promise => await syncstring({ ...opts, client: this }), + string: (opts: Omit): SyncString => + syncstring({ ...opts, client: this }), }; socket = { diff --git a/src/packages/conat/sync-doc/syncstring.ts b/src/packages/conat/sync-doc/syncstring.ts index 2c7fcb795e..474acd8488 100644 --- a/src/packages/conat/sync-doc/syncstring.ts +++ b/src/packages/conat/sync-doc/syncstring.ts @@ -1,6 +1,5 @@ import { SyncClient } from "./sync-client"; import { SyncString } from "@cocalc/sync/editor/string/sync"; -import { once } from "@cocalc/util/async-utils"; import { type Client as ConatClient } from "@cocalc/conat/core/client"; export interface SyncStringOptions { @@ -13,20 +12,18 @@ export interface SyncStringOptions { export type { SyncString }; -export async function syncstring({ +export function syncstring({ project_id, path, client, service, -}: SyncStringOptions): Promise { +}: SyncStringOptions): SyncString { const fs = client.fs({ service, project_id }); const syncClient = new SyncClient(client); - const syncstring = new SyncString({ + return new SyncString({ project_id, path, client: syncClient, fs, }); - await once(syncstring, "ready"); - return syncstring; } diff --git a/src/packages/frontend/frame-editors/generic/client.ts b/src/packages/frontend/frame-editors/generic/client.ts index 7c2bdbfe19..db1c708a39 100644 --- a/src/packages/frontend/frame-editors/generic/client.ts +++ b/src/packages/frontend/frame-editors/generic/client.ts @@ -218,6 +218,7 @@ interface SyncstringOpts2 { } export function syncstring2(opts: SyncstringOpts2): SyncString { + // return webapp_client.conat_client.conat().sync.string(opts); const opts1: any = opts; opts1.client = webapp_client; return webapp_client.sync_client.sync_string(opts1); From 088dbab45ca09c5a6e1f52746d1bed9d50cfcb38 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 20 Jul 2025 03:12:09 +0000 Subject: [PATCH 033/798] syncdb: implement similar to syncstring --- .../backend/conat/test/sync-doc/setup.ts | 2 +- .../conat/test/sync-doc/syncdb.test.ts | 99 +++++++++++++++++++ .../conat/test/sync-doc/syncstring.test.ts | 2 +- src/packages/conat/core/client.ts | 7 ++ src/packages/conat/sync-doc/syncdb.ts | 17 ++++ src/packages/conat/sync-doc/syncstring.ts | 26 ++--- .../frontend/frame-editors/generic/client.ts | 20 ++-- src/packages/sync/editor/string/index.ts | 1 + src/packages/sync/editor/string/sync.ts | 2 + 9 files changed, 149 insertions(+), 27 deletions(-) create mode 100644 src/packages/backend/conat/test/sync-doc/syncdb.test.ts create mode 100644 src/packages/conat/sync-doc/syncdb.ts diff --git a/src/packages/backend/conat/test/sync-doc/setup.ts b/src/packages/backend/conat/test/sync-doc/setup.ts index 0155bb46f4..5af527bf53 100644 --- a/src/packages/backend/conat/test/sync-doc/setup.ts +++ b/src/packages/backend/conat/test/sync-doc/setup.ts @@ -3,7 +3,7 @@ import { after as after0, client as client0, } from "@cocalc/backend/conat/test/setup"; -export { connect, wait, once } from "@cocalc/backend/conat/test/setup"; +export { connect, wait, once, delay } from "@cocalc/backend/conat/test/setup"; import { createPathFileserver, cleanupFileservers, diff --git a/src/packages/backend/conat/test/sync-doc/syncdb.test.ts b/src/packages/backend/conat/test/sync-doc/syncdb.test.ts new file mode 100644 index 0000000000..24e2706f52 --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/syncdb.test.ts @@ -0,0 +1,99 @@ +import { + before, + after, + uuid, + wait, + connect, + server, + once, +} from "./setup"; + +beforeAll(before); +afterAll(after); + +describe("loading/saving syncstring to disk and setting values", () => { + let s; + const project_id = uuid(); + let client; + + it("creates a client", () => { + client = connect(); + }); + + it("a syncdb associated to a file that does not exist on disk is initialized to empty", async () => { + s = client.sync.db({ + project_id, + path: "new.syncdb", + service: server.service, + primary_keys: ["name"], + }); + await once(s, "ready"); + expect(s.to_str()).toBe(""); + expect(s.versions().length).toBe(0); + }); + + it("store a record", async () => { + s.set({ name: "cocalc", value: 10 }); + expect(s.to_str()).toBe('{"name":"cocalc","value":10}'); + const t = s.get_one({ name: "cocalc" }).toJS(); + expect(t).toEqual({ name: "cocalc", value: 10 }); + await s.commit(); + await s.save(); + // [ ] TODO: this save to disk definitely should NOT be needed + await s.save_to_disk(); + }); + + let client2, s2; + it("connect another client", async () => { + client2 = connect(); + // [ ] loading this resets the state if we do not save above. + s2 = client2.sync.db({ + project_id, + path: "new.syncdb", + service: server.service, + primary_keys: ["name"], + }); + await once(s2, "ready"); + expect(s2).not.toBe(s); + expect(s2.to_str()).toBe('{"name":"cocalc","value":10}'); + const t = s2.get_one({ name: "cocalc" }).toJS(); + expect(t).toEqual({ name: "cocalc", value: 10 }); + + s2.set({ name: "conat", date: new Date() }); + s2.commit(); + await s2.save(); + }); + + it("verifies the change on s2 is seen by s (and also that Date objects do NOT work)", async () => { + await wait({ until: () => s.get_one({ name: "conat" }) != null }); + const t = s.get_one({ name: "conat" }).toJS(); + expect(t).toEqual({ name: "conat", date: t.date }); + // They don't work because we're storing syncdb's in jsonl format, + // so json is used. We should have a new format called + // msgpackl and start using that. + expect(t.date instanceof Date).toBe(false); + }); + + const count = 1000; + it(`store ${count} records`, async () => { + const before = s.get().size; + for (let i = 0; i < count; i++) { + s.set({ name: i }); + } + s.commit(); + await s.save(); + expect(s.get().size).toBe(count + before); + }); + + it("confirm file saves to disk with many lines", async () => { + await s.save_to_disk(); + const v = (await s.fs.readFile("new.syncdb", "utf8")).split("\n"); + expect(v.length).toBe(s.get().size); + }); + + it("verifies lookups are not too slow (there is an index)", () => { + for (let i = 0; i < count; i++) { + expect(s.get_one({ name: i }).get("name")).toEqual(i); + } + }); +}); diff --git a/src/packages/backend/conat/test/sync-doc/syncstring.test.ts b/src/packages/backend/conat/test/sync-doc/syncstring.test.ts index d1d73332a7..e67361ddfb 100644 --- a/src/packages/backend/conat/test/sync-doc/syncstring.test.ts +++ b/src/packages/backend/conat/test/sync-doc/syncstring.test.ts @@ -8,7 +8,7 @@ describe("loading/saving syncstring to disk and setting values", () => { const project_id = uuid(); let client; - it("creates the fs client", () => { + it("creates the client", () => { client = connect(); }); diff --git a/src/packages/conat/core/client.ts b/src/packages/conat/core/client.ts index 9a3d8d3dc4..bfa92256f6 100644 --- a/src/packages/conat/core/client.ts +++ b/src/packages/conat/core/client.ts @@ -249,6 +249,11 @@ import { type SyncString, type SyncStringOptions, } from "@cocalc/conat/sync-doc/syncstring"; +import { + syncdb, + type SyncDB, + type SyncDBOptions, +} from "@cocalc/conat/sync-doc/syncdb"; import { fsClient, DEFAULT_FILE_SERVICE } from "@cocalc/conat/files/fs"; import TTL from "@isaacs/ttlcache"; import { @@ -1496,6 +1501,8 @@ export class Client extends EventEmitter { await createSyncTable({ ...opts, client: this }), string: (opts: Omit): SyncString => syncstring({ ...opts, client: this }), + db: (opts: Omit): SyncDB => + syncdb({ ...opts, client: this }), }; socket = { diff --git a/src/packages/conat/sync-doc/syncdb.ts b/src/packages/conat/sync-doc/syncdb.ts new file mode 100644 index 0000000000..1ef7b135c0 --- /dev/null +++ b/src/packages/conat/sync-doc/syncdb.ts @@ -0,0 +1,17 @@ +import { SyncClient } from "./sync-client"; +import { SyncDB, type SyncDBOpts } from "@cocalc/sync/editor/db"; +import { type Client as ConatClient } from "@cocalc/conat/core/client"; + +export interface SyncDBOptions extends Omit { + client: ConatClient; + // name of the file service that hosts this file: + service?: string; +} + +export type { SyncDB }; + +export function syncdb({ client, service, ...opts }: SyncDBOptions): SyncDB { + const fs = client.fs({ service, project_id: opts.project_id }); + const syncClient = new SyncClient(client); + return new SyncDB({ ...opts, fs, client: syncClient }); +} diff --git a/src/packages/conat/sync-doc/syncstring.ts b/src/packages/conat/sync-doc/syncstring.ts index 474acd8488..5fb6d2e2fb 100644 --- a/src/packages/conat/sync-doc/syncstring.ts +++ b/src/packages/conat/sync-doc/syncstring.ts @@ -1,10 +1,11 @@ import { SyncClient } from "./sync-client"; -import { SyncString } from "@cocalc/sync/editor/string/sync"; +import { + SyncString, + type SyncStringOpts, +} from "@cocalc/sync/editor/string/sync"; import { type Client as ConatClient } from "@cocalc/conat/core/client"; -export interface SyncStringOptions { - project_id: string; - path: string; +export interface SyncStringOptions extends Omit { client: ConatClient; // name of the file server that hosts this document: service?: string; @@ -12,18 +13,9 @@ export interface SyncStringOptions { export type { SyncString }; -export function syncstring({ - project_id, - path, - client, - service, -}: SyncStringOptions): SyncString { - const fs = client.fs({ service, project_id }); +export function syncstring({ client, service, ...opts }: SyncStringOptions): SyncString { + const fs = client.fs({ service, project_id: opts.project_id }); const syncClient = new SyncClient(client); - return new SyncString({ - project_id, - path, - client: syncClient, - fs, - }); + return new SyncString({ ...opts, fs, client: syncClient }); } + diff --git a/src/packages/frontend/frame-editors/generic/client.ts b/src/packages/frontend/frame-editors/generic/client.ts index db1c708a39..67d143f53a 100644 --- a/src/packages/frontend/frame-editors/generic/client.ts +++ b/src/packages/frontend/frame-editors/generic/client.ts @@ -200,7 +200,8 @@ export function syncstring(opts: SyncstringOpts): any { delete opts.fake; } opts1.id = schema.client_db.sha1(opts.project_id, opts.path); - return webapp_client.sync_string(opts1); + return webapp_client.conat_client.conat().sync.string(opts1); + // return webapp_client.sync_string(opts1); } import { DataServer } from "@cocalc/sync/editor/generic/sync-doc"; @@ -218,10 +219,10 @@ interface SyncstringOpts2 { } export function syncstring2(opts: SyncstringOpts2): SyncString { - // return webapp_client.conat_client.conat().sync.string(opts); - const opts1: any = opts; - opts1.client = webapp_client; - return webapp_client.sync_client.sync_string(opts1); + return webapp_client.conat_client.conat().sync.string(opts); + // const opts1: any = opts; + // opts1.client = webapp_client; + // return webapp_client.sync_client.sync_string(opts1); } export interface SyncDBOpts { @@ -239,8 +240,10 @@ export interface SyncDBOpts { } export function syncdb(opts: SyncDBOpts): any { - const opts1: any = opts; - return webapp_client.sync_db(opts1); + return webapp_client.conat_client.conat().sync.db(opts); + + // const opts1: any = opts; + // return webapp_client.sync_db(opts1); } import type { SyncDB } from "@cocalc/sync/editor/db/sync"; @@ -251,7 +254,8 @@ export function syncdb2(opts: SyncDBOpts): SyncDB { } const opts1: any = opts; opts1.client = webapp_client; - return webapp_client.sync_client.sync_db(opts1); + return webapp_client.conat_client.conat().sync.db(opts1); + // return webapp_client.sync_client.sync_db(opts1); } interface QueryOpts { diff --git a/src/packages/sync/editor/string/index.ts b/src/packages/sync/editor/string/index.ts index c1fed4e3a5..5a43e5f755 100644 --- a/src/packages/sync/editor/string/index.ts +++ b/src/packages/sync/editor/string/index.ts @@ -1 +1,2 @@ export { SyncString } from "./sync"; +export { type SyncStringOpts } from "./sync"; diff --git a/src/packages/sync/editor/string/sync.ts b/src/packages/sync/editor/string/sync.ts index 780c066370..95494d9f1e 100644 --- a/src/packages/sync/editor/string/sync.ts +++ b/src/packages/sync/editor/string/sync.ts @@ -6,6 +6,8 @@ import { SyncDoc, SyncOpts0, SyncOpts } from "../generic/sync-doc"; import { StringDocument } from "./doc"; +export type SyncStringOpts = SyncOpts0; + export class SyncString extends SyncDoc { constructor(opts: SyncOpts0) { // TS question -- What is the right way to do this? From 1304de4c77b567828dcaf44f1293f5a0ccf89caf Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 20 Jul 2025 03:12:09 +0000 Subject: [PATCH 034/798] syncdb: implement similar to syncstring --- .../backend/conat/test/sync-doc/setup.ts | 2 +- .../conat/test/sync-doc/syncdb.test.ts | 99 +++++++++++++++++++ .../conat/test/sync-doc/syncstring.test.ts | 2 +- src/packages/conat/core/client.ts | 7 ++ src/packages/conat/sync-doc/syncdb.ts | 17 ++++ src/packages/conat/sync-doc/syncstring.ts | 26 ++--- .../frontend/frame-editors/generic/client.ts | 20 ++-- src/packages/sync/editor/db/sync.ts | 6 +- src/packages/sync/editor/string/index.ts | 1 + src/packages/sync/editor/string/sync.ts | 2 + 10 files changed, 152 insertions(+), 30 deletions(-) create mode 100644 src/packages/backend/conat/test/sync-doc/syncdb.test.ts create mode 100644 src/packages/conat/sync-doc/syncdb.ts diff --git a/src/packages/backend/conat/test/sync-doc/setup.ts b/src/packages/backend/conat/test/sync-doc/setup.ts index 0155bb46f4..5af527bf53 100644 --- a/src/packages/backend/conat/test/sync-doc/setup.ts +++ b/src/packages/backend/conat/test/sync-doc/setup.ts @@ -3,7 +3,7 @@ import { after as after0, client as client0, } from "@cocalc/backend/conat/test/setup"; -export { connect, wait, once } from "@cocalc/backend/conat/test/setup"; +export { connect, wait, once, delay } from "@cocalc/backend/conat/test/setup"; import { createPathFileserver, cleanupFileservers, diff --git a/src/packages/backend/conat/test/sync-doc/syncdb.test.ts b/src/packages/backend/conat/test/sync-doc/syncdb.test.ts new file mode 100644 index 0000000000..24e2706f52 --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/syncdb.test.ts @@ -0,0 +1,99 @@ +import { + before, + after, + uuid, + wait, + connect, + server, + once, +} from "./setup"; + +beforeAll(before); +afterAll(after); + +describe("loading/saving syncstring to disk and setting values", () => { + let s; + const project_id = uuid(); + let client; + + it("creates a client", () => { + client = connect(); + }); + + it("a syncdb associated to a file that does not exist on disk is initialized to empty", async () => { + s = client.sync.db({ + project_id, + path: "new.syncdb", + service: server.service, + primary_keys: ["name"], + }); + await once(s, "ready"); + expect(s.to_str()).toBe(""); + expect(s.versions().length).toBe(0); + }); + + it("store a record", async () => { + s.set({ name: "cocalc", value: 10 }); + expect(s.to_str()).toBe('{"name":"cocalc","value":10}'); + const t = s.get_one({ name: "cocalc" }).toJS(); + expect(t).toEqual({ name: "cocalc", value: 10 }); + await s.commit(); + await s.save(); + // [ ] TODO: this save to disk definitely should NOT be needed + await s.save_to_disk(); + }); + + let client2, s2; + it("connect another client", async () => { + client2 = connect(); + // [ ] loading this resets the state if we do not save above. + s2 = client2.sync.db({ + project_id, + path: "new.syncdb", + service: server.service, + primary_keys: ["name"], + }); + await once(s2, "ready"); + expect(s2).not.toBe(s); + expect(s2.to_str()).toBe('{"name":"cocalc","value":10}'); + const t = s2.get_one({ name: "cocalc" }).toJS(); + expect(t).toEqual({ name: "cocalc", value: 10 }); + + s2.set({ name: "conat", date: new Date() }); + s2.commit(); + await s2.save(); + }); + + it("verifies the change on s2 is seen by s (and also that Date objects do NOT work)", async () => { + await wait({ until: () => s.get_one({ name: "conat" }) != null }); + const t = s.get_one({ name: "conat" }).toJS(); + expect(t).toEqual({ name: "conat", date: t.date }); + // They don't work because we're storing syncdb's in jsonl format, + // so json is used. We should have a new format called + // msgpackl and start using that. + expect(t.date instanceof Date).toBe(false); + }); + + const count = 1000; + it(`store ${count} records`, async () => { + const before = s.get().size; + for (let i = 0; i < count; i++) { + s.set({ name: i }); + } + s.commit(); + await s.save(); + expect(s.get().size).toBe(count + before); + }); + + it("confirm file saves to disk with many lines", async () => { + await s.save_to_disk(); + const v = (await s.fs.readFile("new.syncdb", "utf8")).split("\n"); + expect(v.length).toBe(s.get().size); + }); + + it("verifies lookups are not too slow (there is an index)", () => { + for (let i = 0; i < count; i++) { + expect(s.get_one({ name: i }).get("name")).toEqual(i); + } + }); +}); diff --git a/src/packages/backend/conat/test/sync-doc/syncstring.test.ts b/src/packages/backend/conat/test/sync-doc/syncstring.test.ts index d1d73332a7..e67361ddfb 100644 --- a/src/packages/backend/conat/test/sync-doc/syncstring.test.ts +++ b/src/packages/backend/conat/test/sync-doc/syncstring.test.ts @@ -8,7 +8,7 @@ describe("loading/saving syncstring to disk and setting values", () => { const project_id = uuid(); let client; - it("creates the fs client", () => { + it("creates the client", () => { client = connect(); }); diff --git a/src/packages/conat/core/client.ts b/src/packages/conat/core/client.ts index 9a3d8d3dc4..bfa92256f6 100644 --- a/src/packages/conat/core/client.ts +++ b/src/packages/conat/core/client.ts @@ -249,6 +249,11 @@ import { type SyncString, type SyncStringOptions, } from "@cocalc/conat/sync-doc/syncstring"; +import { + syncdb, + type SyncDB, + type SyncDBOptions, +} from "@cocalc/conat/sync-doc/syncdb"; import { fsClient, DEFAULT_FILE_SERVICE } from "@cocalc/conat/files/fs"; import TTL from "@isaacs/ttlcache"; import { @@ -1496,6 +1501,8 @@ export class Client extends EventEmitter { await createSyncTable({ ...opts, client: this }), string: (opts: Omit): SyncString => syncstring({ ...opts, client: this }), + db: (opts: Omit): SyncDB => + syncdb({ ...opts, client: this }), }; socket = { diff --git a/src/packages/conat/sync-doc/syncdb.ts b/src/packages/conat/sync-doc/syncdb.ts new file mode 100644 index 0000000000..a4437d7459 --- /dev/null +++ b/src/packages/conat/sync-doc/syncdb.ts @@ -0,0 +1,17 @@ +import { SyncClient } from "./sync-client"; +import { SyncDB, type SyncDBOpts0 } from "@cocalc/sync/editor/db"; +import { type Client as ConatClient } from "@cocalc/conat/core/client"; + +export interface SyncDBOptions extends Omit { + client: ConatClient; + // name of the file service that hosts this file: + service?: string; +} + +export type { SyncDB }; + +export function syncdb({ client, service, ...opts }: SyncDBOptions): SyncDB { + const fs = client.fs({ service, project_id: opts.project_id }); + const syncClient = new SyncClient(client); + return new SyncDB({ ...opts, fs, client: syncClient }); +} diff --git a/src/packages/conat/sync-doc/syncstring.ts b/src/packages/conat/sync-doc/syncstring.ts index 474acd8488..5fb6d2e2fb 100644 --- a/src/packages/conat/sync-doc/syncstring.ts +++ b/src/packages/conat/sync-doc/syncstring.ts @@ -1,10 +1,11 @@ import { SyncClient } from "./sync-client"; -import { SyncString } from "@cocalc/sync/editor/string/sync"; +import { + SyncString, + type SyncStringOpts, +} from "@cocalc/sync/editor/string/sync"; import { type Client as ConatClient } from "@cocalc/conat/core/client"; -export interface SyncStringOptions { - project_id: string; - path: string; +export interface SyncStringOptions extends Omit { client: ConatClient; // name of the file server that hosts this document: service?: string; @@ -12,18 +13,9 @@ export interface SyncStringOptions { export type { SyncString }; -export function syncstring({ - project_id, - path, - client, - service, -}: SyncStringOptions): SyncString { - const fs = client.fs({ service, project_id }); +export function syncstring({ client, service, ...opts }: SyncStringOptions): SyncString { + const fs = client.fs({ service, project_id: opts.project_id }); const syncClient = new SyncClient(client); - return new SyncString({ - project_id, - path, - client: syncClient, - fs, - }); + return new SyncString({ ...opts, fs, client: syncClient }); } + diff --git a/src/packages/frontend/frame-editors/generic/client.ts b/src/packages/frontend/frame-editors/generic/client.ts index db1c708a39..67d143f53a 100644 --- a/src/packages/frontend/frame-editors/generic/client.ts +++ b/src/packages/frontend/frame-editors/generic/client.ts @@ -200,7 +200,8 @@ export function syncstring(opts: SyncstringOpts): any { delete opts.fake; } opts1.id = schema.client_db.sha1(opts.project_id, opts.path); - return webapp_client.sync_string(opts1); + return webapp_client.conat_client.conat().sync.string(opts1); + // return webapp_client.sync_string(opts1); } import { DataServer } from "@cocalc/sync/editor/generic/sync-doc"; @@ -218,10 +219,10 @@ interface SyncstringOpts2 { } export function syncstring2(opts: SyncstringOpts2): SyncString { - // return webapp_client.conat_client.conat().sync.string(opts); - const opts1: any = opts; - opts1.client = webapp_client; - return webapp_client.sync_client.sync_string(opts1); + return webapp_client.conat_client.conat().sync.string(opts); + // const opts1: any = opts; + // opts1.client = webapp_client; + // return webapp_client.sync_client.sync_string(opts1); } export interface SyncDBOpts { @@ -239,8 +240,10 @@ export interface SyncDBOpts { } export function syncdb(opts: SyncDBOpts): any { - const opts1: any = opts; - return webapp_client.sync_db(opts1); + return webapp_client.conat_client.conat().sync.db(opts); + + // const opts1: any = opts; + // return webapp_client.sync_db(opts1); } import type { SyncDB } from "@cocalc/sync/editor/db/sync"; @@ -251,7 +254,8 @@ export function syncdb2(opts: SyncDBOpts): SyncDB { } const opts1: any = opts; opts1.client = webapp_client; - return webapp_client.sync_client.sync_db(opts1); + return webapp_client.conat_client.conat().sync.db(opts1); + // return webapp_client.sync_client.sync_db(opts1); } interface QueryOpts { diff --git a/src/packages/sync/editor/db/sync.ts b/src/packages/sync/editor/db/sync.ts index db970235b2..b8db32366c 100644 --- a/src/packages/sync/editor/db/sync.ts +++ b/src/packages/sync/editor/db/sync.ts @@ -9,7 +9,7 @@ import { Document, DocType } from "../generic/types"; export interface SyncDBOpts0 extends SyncOpts0 { primary_keys: string[]; - string_cols: string[]; + string_cols?: string[]; } export interface SyncDBOpts extends SyncDBOpts0 { @@ -25,13 +25,13 @@ export class SyncDB extends SyncDoc { throw Error("primary_keys must have length at least 1"); } opts1.from_str = (str) => - from_str(str, opts1.primary_keys, opts1.string_cols); + from_str(str, opts1.primary_keys, opts1.string_cols ?? []); opts1.doctype = { type: "db", patch_format: 1, opts: { primary_keys: opts1.primary_keys, - string_cols: opts1.string_cols, + string_cols: opts1.string_cols ?? [], }, }; super(opts1 as SyncOpts); diff --git a/src/packages/sync/editor/string/index.ts b/src/packages/sync/editor/string/index.ts index c1fed4e3a5..5a43e5f755 100644 --- a/src/packages/sync/editor/string/index.ts +++ b/src/packages/sync/editor/string/index.ts @@ -1 +1,2 @@ export { SyncString } from "./sync"; +export { type SyncStringOpts } from "./sync"; diff --git a/src/packages/sync/editor/string/sync.ts b/src/packages/sync/editor/string/sync.ts index 780c066370..95494d9f1e 100644 --- a/src/packages/sync/editor/string/sync.ts +++ b/src/packages/sync/editor/string/sync.ts @@ -6,6 +6,8 @@ import { SyncDoc, SyncOpts0, SyncOpts } from "../generic/sync-doc"; import { StringDocument } from "./doc"; +export type SyncStringOpts = SyncOpts0; + export class SyncString extends SyncDoc { constructor(opts: SyncOpts0) { // TS question -- What is the right way to do this? From 592915fd136e80f324e02c3f36f7c0cdfb92f796 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 20:27:05 -0700 Subject: [PATCH 035/798] fix some depcheck issues --- src/packages/backend/package.json | 13 ++++++++++--- src/packages/conat/tsconfig.json | 2 +- src/packages/file-server/package.json | 13 ++++++++++--- src/packages/pnpm-lock.yaml | 6 ------ 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index 1ceb59e9c6..4a71ea0a22 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -13,7 +13,10 @@ "./auth/*": "./dist/auth/*.js", "./auth/tokens/*": "./dist/auth/tokens/*.js" }, - "keywords": ["utilities", "cocalc"], + "keywords": [ + "utilities", + "cocalc" + ], "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf dist node_modules", @@ -31,13 +34,17 @@ "conat-persist": "DEBUG=cocalc:* node ./bin/conat-persist.cjs", "conat-test-server": "node ./bin/conat-test-server.cjs" }, - "files": ["dist/**", "bin/**", "README.md", "package.json"], + "files": [ + "dist/**", + "bin/**", + "README.md", + "package.json" + ], "author": "SageMath, Inc.", "license": "SEE LICENSE.md", "dependencies": { "@cocalc/backend": "workspace:*", "@cocalc/conat": "workspace:*", - "@cocalc/sync": "workspace:*", "@cocalc/util": "workspace:*", "@types/debug": "^4.1.12", "@types/jest": "^29.5.14", diff --git a/src/packages/conat/tsconfig.json b/src/packages/conat/tsconfig.json index 687201523d..5a8dd8655a 100644 --- a/src/packages/conat/tsconfig.json +++ b/src/packages/conat/tsconfig.json @@ -6,5 +6,5 @@ }, "exclude": ["node_modules", "dist", "test"], "references_comment": "Do not define path:../comm because that causes a circular references.", - "references": [{ "path": "../util" }] + "references": [{ "path": "../util", "path": "../sync" }] } diff --git a/src/packages/file-server/package.json b/src/packages/file-server/package.json index e12191d181..1a952b2369 100644 --- a/src/packages/file-server/package.json +++ b/src/packages/file-server/package.json @@ -17,13 +17,20 @@ "depcheck": "pnpx depcheck", "clean": "rm -rf node_modules dist" }, - "files": ["dist/**", "README.md", "package.json"], + "files": [ + "dist/**", + "README.md", + "package.json" + ], "author": "SageMath, Inc.", - "keywords": ["utilities", "btrfs", "cocalc"], + "keywords": [ + "utilities", + "btrfs", + "cocalc" + ], "license": "SEE LICENSE.md", "dependencies": { "@cocalc/backend": "workspace:*", - "@cocalc/conat": "workspace:*", "@cocalc/file-server": "workspace:*", "@cocalc/util": "workspace:*", "awaiting": "^3.0.0" diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 135d788520..74e8838273 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -81,9 +81,6 @@ importers: '@cocalc/conat': specifier: workspace:* version: link:../conat - '@cocalc/sync': - specifier: workspace:* - version: link:../sync '@cocalc/util': specifier: workspace:* version: link:../util @@ -292,9 +289,6 @@ importers: '@cocalc/backend': specifier: workspace:* version: link:../backend - '@cocalc/conat': - specifier: workspace:* - version: link:../conat '@cocalc/file-server': specifier: workspace:* version: 'link:' From f1885802e234a459ef44fb75cf546073219f7d5b Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 20 Jul 2025 03:30:30 +0000 Subject: [PATCH 036/798] fix circular ref --- src/packages/sync/tsconfig.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/packages/sync/tsconfig.json b/src/packages/sync/tsconfig.json index 0bdfd8b3a4..6cdb913e19 100644 --- a/src/packages/sync/tsconfig.json +++ b/src/packages/sync/tsconfig.json @@ -5,5 +5,5 @@ "outDir": "dist" }, "exclude": ["node_modules", "dist", "test"], - "references": [{ "path": "../util" }, { "path": "../conat" }] + "references": [{ "path": "../util" }] } From 24766743b9f8bed0436facabd110895e7a7dde5a Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 20 Jul 2025 03:33:43 +0000 Subject: [PATCH 037/798] fix a test now that I made EventIterator work much better --- src/packages/file-server/btrfs/test/subvolume.test.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index ecc5ed9756..456282d8b3 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -151,11 +151,10 @@ describe("the filesystem operations", () => { expect(done).toBe(false); expect(value).toEqual({ eventType: "change", filename: "w.txt" }); ac.abort(); - - expect(async () => { - // @ts-ignore - await watcher.next(); - }).rejects.toThrow("aborted"); + { + const { done } = await watcher.next(); + expect(done).toBe(true); + } }); it("rename a file", async () => { From 3eaf09128a42cb79b18c8bd39338b73b45c63e6d Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 20 Jul 2025 15:30:35 +0000 Subject: [PATCH 038/798] writing some sync-doc tests --- .../backend/conat/test/sync-doc/merge.test.ts | 95 +++++++++++++++++++ .../test/sync-doc/syncstring-bench.test.ts | 32 +++++++ 2 files changed, 127 insertions(+) create mode 100644 src/packages/backend/conat/test/sync-doc/merge.test.ts create mode 100644 src/packages/backend/conat/test/sync-doc/syncstring-bench.test.ts diff --git a/src/packages/backend/conat/test/sync-doc/merge.test.ts b/src/packages/backend/conat/test/sync-doc/merge.test.ts new file mode 100644 index 0000000000..bea90510dd --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/merge.test.ts @@ -0,0 +1,95 @@ +/* +Illustrate and test behavior when there is conflict. +*/ + +import { + before, + after, + uuid, + wait, + connect, + server, + once, + delay, +} from "./setup"; + +beforeAll(before); +afterAll(after); + +describe("synchronized editing with branching and merging", () => { + const project_id = uuid(); + let s1, s2, client1, client2; + + it("creates two clients", async () => { + client1 = connect(); + client2 = connect(); + s1 = client1.sync.string({ + project_id, + path: "a.txt", + service: server.service, + }); + await once(s1, "ready"); + + s2 = client2.sync.string({ + project_id, + path: "a.txt", + service: server.service, + }); + await once(s2, "ready"); + expect(s1.to_str()).toBe(""); + expect(s2.to_str()).toBe(""); + expect(s1 === s2).toBe(false); + }); + + it("both clients set the first version independently and inconsistently", async () => { + s1.from_str("x"); + s2.from_str("y"); + s1.commit(); + s2.commit(); + await s1.save(); + await s2.save(); + }); + + it("wait until both clients see two heads", async () => { + let heads1, heads2; + await wait({ + until: () => { + heads1 = s1.patch_list.getHeads(); + heads2 = s2.patch_list.getHeads(); + return heads1.length == 2 && heads2.length == 2; + }, + }); + expect(heads1.length).toBe(2); + expect(heads2.length).toBe(2); + expect(heads1).toEqual(heads2); + }); + + it("get the current value, which is a merge", () => { + const v1 = s1.to_str(); + const v2 = s2.to_str(); + expect(v1).toEqual("xy"); + expect(v2).toEqual("xy"); + }); + + // this is broken already: + it.skip("commit current value and see that there is a new single head that both share, thus resolving the merge in this way", async () => { + s1.commit(); + await s1.save(); + s1.show_history(); + let heads1, heads2; + await wait({ + until: () => { + heads1 = s1.patch_list.getHeads(); + heads2 = s2.patch_list.getHeads(); + console.log({ heads1, heads2 }); + return heads1.length == 1 && heads2.length == 1; + }, + }); + expect(heads1.length).toBe(1); + expect(heads2.length).toBe(1); + expect(heads1).toEqual(heads2); + }); + + // set values inconsistently again and explicitly resolve the merge conflict + // in a way that is different than the default. +}); diff --git a/src/packages/backend/conat/test/sync-doc/syncstring-bench.test.ts b/src/packages/backend/conat/test/sync-doc/syncstring-bench.test.ts new file mode 100644 index 0000000000..5bbf634377 --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/syncstring-bench.test.ts @@ -0,0 +1,32 @@ +import { before, after, uuid, client, server, once } from "./setup"; + +beforeAll(before); +afterAll(after); + +const log = console.log; + +describe("loading/saving syncstring to disk and setting values", () => { + let s; + const project_id = uuid(); + let fs; + it("time opening a syncstring for editing a file that already exists on disk", async () => { + fs = client.fs({ project_id, service: server.service }); + await fs.writeFile("a.txt", "hello"); + + const t0 = Date.now(); + await fs.readFile("a.txt", "utf8"); + console.log("lower bound: time to read file", Date.now() - t0, "ms"); + + const start = Date.now(); + s = client.sync.string({ + project_id, + path: "a.txt", + service: server.service, + }); + await once(s, "ready"); + const total = Date.now() - start; + log("time to open", total); + + expect(s.to_str()).toBe("hello"); + }); +}); From ad13e3b23e0f632aecc8a9c6e32f926434f2aa0f Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 20 Jul 2025 17:20:35 +0000 Subject: [PATCH 039/798] write some merge conflict related tests - these need noAutosave support for syncdocs, which doesn't work properly yet --- .../conat/test/sync-doc/conflict.test.ts | 202 ++++++++++++++++++ .../backend/conat/test/sync-doc/merge.test.ts | 95 -------- .../backend/conat/test/sync-doc/setup.ts | 18 ++ .../test/sync-doc/syncstring-bench.test.ts | 6 +- src/packages/conat/sync/dstream.ts | 1 + src/packages/conat/sync/synctable-kv.ts | 6 + src/packages/conat/sync/synctable-stream.ts | 5 + src/packages/conat/sync/synctable.ts | 1 + .../sync/editor/generic/sorted-patch-list.ts | 10 +- src/packages/sync/editor/generic/sync-doc.ts | 45 +++- 10 files changed, 278 insertions(+), 111 deletions(-) create mode 100644 src/packages/backend/conat/test/sync-doc/conflict.test.ts delete mode 100644 src/packages/backend/conat/test/sync-doc/merge.test.ts diff --git a/src/packages/backend/conat/test/sync-doc/conflict.test.ts b/src/packages/backend/conat/test/sync-doc/conflict.test.ts new file mode 100644 index 0000000000..4c5c521d02 --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/conflict.test.ts @@ -0,0 +1,202 @@ +/* +Illustrate and test behavior when there is conflict. + +TODO: we must get noAutosave to fully work so we can make +the tests of conflicts, etc., better. + +E.g, the test below WILL RANDOMLY FAIL right now due to autosave randomness... +*/ + +import { + before, + after, + uuid, + connect, + server, + once, + delay, + waitUntilSynced, +} from "./setup"; + +beforeAll(before); +afterAll(after); + +describe("synchronized editing with branching and merging", () => { + const project_id = uuid(); + let s1, s2, client1, client2; + + it("creates two clients", async () => { + client1 = connect(); + client2 = connect(); + s1 = client1.sync.string({ + project_id, + path: "a.txt", + service: server.service, + }); + await once(s1, "ready"); + + s2 = client2.sync.string({ + project_id, + path: "a.txt", + service: server.service, + }); + await once(s2, "ready"); + expect(s1.to_str()).toBe(""); + expect(s2.to_str()).toBe(""); + expect(s1 === s2).toBe(false); + }); + + it("both clients set the first version independently and inconsistently", async () => { + s1.from_str("x"); + s2.from_str("y"); + s1.commit(); + // delay so s2's time is always bigger than s1's so our unit test + // is well defined + await delay(1); + s2.commit(); + await s1.save(); + await s2.save(); + }); + + it("wait until both clients see two heads", async () => { + await waitUntilSynced([s1, s2]); + const heads1 = s1.patch_list.getHeads(); + const heads2 = s2.patch_list.getHeads(); + expect(heads1.length).toBe(2); + expect(heads2.length).toBe(2); + expect(heads1).toEqual(heads2); + }); + + it("get the current value, which is a merge", () => { + const v1 = s1.to_str(); + const v2 = s2.to_str(); + expect(v1).toEqual("xy"); + expect(v2).toEqual("xy"); + }); + + it("commit current value and see that there is a new single head that both share, thus resolving the merge in this way", async () => { + s1.commit(); + await s1.save(); + await waitUntilSynced([s1, s2]); + const heads1 = s1.patch_list.getHeads(); + const heads2 = s2.patch_list.getHeads(); + expect(heads1.length).toBe(1); + expect(heads2.length).toBe(1); + expect(heads1).toEqual(heads2); + }); + + it("set values inconsistently again and explicitly resolve the merge conflict in a way that is different than the default", async () => { + s1.from_str("xy1"); + s1.commit(); + await delay(1); + s2.from_str("xy2"); + s2.commit(); + await s1.save(); + await s2.save(); + + await waitUntilSynced([s1, s2]); + expect(s1.to_str()).toEqual("xy12"); + expect(s2.to_str()).toEqual("xy12"); + + // how we resolve the conflict + s1.from_str("xy3"); + s1.commit(); + await waitUntilSynced([s1, s2]); + + // everybody has this state now + expect(s1.to_str()).toEqual("xy3"); + expect(s2.to_str()).toEqual("xy3"); + }); +}); + +describe.only("do the example in the blog post 'Lies I was Told About Collaborative Editing, Part 1: Algorithms for offline editing' -- https://www.moment.dev/blog/lies-i-was-told-pt-1", () => { + const project_id = uuid(); + let client1, client2; + + async function getInitialState(path: string) { + client1 ??= connect(); + client2 ??= connect(); + client1 + .fs({ project_id, service: server.service }) + .writeFile(path, "The Color of Pomegranates"); + const alice = client1.sync.string({ + project_id, + path, + service: server.service, + }); + await once(alice, "ready"); + + const bob = client2.sync.string({ + project_id, + path, + service: server.service, + }); + await once(bob, "ready"); + return { alice, bob }; + } + + let alice, bob; + it("creates two clients", async () => { + ({ alice, bob } = await getInitialState("first.txt")); + expect(alice.to_str()).toEqual("The Color of Pomegranates"); + expect(bob.to_str()).toEqual("The Color of Pomegranates"); + }); + + it("Bob changes the spelling of Color to the British Colour and unaware Alice deletes all of the text.", async () => { + bob.from_str("The Colour of Pomegranates"); + bob.commit(); + alice.from_str(""); + alice.commit(); + }); + + it("Both come back online -- the resolution is the empty (with either order above) string because the **best effort** application of inserting the u (with context) to either is a no-op.", async () => { + await bob.save(); + await alice.save(); + await waitUntilSynced([bob, alice]); + expect(alice.to_str()).toEqual(""); + expect(bob.to_str()).toEqual(""); + }); + + it("the important thing about the cocalc approach is that a consistent history is saved, so everybody knows precisely what happened. **I.e., the fact that at one point Bob adding a British u is not lost to either party!**", () => { + const v = alice.versions(); + const x = v.map((t) => alice.version(t).to_str()); + expect(new Set(x)).toEqual( + new Set(["The Color of Pomegranates", "The Colour of Pomegranates", ""]), + ); + + const w = alice.versions(); + const y = w.map((t) => bob.version(t).to_str()); + expect(y).toEqual(x); + }); + + it("reset -- create alicea and bob again", async () => { + ({ alice, bob } = await getInitialState("second.txt")); + }); + + // opposite order this time + it("Bob changes the spelling of Color to the British Colour and unaware Alice deletes all of the text.", async () => { + alice.from_str(""); + alice.commit(); + bob.from_str("The Colour of Pomegranates"); + bob.commit(); + }); + + it("both empty again", async () => { + await bob.save(); + await alice.save(); + await waitUntilSynced([bob, alice]); + expect(alice.to_str()).toEqual(""); + expect(bob.to_str()).toEqual(""); + }); + + it("There are in fact two heads right now, and either party can resolve the merge conflict however they want.", async () => { + expect(alice.patch_list.getHeads().length).toBe(2); + expect(bob.patch_list.getHeads().length).toBe(2); + bob.from_str("The Colour of Pomegranates"); + bob.commit(); + + await waitUntilSynced([bob, alice]); + expect(alice.to_str()).toEqual("The Colour of Pomegranates"); + expect(bob.to_str()).toEqual("The Colour of Pomegranates"); + }); +}); diff --git a/src/packages/backend/conat/test/sync-doc/merge.test.ts b/src/packages/backend/conat/test/sync-doc/merge.test.ts deleted file mode 100644 index bea90510dd..0000000000 --- a/src/packages/backend/conat/test/sync-doc/merge.test.ts +++ /dev/null @@ -1,95 +0,0 @@ -/* -Illustrate and test behavior when there is conflict. -*/ - -import { - before, - after, - uuid, - wait, - connect, - server, - once, - delay, -} from "./setup"; - -beforeAll(before); -afterAll(after); - -describe("synchronized editing with branching and merging", () => { - const project_id = uuid(); - let s1, s2, client1, client2; - - it("creates two clients", async () => { - client1 = connect(); - client2 = connect(); - s1 = client1.sync.string({ - project_id, - path: "a.txt", - service: server.service, - }); - await once(s1, "ready"); - - s2 = client2.sync.string({ - project_id, - path: "a.txt", - service: server.service, - }); - await once(s2, "ready"); - expect(s1.to_str()).toBe(""); - expect(s2.to_str()).toBe(""); - expect(s1 === s2).toBe(false); - }); - - it("both clients set the first version independently and inconsistently", async () => { - s1.from_str("x"); - s2.from_str("y"); - s1.commit(); - s2.commit(); - await s1.save(); - await s2.save(); - }); - - it("wait until both clients see two heads", async () => { - let heads1, heads2; - await wait({ - until: () => { - heads1 = s1.patch_list.getHeads(); - heads2 = s2.patch_list.getHeads(); - return heads1.length == 2 && heads2.length == 2; - }, - }); - expect(heads1.length).toBe(2); - expect(heads2.length).toBe(2); - expect(heads1).toEqual(heads2); - }); - - it("get the current value, which is a merge", () => { - const v1 = s1.to_str(); - const v2 = s2.to_str(); - expect(v1).toEqual("xy"); - expect(v2).toEqual("xy"); - }); - - // this is broken already: - it.skip("commit current value and see that there is a new single head that both share, thus resolving the merge in this way", async () => { - s1.commit(); - await s1.save(); - s1.show_history(); - let heads1, heads2; - await wait({ - until: () => { - heads1 = s1.patch_list.getHeads(); - heads2 = s2.patch_list.getHeads(); - console.log({ heads1, heads2 }); - return heads1.length == 1 && heads2.length == 1; - }, - }); - expect(heads1.length).toBe(1); - expect(heads2.length).toBe(1); - expect(heads1).toEqual(heads2); - }); - - // set values inconsistently again and explicitly resolve the merge conflict - // in a way that is different than the default. -}); diff --git a/src/packages/backend/conat/test/sync-doc/setup.ts b/src/packages/backend/conat/test/sync-doc/setup.ts index 5af527bf53..b6ce38042d 100644 --- a/src/packages/backend/conat/test/sync-doc/setup.ts +++ b/src/packages/backend/conat/test/sync-doc/setup.ts @@ -2,6 +2,7 @@ import { before as before0, after as after0, client as client0, + wait, } from "@cocalc/backend/conat/test/setup"; export { connect, wait, once, delay } from "@cocalc/backend/conat/test/setup"; import { @@ -23,3 +24,20 @@ export async function after() { await cleanupFileservers(); await after0(); } + +// wait until the state of several syncdocs all have same heads- they may have multiple +// heads, but they all have the same heads +export async function waitUntilSynced(syncdocs: any[]) { + await wait({ + until: () => { + const X = new Set(); + for (const s of syncdocs) { + X.add(JSON.stringify(s.patch_list.getHeads()?.sort())); + if (X.size > 1) { + return false; + } + } + return true; + }, + }); +} diff --git a/src/packages/backend/conat/test/sync-doc/syncstring-bench.test.ts b/src/packages/backend/conat/test/sync-doc/syncstring-bench.test.ts index 5bbf634377..2e5a64ef2a 100644 --- a/src/packages/backend/conat/test/sync-doc/syncstring-bench.test.ts +++ b/src/packages/backend/conat/test/sync-doc/syncstring-bench.test.ts @@ -3,7 +3,7 @@ import { before, after, uuid, client, server, once } from "./setup"; beforeAll(before); afterAll(after); -const log = console.log; +const log = process.env.BENCH ? console.log : (..._args) => {}; describe("loading/saving syncstring to disk and setting values", () => { let s; @@ -15,7 +15,7 @@ describe("loading/saving syncstring to disk and setting values", () => { const t0 = Date.now(); await fs.readFile("a.txt", "utf8"); - console.log("lower bound: time to read file", Date.now() - t0, "ms"); + log("lower bound: time to read file", Date.now() - t0, "ms"); const start = Date.now(); s = client.sync.string({ @@ -25,7 +25,7 @@ describe("loading/saving syncstring to disk and setting values", () => { }); await once(s, "ready"); const total = Date.now() - start; - log("time to open", total); + log("actual time to open sync document", total); expect(s.to_str()).toBe("hello"); }); diff --git a/src/packages/conat/sync/dstream.ts b/src/packages/conat/sync/dstream.ts index 4bc6121c26..26d0c997f9 100644 --- a/src/packages/conat/sync/dstream.ts +++ b/src/packages/conat/sync/dstream.ts @@ -290,6 +290,7 @@ export class DStream extends EventEmitter { }; save = reuseInFlight(async () => { + //console.log("save", this.noAutosave); await until( async () => { if (this.isClosed()) { diff --git a/src/packages/conat/sync/synctable-kv.ts b/src/packages/conat/sync/synctable-kv.ts index 7950305ed4..9e56393fff 100644 --- a/src/packages/conat/sync/synctable-kv.ts +++ b/src/packages/conat/sync/synctable-kv.ts @@ -33,6 +33,7 @@ export class SyncTableKV extends EventEmitter { private config?: Partial; private desc?: JSONValue; private ephemeral?: boolean; + private noAutosave?: boolean; constructor({ query, @@ -44,6 +45,7 @@ export class SyncTableKV extends EventEmitter { config, desc, ephemeral, + noAutosave, }: { query; client: Client; @@ -54,6 +56,7 @@ export class SyncTableKV extends EventEmitter { config?: Partial; desc?: JSONValue; ephemeral?: boolean; + noAutosave?: boolean; }) { super(); this.setMaxListeners(1000); @@ -64,6 +67,7 @@ export class SyncTableKV extends EventEmitter { this.client = client; this.desc = desc; this.ephemeral = ephemeral; + this.noAutosave = noAutosave; this.table = keys(query)[0]; if (query[this.table][0].string_id && query[this.table][0].project_id) { this.project_id = query[this.table][0].project_id; @@ -126,6 +130,7 @@ export class SyncTableKV extends EventEmitter { config: this.config, desc: this.desc, ephemeral: this.ephemeral, + noAutosave: this.noAutosave, }); } else { this.dkv = await createDko({ @@ -136,6 +141,7 @@ export class SyncTableKV extends EventEmitter { config: this.config, desc: this.desc, ephemeral: this.ephemeral, + noAutosave: this.noAutosave, }); } // For some reason this one line confuses typescript and break building the compute server package (nothing else similar happens). diff --git a/src/packages/conat/sync/synctable-stream.ts b/src/packages/conat/sync/synctable-stream.ts index 36b1b3e27e..597cfc8d95 100644 --- a/src/packages/conat/sync/synctable-stream.ts +++ b/src/packages/conat/sync/synctable-stream.ts @@ -45,6 +45,7 @@ export class SyncTableStream extends EventEmitter { private config?: Partial; private start_seq?: number; private noInventory?: boolean; + private noAutosave?: boolean; private ephemeral?: boolean; constructor({ @@ -57,6 +58,7 @@ export class SyncTableStream extends EventEmitter { start_seq, noInventory, ephemeral, + noAutosave, }: { query; client: Client; @@ -67,10 +69,12 @@ export class SyncTableStream extends EventEmitter { start_seq?: number; noInventory?: boolean; ephemeral?: boolean; + noAutosave?: boolean; }) { super(); this.client = client; this.noInventory = noInventory; + this.noAutosave = noAutosave; this.ephemeral = ephemeral; this.setMaxListeners(1000); this.getHook = immutable ? fromJS : (x) => x; @@ -107,6 +111,7 @@ export class SyncTableStream extends EventEmitter { start_seq: this.start_seq, noInventory: this.noInventory, ephemeral: this.ephemeral, + noAutosave: this.noAutosave, }); this.dstream.on("change", (mesg) => { this.handle(mesg, true); diff --git a/src/packages/conat/sync/synctable.ts b/src/packages/conat/sync/synctable.ts index 8f69000eee..95048b86d2 100644 --- a/src/packages/conat/sync/synctable.ts +++ b/src/packages/conat/sync/synctable.ts @@ -43,6 +43,7 @@ export interface SyncTableOptions { start_seq?: number; noInventory?: boolean; ephemeral?: boolean; + noAutosave?: boolean; } export const createSyncTable = refCache({ diff --git a/src/packages/sync/editor/generic/sorted-patch-list.ts b/src/packages/sync/editor/generic/sorted-patch-list.ts index 5d35c08bec..6a2be95f95 100644 --- a/src/packages/sync/editor/generic/sorted-patch-list.ts +++ b/src/packages/sync/editor/generic/sorted-patch-list.ts @@ -102,7 +102,7 @@ export class SortedPatchList extends EventEmitter { }; /* Choose the next available time in ms that is congruent to - m modulo n and is larger than any current times. + m modulo n and is larger than any current times. This is a LOGICAL TIME; it does not have to equal the actual wall clock. The key is that it is increasing. The congruence condition is so that any time @@ -134,9 +134,13 @@ export class SortedPatchList extends EventEmitter { if (n <= 0) { n = 1; } - let a = m - (time % n); + // we add 50 to the modulus so that if a bunch of new users are joining at the exact same moment, + // they don't have to be instantly aware of each other for this to keep working. Basically, we + // give ourself a buffer of 10 + const modulus = n + 10; + let a = m - (time % modulus); if (a < 0) { - a += n; + a += modulus; } time += a; // now time = m (mod n) // There is also no possibility of a conflict with a known time diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 2fed66dd7a..33475c3920 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -160,6 +160,10 @@ export interface SyncOpts0 { // optional filesystem interface. fs?: SyncDocFilesystem; + + // if true, do not implicitly save on commit. This is very + // useful for unit testing to easily simulate offline state. + noAutosave?: boolean; } export interface SyncOpts extends SyncOpts0 { @@ -275,6 +279,8 @@ export class SyncDoc extends EventEmitter { private fs?: SyncDocFilesystem; + private noAutosave?: boolean; + constructor(opts: SyncOpts) { super(); if (opts.string_id === undefined) { @@ -297,6 +303,7 @@ export class SyncDoc extends EventEmitter { "data_server", "ephemeral", "fs", + "noAutosave", ]) { if (opts[field] != undefined) { this[field] = opts[field]; @@ -1289,6 +1296,7 @@ export class SyncDoc extends EventEmitter { desc: { path: this.path }, start_seq: this.last_seq, ephemeral, + noAutosave: this.noAutosave, }); if (this.last_seq) { @@ -1308,6 +1316,7 @@ export class SyncDoc extends EventEmitter { atomic: true, desc: { path: this.path }, ephemeral, + noAutosave: this.noAutosave, }); // also find the correct last_seq: @@ -1355,6 +1364,7 @@ export class SyncDoc extends EventEmitter { immutable: true, desc: { path: this.path }, ephemeral, + noAutosave: this.noAutosave, }); } else if (this.useConat && query.ipywidgets) { synctable = await this.client.synctable_conat(query, { @@ -1370,6 +1380,7 @@ export class SyncDoc extends EventEmitter { config: { max_age: 1000 * 60 * 60 * 24 }, desc: { path: this.path }, ephemeral: true, // ipywidgets state always ephemeral + noAutosave: this.noAutosave, }); } else if (this.useConat && (query.eval_inputs || query.eval_outputs)) { synctable = await this.client.synctable_conat(query, { @@ -1383,6 +1394,7 @@ export class SyncDoc extends EventEmitter { config: { max_age: 5 * 60 * 1000 }, desc: { path: this.path }, ephemeral: true, // eval state (for sagews) is always ephemeral + noAutosave: this.noAutosave, }); } else if (this.useConat) { synctable = await this.client.synctable_conat(query, { @@ -1395,6 +1407,7 @@ export class SyncDoc extends EventEmitter { immutable: true, desc: { path: this.path }, ephemeral, + noAutosave: this.noAutosave, }); } else { // only used for unit tests and the ephemeral messaging composer @@ -3181,10 +3194,15 @@ export class SyncDoc extends EventEmitter { // fine offline, and does not wait until anything // is saved to the network, etc. commit = (emitChangeImmediately = false): boolean => { - if (this.last == null || this.doc == null || this.last.is_equal(this.doc)) { + if ( + this.last == null || + this.doc == null || + (this.last.is_equal(this.doc) && + (this.patch_list?.getHeads().length ?? 0) <= 1) + ) { return false; } - // console.trace('commit'); + // console.trace("commit"); if (emitChangeImmediately) { // used for local clients. NOTE: don't do this without explicit @@ -3197,12 +3215,14 @@ export class SyncDoc extends EventEmitter { // Now save to backend as a new patch: this.emit("user-change"); - const patch = this.last.make_patch(this.doc); // must be nontrivial + const patch = this.last.make_patch(this.doc); this.last = this.doc; // ... and save that to patches table const time = this.next_patch_time(); this.commit_patch(time, patch); - this.save(); // so eventually also gets sent out. + if (!this.noAutosave) { + this.save(); // so eventually also gets sync'd out to other clients + } this.touchProject(); return true; }; @@ -3626,10 +3646,13 @@ export class SyncDoc extends EventEmitter { return; } - // Critical to save what we have now so it doesn't get overwritten during - // before-change or setting this.doc below. This caused - // https://github.com/sagemathinc/cocalc/issues/5871 - this.commit(); + if (!this.last.is_equal(this.doc)) { + // If live versions differs from last commit (or merge of heads), it is + // commit what we have now so it doesn't get overwritten during + // before-change or setting this.doc below. This caused + // https://github.com/sagemathinc/cocalc/issues/5871 + this.commit(); + } if (upstreamPatches && this.state == "ready") { // First save any unsaved changes from the live document, which this @@ -3637,10 +3660,12 @@ export class SyncDoc extends EventEmitter { // rapidly changing live editor with changes not yet saved here. this.emit("before-change"); // As a result of the emit in the previous line, all kinds of - // nontrivial listener code probably just ran, and it should + // nontrivial listener code may have just ran, and it could // have updated this.doc. We commit this.doc, so that the // upstream patches get applied against the correct live this.doc. - this.commit(); + if (!this.last.is_equal(this.doc)) { + this.commit(); + } } // Compute the global current state of the document, From 0ce82c72cc3ae9bf20c388a14272d4b67fc61a44 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 20 Jul 2025 20:22:00 +0000 Subject: [PATCH 040/798] sync-doc: tests using noAutosave --- .../conat/test/sync-doc/conflict.test.ts | 18 +++- .../conat/test/sync-doc/no-autosave.test.ts | 88 +++++++++++++++++++ .../conat/test/sync-doc/syncstring.test.ts | 2 +- src/packages/conat/sync-doc/sync-client.ts | 9 +- src/packages/sync/editor/generic/sync-doc.ts | 16 ++-- src/scripts/runoo | 32 +++++++ 6 files changed, 151 insertions(+), 14 deletions(-) create mode 100644 src/packages/backend/conat/test/sync-doc/no-autosave.test.ts create mode 100755 src/scripts/runoo diff --git a/src/packages/backend/conat/test/sync-doc/conflict.test.ts b/src/packages/backend/conat/test/sync-doc/conflict.test.ts index 4c5c521d02..0533878fe5 100644 --- a/src/packages/backend/conat/test/sync-doc/conflict.test.ts +++ b/src/packages/backend/conat/test/sync-doc/conflict.test.ts @@ -32,6 +32,7 @@ describe("synchronized editing with branching and merging", () => { project_id, path: "a.txt", service: server.service, + noAutosave: true, }); await once(s1, "ready"); @@ -39,6 +40,7 @@ describe("synchronized editing with branching and merging", () => { project_id, path: "a.txt", service: server.service, + noAutosave: true, }); await once(s2, "ready"); expect(s1.to_str()).toBe(""); @@ -52,7 +54,7 @@ describe("synchronized editing with branching and merging", () => { s1.commit(); // delay so s2's time is always bigger than s1's so our unit test // is well defined - await delay(1); + await delay(75); s2.commit(); await s1.save(); await s2.save(); @@ -88,7 +90,7 @@ describe("synchronized editing with branching and merging", () => { it("set values inconsistently again and explicitly resolve the merge conflict in a way that is different than the default", async () => { s1.from_str("xy1"); s1.commit(); - await delay(1); + await delay(75); s2.from_str("xy2"); s2.commit(); await s1.save(); @@ -101,6 +103,7 @@ describe("synchronized editing with branching and merging", () => { // how we resolve the conflict s1.from_str("xy3"); s1.commit(); + await s1.save(); await waitUntilSynced([s1, s2]); // everybody has this state now @@ -109,7 +112,7 @@ describe("synchronized editing with branching and merging", () => { }); }); -describe.only("do the example in the blog post 'Lies I was Told About Collaborative Editing, Part 1: Algorithms for offline editing' -- https://www.moment.dev/blog/lies-i-was-told-pt-1", () => { +describe("do the example in the blog post 'Lies I was Told About Collaborative Editing, Part 1: Algorithms for offline editing' -- https://www.moment.dev/blog/lies-i-was-told-pt-1", () => { const project_id = uuid(); let client1, client2; @@ -123,15 +126,21 @@ describe.only("do the example in the blog post 'Lies I was Told About Collaborat project_id, path, service: server.service, + noAutosave: true, }); await once(alice, "ready"); + await alice.save(); const bob = client2.sync.string({ project_id, path, service: server.service, + noAutosave: true, }); await once(bob, "ready"); + await bob.save(); + await waitUntilSynced([bob, alice]); + return { alice, bob }; } @@ -189,11 +198,12 @@ describe.only("do the example in the blog post 'Lies I was Told About Collaborat expect(bob.to_str()).toEqual(""); }); - it("There are in fact two heads right now, and either party can resolve the merge conflict however they want.", async () => { + it("There are two heads; either client can resolve the merge conflict.", async () => { expect(alice.patch_list.getHeads().length).toBe(2); expect(bob.patch_list.getHeads().length).toBe(2); bob.from_str("The Colour of Pomegranates"); bob.commit(); + await bob.save(); await waitUntilSynced([bob, alice]); expect(alice.to_str()).toEqual("The Colour of Pomegranates"); diff --git a/src/packages/backend/conat/test/sync-doc/no-autosave.test.ts b/src/packages/backend/conat/test/sync-doc/no-autosave.test.ts new file mode 100644 index 0000000000..032526f381 --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/no-autosave.test.ts @@ -0,0 +1,88 @@ +import { + before, + after, + uuid, + connect, + server, + once, + delay, + waitUntilSynced, +} from "./setup"; + +beforeAll(before); +afterAll(after); + +describe("confirm noAutosave works", () => { + const project_id = uuid(); + const path = "a.txt"; + let client1, client2, s1, s2; + + it("creates two clients with noAutosave enabled", async () => { + client1 = connect(); + client2 = connect(); + await client1 + .fs({ project_id, service: server.service }) + .writeFile(path, ""); + s1 = client1.sync.string({ + project_id, + path, + service: server.service, + noAutosave: true, + }); + + await once(s1, "ready"); + + s2 = client2.sync.string({ + project_id, + path, + service: server.service, + noAutosave: true, + }); + await once(s2, "ready"); + expect(s1.noAutosave).toEqual(true); + expect(s2.noAutosave).toEqual(true); + }); + + const howLong = 750; + it(`write a change to s1 and commit it, but observe s2 does not see it even after ${howLong}ms (which should be plenty of time)`, async () => { + s1.from_str("new-ver"); + s1.commit(); + + expect(s2.to_str()).toEqual(""); + await delay(howLong); + expect(s2.to_str()).toEqual(""); + }); + + it("explicitly save and see s2 does get the change", async () => { + await s1.save(); + await waitUntilSynced([s1, s2]); + expect(s2.to_str()).toEqual("new-ver"); + }); + + it("make a change resulting in two heads", async () => { + s2.from_str("new-ver-1"); + s2.commit(); + // no background saving happening: + await delay(100); + s1.from_str("new-ver-2"); + s1.commit(); + await Promise.all([s1.save(), s2.save()]); + }); + + it("there are two heads and value is merged", async () => { + await waitUntilSynced([s1, s2]); + expect(s1.to_str()).toEqual("new-ver-1-2"); + expect(s2.to_str()).toEqual("new-ver-1-2"); + expect(s1.patch_list.getHeads().length).toBe(2); + expect(s2.patch_list.getHeads().length).toBe(2); + }); + + it("string state info matches", async () => { + const a1 = s1.syncstring_table_get_one().toJS(); + const a2 = s2.syncstring_table_get_one().toJS(); + expect(a1).toEqual(a2); + expect(new Set(a1.users)).toEqual( + new Set([s1.client.client_id(), s2.client.client_id()]), + ); + }); +}); diff --git a/src/packages/backend/conat/test/sync-doc/syncstring.test.ts b/src/packages/backend/conat/test/sync-doc/syncstring.test.ts index e67361ddfb..3990baaa55 100644 --- a/src/packages/backend/conat/test/sync-doc/syncstring.test.ts +++ b/src/packages/backend/conat/test/sync-doc/syncstring.test.ts @@ -124,4 +124,4 @@ describe("synchronized editing with two copies of a syncstring", () => { s2.show_history({ log: (x) => v2.push(x) }); expect(v1).toEqual(v2); }); -}); +}); \ No newline at end of file diff --git a/src/packages/conat/sync-doc/sync-client.ts b/src/packages/conat/sync-doc/sync-client.ts index ebc6377f8e..9609dce5a3 100644 --- a/src/packages/conat/sync-doc/sync-client.ts +++ b/src/packages/conat/sync-doc/sync-client.ts @@ -56,8 +56,13 @@ export class SyncClient extends EventEmitter implements Client0 { return new PubSub({ client: this.client, ...opts }); }; - // account_id or project_id - client_id = (): string => this.client.id; + // account_id or project_id or hub_id or fallback client.id + client_id = (): string => { + const user = this.client.info?.user; + return ( + user?.account_id ?? user?.project_id ?? user?.hub_id ?? this.client.id + ); + }; server_time = (): Date => { return new Date(); diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 33475c3920..66f0939626 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -1626,7 +1626,7 @@ export class SyncDoc extends EventEmitter { // normally this only happens in a later event loop, // so force it now. dbg("handling patch update queue since", this.patch_list.count()); - await this.handle_patch_update_queue(); + await this.handle_patch_update_queue(true); assertDefined(this.patch_list); dbg("done handling, now ", this.patch_list.count()); if (this.patch_list.count() === 0) { @@ -1636,7 +1636,7 @@ export class SyncDoc extends EventEmitter { // This is the root cause of https://github.com/sagemathinc/cocalc/issues/2382 await once(this.patches_table, "change"); dbg("got patches_table change"); - await this.handle_patch_update_queue(); + await this.handle_patch_update_queue(true); dbg("handled update queue"); } } @@ -2269,7 +2269,7 @@ export class SyncDoc extends EventEmitter { // above async waits could have resulted in state change. return; } - await this.handle_patch_update_queue(); + await this.handle_patch_update_queue(true); if (this.state != "ready") { return; } @@ -3539,7 +3539,7 @@ export class SyncDoc extends EventEmitter { Whenever new patches are added to this.patches_table, their timestamp gets added to this.patch_update_queue. */ - private handle_patch_update_queue = async (): Promise => { + private handle_patch_update_queue = async (save = false): Promise => { const dbg = this.dbg("handle_patch_update_queue"); try { this.handle_patch_update_queue_running = true; @@ -3574,9 +3574,11 @@ export class SyncDoc extends EventEmitter { dbg("waiting for remote and doc to sync..."); this.sync_remote_and_doc(v.length > 0); - await this.patches_table.save(); - if (this.state === ("closed" as State)) return; // closed during await; nothing further to do - dbg("remote and doc now synced"); + if (save || !this.noAutosave) { + await this.patches_table.save(); + if (this.state === ("closed" as State)) return; // closed during await; nothing further to do + dbg("remote and doc now synced"); + } if (this.patch_update_queue.length > 0) { // It is very important that next loop happen in a later diff --git a/src/scripts/runoo b/src/scripts/runoo new file mode 100755 index 0000000000..eb8a32fe8b --- /dev/null +++ b/src/scripts/runoo @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 + +""" +This is just meant to be a quick and dirty python script so I can run other +scripts (e.g., a unit test runner) to get a sense if they are flaky or not.: + +runoo 20 python test.py # runs "python test.py" 20 times in parallel, ncpus at once + +""" + +import sys, os, time +from concurrent.futures import ProcessPoolExecutor, as_completed +import multiprocessing + +def run_cmd(cmd, i): + print('\n'*10) + print(f'Loop: {i+1} , Time: {round(time.time() - start)}, Command: {cmd}') + ret = os.system(cmd) + if ret: + raise RuntimeError(f'Command failed on run {i+1}') + return ret + +if __name__ == '__main__': + n = int(sys.argv[1]) + cmd = ' '.join(sys.argv[2:]) + k = multiprocessing.cpu_count() + start = time.time() + + with ProcessPoolExecutor(max_workers=k) as executor: + futures = [executor.submit(run_cmd, cmd, i) for i in range(n)] + for f in as_completed(futures): + f.result() # Raises exception if failed \ No newline at end of file From 69ea3a1f29ea4bc70fefea194b87b39a754caa1c Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 20 Jul 2025 20:37:15 +0000 Subject: [PATCH 041/798] improve runoo --- .../conat/test/sync-doc/conflict.test.ts | 10 ++++--- src/scripts/runoo | 26 ++++++++++++++----- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/src/packages/backend/conat/test/sync-doc/conflict.test.ts b/src/packages/backend/conat/test/sync-doc/conflict.test.ts index 0533878fe5..00b590f8fe 100644 --- a/src/packages/backend/conat/test/sync-doc/conflict.test.ts +++ b/src/packages/backend/conat/test/sync-doc/conflict.test.ts @@ -21,6 +21,8 @@ import { beforeAll(before); afterAll(after); +const GAP_DELAY = 50; + describe("synchronized editing with branching and merging", () => { const project_id = uuid(); let s1, s2, client1, client2; @@ -49,12 +51,12 @@ describe("synchronized editing with branching and merging", () => { }); it("both clients set the first version independently and inconsistently", async () => { - s1.from_str("x"); s2.from_str("y"); + s1.from_str("x"); s1.commit(); // delay so s2's time is always bigger than s1's so our unit test // is well defined - await delay(75); + await delay(GAP_DELAY); s2.commit(); await s1.save(); await s2.save(); @@ -90,7 +92,7 @@ describe("synchronized editing with branching and merging", () => { it("set values inconsistently again and explicitly resolve the merge conflict in a way that is different than the default", async () => { s1.from_str("xy1"); s1.commit(); - await delay(75); + await delay(GAP_DELAY); s2.from_str("xy2"); s2.commit(); await s1.save(); @@ -100,7 +102,7 @@ describe("synchronized editing with branching and merging", () => { expect(s1.to_str()).toEqual("xy12"); expect(s2.to_str()).toEqual("xy12"); - // how we resolve the conflict + // resolve the conflict in our own way s1.from_str("xy3"); s1.commit(); await s1.save(); diff --git a/src/scripts/runoo b/src/scripts/runoo index eb8a32fe8b..4445904c26 100755 --- a/src/scripts/runoo +++ b/src/scripts/runoo @@ -12,13 +12,23 @@ import sys, os, time from concurrent.futures import ProcessPoolExecutor, as_completed import multiprocessing +failed = False def run_cmd(cmd, i): - print('\n'*10) - print(f'Loop: {i+1} , Time: {round(time.time() - start)}, Command: {cmd}') - ret = os.system(cmd) - if ret: - raise RuntimeError(f'Command failed on run {i+1}') - return ret + global failed + if failed: + return + print('\n'*5) + print('*'*60) + print(f'* Loop: {i+1}/{n} , Time: {round(time.time() - start)}, Command: {cmd}') + print('*'*60) + if os.system(cmd): + failed = True + print('\n'*5) + print('*'*60) + print(f'* Command failed on run {i+1}**') + print('*'*60) + print('\n'*5) + sys.exit(1); if __name__ == '__main__': n = int(sys.argv[1]) @@ -29,4 +39,6 @@ if __name__ == '__main__': with ProcessPoolExecutor(max_workers=k) as executor: futures = [executor.submit(run_cmd, cmd, i) for i in range(n)] for f in as_completed(futures): - f.result() # Raises exception if failed \ No newline at end of file + f.result() # Raises exception if failed + + print(f"successfully ran {n} times") \ No newline at end of file From c17234b5386371f0e8803581e46f1a13581d9275 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 20 Jul 2025 20:57:44 +0000 Subject: [PATCH 042/798] syncdoc: unit test involving merging many heads --- .../conat/test/sync-doc/conflict.test.ts | 63 +++++++++++++++++++ src/packages/conat/sync/dko.ts | 3 +- src/scripts/runoo | 6 +- 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/src/packages/backend/conat/test/sync-doc/conflict.test.ts b/src/packages/backend/conat/test/sync-doc/conflict.test.ts index 00b590f8fe..34797cc53e 100644 --- a/src/packages/backend/conat/test/sync-doc/conflict.test.ts +++ b/src/packages/backend/conat/test/sync-doc/conflict.test.ts @@ -17,6 +17,7 @@ import { delay, waitUntilSynced, } from "./setup"; +import { split } from "@cocalc/util/misc"; beforeAll(before); afterAll(after); @@ -212,3 +213,65 @@ describe("do the example in the blog post 'Lies I was Told About Collaborative E expect(bob.to_str()).toEqual("The Colour of Pomegranates"); }); }); + +const numHeads = 15; +describe.only(`create editing conflict with ${numHeads} heads`, () => { + const project_id = uuid(); + let docs: any[] = [], + clients: any[] = []; + + it(`create ${numHeads} clients`, async () => { + const v: any[] = []; + for (let i = 0; i < numHeads; i++) { + const client = connect(); + clients.push(client); + const doc = client.sync.string({ + project_id, + path: "a.txt", + service: server.service, + noAutosave: true, + }); + docs.push(doc); + v.push(once(doc, "ready")); + } + await Promise.all(v); + }); + + it("every client writes a different value all at once", async () => { + for (let i = 0; i < numHeads; i++) { + docs[i].from_str(`${i} `); + docs[i].commit(); + docs[i].save(); + } + await waitUntilSynced(docs); + const heads = docs[0].patch_list.getHeads(); + expect(heads.length).toBe(docs.length); + }); + + it("merge -- order is random, but value is consistent", async () => { + const value = docs[0].to_str(); + let v = new Set(); + for (let i = 0; i < numHeads; i++) { + v.add(`${i}`); + expect(docs[i].to_str()).toEqual(value); + } + const t = new Set(split(docs[0].to_str())); + expect(t).toEqual(v); + }); + + it(`resolve the merge conflict -- all ${numHeads} clients then see the resolution`, async () => { + let r = ""; + for (let i = 0; i < numHeads; i++) { + r += `${i} `; + } + docs[0].from_str(r); + docs[0].commit(); + await docs[0].save(); + + await waitUntilSynced(docs); + for (let i = 0; i < numHeads; i++) { + expect(docs[i].to_str()).toEqual(r); + } + // docs[0].show_history(); + }); +}); diff --git a/src/packages/conat/sync/dko.ts b/src/packages/conat/sync/dko.ts index f4e773a90f..c0c04c1b06 100644 --- a/src/packages/conat/sync/dko.ts +++ b/src/packages/conat/sync/dko.ts @@ -1,7 +1,7 @@ /* Distributed eventually consistent key:object store, where changes propogate sparsely. -The "values" MUST be objects and no keys or fields of objects can container the +The "values" MUST be objects and no keys or fields of objects can container the sep character, which is '|' by default. NOTE: Whenever you do a set, the lodash isEqual function is used to see which fields @@ -38,6 +38,7 @@ export class DKO extends EventEmitter { constructor(private opts: DKVOptions) { super(); + this.setMaxListeners(1000); return new Proxy(this, { deleteProperty(target, prop) { if (typeof prop == "string") { diff --git a/src/scripts/runoo b/src/scripts/runoo index 4445904c26..4c0f8e93cc 100755 --- a/src/scripts/runoo +++ b/src/scripts/runoo @@ -41,4 +41,8 @@ if __name__ == '__main__': for f in as_completed(futures): f.result() # Raises exception if failed - print(f"successfully ran {n} times") \ No newline at end of file + print('\n'*5) + print('*'*60) + print(f"Successfully ran {n} times") + print('*'*60) + \ No newline at end of file From 8da8ef57ba93fc532876afaccbf7e70555eb14c2 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 20 Jul 2025 21:10:51 +0000 Subject: [PATCH 043/798] sync-doc: make the "read from disk" command "readFile" and be public; start writing unit tests for watch (not implemented) --- .../conat/test/sync-doc/watch-file.test.ts | 39 +++++ src/packages/sync/editor/generic/sync-doc.ts | 139 +++++++++--------- 2 files changed, 107 insertions(+), 71 deletions(-) create mode 100644 src/packages/backend/conat/test/sync-doc/watch-file.test.ts diff --git a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts new file mode 100644 index 0000000000..8e0a2dd92d --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts @@ -0,0 +1,39 @@ +import { before, after, uuid, connect, server, once, wait } from "./setup"; + +beforeAll(before); +afterAll(after); + +describe("basic watching of file on disk happens automatically", () => { + const project_id = uuid(); + const path = "a.txt"; + let client, s, fs; + + it("creates two clients with noAutosave enabled", async () => { + client = connect(); + fs = client.fs({ project_id, service: server.service }); + await fs.writeFile(path, "init"); + s = client.sync.string({ + project_id, + path, + service: server.service, + }); + await once(s, "ready"); + expect(s.to_str()).toEqual("init"); + }); + + it("changes the file on disk and call readFile to immediately update", async () => { + await fs.writeFile(path, "modified"); + await s.readFile(); + expect(s.to_str()).toEqual("modified"); + }); + + // this is not implemented yet + it.skip("changes the file on disk and the watcher automatically updates with no explicit call needed", async () => { + await fs.writeFile(path, "changed again!"); + await wait({ + until: () => { + return s.to_str() == "changed again!"; + }, + }); + }); +}); diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 66f0939626..acc042b628 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -81,7 +81,6 @@ import { cancel_scheduled, once, retry_until_success, - reuse_in_flight_methods, until, } from "@cocalc/util/async-utils"; import { wait } from "@cocalc/util/async-wait"; @@ -342,13 +341,6 @@ export class SyncDoc extends EventEmitter { // to update this). this.cursor_last_time = this.client?.server_time(); - reuse_in_flight_methods(this, [ - "save", - "save_to_disk", - "load_from_disk", - "handle_patch_update_queue", - ]); - if (this.change_throttle) { this.emit_change = throttle(this.emit_change, this.change_throttle); } @@ -555,7 +547,7 @@ export class SyncDoc extends EventEmitter { await this.update_watch_path(); } else { // load our state from the disk - await this.load_from_disk(); + await this.readFile(); // we were not acting as the file server, but now we need. Let's // step up to the plate. // start watching filesystem @@ -1845,7 +1837,7 @@ export class SyncDoc extends EventEmitter { dbg( `disk file changed more recently than edits (or first load), so loading, ${stats.ctime} > ${last_changed}; firstLoad=${firstLoad}`, ); - size = await this.load_from_disk(); + size = await this.readFile(); if (firstLoad) { dbg("emitting first-load event"); // this event is emited the first time the document is ever loaded from disk. @@ -3000,7 +2992,7 @@ export class SyncDoc extends EventEmitter { // to save was at least RECENT_SAVE_TO_DISK_MS ago, and it finished, // so definitely this change event was not caused by it. dbg("load_from_disk since no recent save to disk"); - await this.load_from_disk(); + await this.readFile(); return; } }; @@ -3039,7 +3031,10 @@ export class SyncDoc extends EventEmitter { return size; }; - private load_from_disk = async (): Promise => { + readFile = reuseInFlight(async (): Promise => { + if (this.fs != null) { + return await this.fsLoadFromDisk(); + } if (this.client.path_exists == null) { throw Error("legacy clients must define path_exists"); } @@ -3075,7 +3070,7 @@ export class SyncDoc extends EventEmitter { // save new version to database, which we just set via from_str. await this.save(); return size; - }; + }); private set_save = async (save: { state: string; @@ -3240,7 +3235,7 @@ export class SyncDoc extends EventEmitter { /* Initiates a save of file to disk, then waits for the state to change. */ - save_to_disk = async (): Promise => { + save_to_disk = reuseInFlight(async (): Promise => { if (this.state != "ready") { // We just make save_to_disk a successful // no operation, if the document is either @@ -3309,7 +3304,7 @@ export class SyncDoc extends EventEmitter { await this.wait_for_save_to_disk_done(); } this.update_has_unsaved_changes(); - }; + }); /* Export the (currently loaded) history of editing of this document to a simple JSON-able object. */ @@ -3539,68 +3534,70 @@ export class SyncDoc extends EventEmitter { Whenever new patches are added to this.patches_table, their timestamp gets added to this.patch_update_queue. */ - private handle_patch_update_queue = async (save = false): Promise => { - const dbg = this.dbg("handle_patch_update_queue"); - try { - this.handle_patch_update_queue_running = true; - while (this.state != "closed" && this.patch_update_queue.length > 0) { - dbg("queue size = ", this.patch_update_queue.length); - const v: Patch[] = []; - for (const key of this.patch_update_queue) { - let x = this.patches_table.get(key); - if (x == null) { - continue; + private handle_patch_update_queue = reuseInFlight( + async (save = false): Promise => { + const dbg = this.dbg("handle_patch_update_queue"); + try { + this.handle_patch_update_queue_running = true; + while (this.state != "closed" && this.patch_update_queue.length > 0) { + dbg("queue size = ", this.patch_update_queue.length); + const v: Patch[] = []; + for (const key of this.patch_update_queue) { + let x = this.patches_table.get(key); + if (x == null) { + continue; + } + if (!Map.isMap(x)) { + // TODO: my NATS synctable-stream doesn't convert to immutable on get. + x = fromJS(x); + } + // may be null, e.g., when deleted. + const t = x.get("time"); + // Optimization: only need to process patches that we didn't + // create ourselves during this session. + if (t && !this.my_patches[t.valueOf()]) { + const p = this.processPatch({ x }); + //dbg(`patch=${JSON.stringify(p)}`); + if (p != null) { + v.push(p); + } + } } - if (!Map.isMap(x)) { - // TODO: my NATS synctable-stream doesn't convert to immutable on get. - x = fromJS(x); + this.patch_update_queue = []; + this.emit("patch-update-queue-empty"); + assertDefined(this.patch_list); + this.patch_list.add(v); + + dbg("waiting for remote and doc to sync..."); + this.sync_remote_and_doc(v.length > 0); + if (save || !this.noAutosave) { + await this.patches_table.save(); + if (this.state === ("closed" as State)) return; // closed during await; nothing further to do + dbg("remote and doc now synced"); } - // may be null, e.g., when deleted. - const t = x.get("time"); - // Optimization: only need to process patches that we didn't - // create ourselves during this session. - if (t && !this.my_patches[t.valueOf()]) { - const p = this.processPatch({ x }); - //dbg(`patch=${JSON.stringify(p)}`); - if (p != null) { - v.push(p); - } + + if (this.patch_update_queue.length > 0) { + // It is very important that next loop happen in a later + // event loop to avoid the this.sync_remote_and_doc call + // in this.handle_patch_update_queue above from causing + // sync_remote_and_doc to get called from within itself, + // due to synctable changes being emited on save. + dbg("wait for next event loop"); + await delay(1); } } - this.patch_update_queue = []; - this.emit("patch-update-queue-empty"); - assertDefined(this.patch_list); - this.patch_list.add(v); - - dbg("waiting for remote and doc to sync..."); - this.sync_remote_and_doc(v.length > 0); - if (save || !this.noAutosave) { - await this.patches_table.save(); - if (this.state === ("closed" as State)) return; // closed during await; nothing further to do - dbg("remote and doc now synced"); - } + } finally { + if (this.state == "closed") return; // got closed, so nothing further to do - if (this.patch_update_queue.length > 0) { - // It is very important that next loop happen in a later - // event loop to avoid the this.sync_remote_and_doc call - // in this.handle_patch_update_queue above from causing - // sync_remote_and_doc to get called from within itself, - // due to synctable changes being emited on save. - dbg("wait for next event loop"); - await delay(1); - } + // OK, done and nothing in the queue + // Notify save() to try again -- it may have + // paused waiting for this to clear. + dbg("done"); + this.handle_patch_update_queue_running = false; + this.emit("handle_patch_update_queue_done"); } - } finally { - if (this.state == "closed") return; // got closed, so nothing further to do - - // OK, done and nothing in the queue - // Notify save() to try again -- it may have - // paused waiting for this to clear. - dbg("done"); - this.handle_patch_update_queue_running = false; - this.emit("handle_patch_update_queue_done"); - } - }; + }, + ); /* Disable and enable sync. When disabled we still collect patches from upstream (but do not apply them From 681e2a2820e9aa14819fae59861931c80ec0c1df Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 20 Jul 2025 22:27:09 +0000 Subject: [PATCH 044/798] sync-doc: first steps toward client-only fs listener --- .../conat/test/sync-doc/watch-file.test.ts | 13 +++- src/packages/sync/editor/generic/sync-doc.ts | 64 +++++++++++++++---- 2 files changed, 62 insertions(+), 15 deletions(-) diff --git a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts index 8e0a2dd92d..873b81c886 100644 --- a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts +++ b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts @@ -28,7 +28,7 @@ describe("basic watching of file on disk happens automatically", () => { }); // this is not implemented yet - it.skip("changes the file on disk and the watcher automatically updates with no explicit call needed", async () => { + it("change file on disk and it automatically updates with no explicit call needed", async () => { await fs.writeFile(path, "changed again!"); await wait({ until: () => { @@ -37,3 +37,14 @@ describe("basic watching of file on disk happens automatically", () => { }); }); }); + +/* +watching of file with multiple clients + +-- only one does the actual file load + +-- when one writes file to disk, another doesn't try to load it + +(various ways to do that: sticky fs server would mean only one is +writing backend can ignore the resulting change event) +*/ diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index acc042b628..538145d147 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -117,6 +117,7 @@ import { CONAT_OPEN_FILE_TOUCH_INTERVAL } from "@cocalc/util/conat"; import mergeDeep from "@cocalc/util/immutable-deep-merge"; import { JUPYTER_SYNCDB_EXTENSIONS } from "@cocalc/util/jupyter/names"; import { LegacyHistory } from "./legacy"; +import { type Filesystem } from "@cocalc/conat/files/fs"; import { getLogger } from "@cocalc/conat/client"; const DEBUG = false; @@ -124,12 +125,6 @@ const DEBUG = false; export type State = "init" | "ready" | "closed"; export type DataServer = "project" | "database"; -export interface SyncDocFilesystem { - readFile: (path: string, encoding?: any) => Promise; - writeFile: (path: string, data: string | Buffer) => Promise; - stat: (path: string) => Promise; // todo -} - export interface SyncOpts0 { project_id: string; path: string; @@ -157,8 +152,8 @@ export interface SyncOpts0 { // which data/changefeed server to use data_server?: DataServer; - // optional filesystem interface. - fs?: SyncDocFilesystem; + // filesystem interface. + fs?: Filesystem; // if true, do not implicitly save on commit. This is very // useful for unit testing to easily simulate offline state. @@ -276,7 +271,7 @@ export class SyncDoc extends EventEmitter { private useConat: boolean; legacy: LegacyHistory; - private fs?: SyncDocFilesystem; + private fs?: Filesystem; private noAutosave?: boolean; @@ -396,7 +391,6 @@ export class SyncDoc extends EventEmitter { // Success -- everything initialized with no issues. this.set_state("ready"); - this.init_watch(); this.emit_change(); // from nothing to something. }; @@ -1174,6 +1168,8 @@ export class SyncDoc extends EventEmitter { // a file change in its current state. this.update_watch_path(); // no input = closes it, if open + this.fsCloseFileWatcher(); + if (this.patch_list != null) { // not async -- just a data structure in memory this.patch_list.close(); @@ -1494,6 +1490,7 @@ export class SyncDoc extends EventEmitter { this.init_cursors(), this.init_evaluator(), this.init_ipywidgets(), + this.initFileWatcher(), ]); this.assert_not_closed( "initAll -- successful init patch_list, cursors, evaluator, and ipywidgets", @@ -2871,7 +2868,11 @@ export class SyncDoc extends EventEmitter { this.emit("metadata-change"); }; - private init_watch = async (): Promise => { + private initFileWatcher = async (): Promise => { + if (this.fs != null) { + return await this.fsInitFileWatcher(); + } + if (!(await this.isFileServer())) { // ensures we are NOT watching anything await this.update_watch_path(); @@ -3026,8 +3027,9 @@ export class SyncDoc extends EventEmitter { } } // save new version to stream, which we just set via from_str - this.commit(); + this.commit(true); await this.save(); + this.emit("after-change"); return size; }; @@ -3229,8 +3231,12 @@ export class SyncDoc extends EventEmitter { return; } dbg(); - if (this.fs == null) throw Error("bug"); - await this.fs.writeFile(this.path, this.to_str()); + if (this.fs == null) { + throw Error("bug"); + } + const value = this.to_str(); + console.log("fs.writeFile", this.path); + await this.fs.writeFile(this.path, value); }; /* Initiates a save of file to disk, then waits for the @@ -3754,6 +3760,36 @@ export class SyncDoc extends EventEmitter { }, ); }; + + private fsFileWatcher?: any; + private fsInitFileWatcher = async () => { + if (this.fs == null) { + throw Error("this.fs must be defined"); + } + console.log("watching for changes"); + // use this.fs interface to watch path for changes. + this.fsFileWatcher = await this.fs.watch(this.path); + (async () => { + for await (const { eventType } of this.fsFileWatcher) { + console.log("got change", eventType); + if (eventType == "change" || eventType == "rename") { + await this.fsLoadFromDisk(); + } + if (eventType == "rename") { + this.fsFileWatcher.close(); + // start a new watcher since file descriptor changed + this.fsInitFileWatcher(); + return; + } + } + console.log("done watching"); + })(); + }; + + private fsCloseFileWatcher = () => { + this.fsFileWatcher?.close(); + delete this.fsFileWatcher; + }; } function isCompletePatchStream(dstream) { From f804b01b61be119b3a83570c3c5916169589ba9d Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 20 Jul 2025 23:16:29 +0000 Subject: [PATCH 045/798] sync-doc: working on multiple clients efficiently watching for fs changes without interfering with each other --- .../conat/test/sync-doc/watch-file.test.ts | 25 +++++++- src/packages/backend/files/sandbox/index.ts | 13 +--- src/packages/conat/files/watch.ts | 62 ++++++++++++++----- src/packages/sync/editor/generic/sync-doc.ts | 25 ++++++-- 4 files changed, 93 insertions(+), 32 deletions(-) diff --git a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts index 873b81c886..fc345b0947 100644 --- a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts +++ b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts @@ -27,7 +27,6 @@ describe("basic watching of file on disk happens automatically", () => { expect(s.to_str()).toEqual("modified"); }); - // this is not implemented yet it("change file on disk and it automatically updates with no explicit call needed", async () => { await fs.writeFile(path, "changed again!"); await wait({ @@ -36,6 +35,30 @@ describe("basic watching of file on disk happens automatically", () => { }, }); }); + + let client2, s2; + it("file watching also works if there are multiple clients, with only one handling the change", async () => { + client2 = connect(); + s2 = client2.sync.string({ + project_id, + path, + service: server.service, + }); + let c = 0, + c2 = 0; + s.on("after-change", () => c++); + s2.on("after-change", () => c2++); + + await fs.writeFile(path, "version3"); + await wait({ + until: () => { + return s2.to_str() == "version3" && s.to_str() == "version3"; + }, + }); + expect(s.to_str()).toEqual("version3"); + expect(s2.to_str()).toEqual("version3"); + expect(c + c2).toBe(1); + }); }); /* diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index 2888e3a480..27b4a800ad 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -69,6 +69,7 @@ import getListing from "@cocalc/backend/get-listing"; import { join, resolve } from "path"; import { replace_all } from "@cocalc/util/misc"; import { EventIterator } from "@cocalc/util/event-iterator"; +import { type WatchOptions } from "@cocalc/conat/files/watch"; export class SandboxedFilesystem { // path should be the path to a FOLDER on the filesystem (not a file) @@ -231,17 +232,7 @@ export class SandboxedFilesystem { await utimes(await this.safeAbsPath(path), atime, mtime); }; - watch = async ( - filename: string, - options?: { - persistent?: boolean; - recursive?: boolean; - encoding?: string; - signal?: AbortSignal; - maxQueue?: number; - overflow?: "ignore" | "throw"; - }, - ) => { + watch = async (filename: string, options?: WatchOptions) => { // NOTE: in node v24 they fixed the fs/promises watch to have a queue, but previous // versions were clearly badly implemented so we reimplement it from scratch // using the non-promise watch. diff --git a/src/packages/conat/files/watch.ts b/src/packages/conat/files/watch.ts index 47dd65d939..942d3116b5 100644 --- a/src/packages/conat/files/watch.ts +++ b/src/packages/conat/files/watch.ts @@ -15,7 +15,21 @@ const logger = getLogger("conat:files:watch"); // (path:string, options:WatchOptions) => AsyncIterator type AsyncWatchFunction = any; -type WatchOptions = any; + +// see https://nodejs.org/docs/latest/api/fs.html#fspromiseswatchfilename-options +export interface WatchOptions { + persistent?: boolean; + recursive?: boolean; + encoding?: string; + signal?: AbortSignal; + maxQueue?: number; + overflow?: "ignore" | "throw"; + + // if more than one client is actively watching the same path and has unique set, only one + // will receive updates. Also, if there are multiple clients with unique set, the options + // of all but the first are ignored. + unique?: boolean; +} export function watchServer({ client, @@ -29,29 +43,47 @@ export function watchServer({ const server: ConatSocketServer = client.socket.listen(subject); logger.debug("server: listening on ", { subject }); + //const unique; + async function handleUnique({ mesg, socket, path, options }) { + const w = await watch(path, options); + socket.once("closed", () => { + w.close(); + }); + await mesg.respond(); + for await (const event of w) { + socket.write(event); + } + } + + async function handleNonUnique({ mesg, socket, path, options }) { + const w = await watch(path, options); + socket.once("closed", () => { + w.close(); + }); + await mesg.respond(); + for await (const event of w) { + socket.write(event); + } + } + server.on("connection", (socket: ServerSocket) => { logger.debug("server: got new connection", { id: socket.id, subject: socket.subject, }); - let w: undefined | ReturnType = undefined; - socket.on("closed", () => { - w?.close(); - w = undefined; - }); - + let initialized = false; socket.on("request", async (mesg) => { try { + if (initialized) { + throw Error("already initialized"); + } + initialized = true; const { path, options } = mesg.data; logger.debug("got request", { path, options }); - if (w != null) { - w.close(); - w = undefined; - } - w = await watch(path, options); - await mesg.respond(); - for await (const event of w) { - socket.write(event); + if (options?.unique) { + await handleUnique({ mesg, socket, path, options }); + } else { + await handleNonUnique({ mesg, socket, path, options }); } } catch (err) { mesg.respondSync(null, { diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 538145d147..79c65b3739 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -82,6 +82,7 @@ import { once, retry_until_success, until, + asyncDebounce, } from "@cocalc/util/async-utils"; import { wait } from "@cocalc/util/async-wait"; import { @@ -3018,6 +3019,7 @@ export class SyncDoc extends EventEmitter { size = contents.length; this.from_str(contents); } catch (err) { + console.log(err); if (err.code == "ENOENT") { dbg("file no longer exists -- setting to blank"); size = 0; @@ -3761,19 +3763,32 @@ export class SyncDoc extends EventEmitter { ); }; + private fsLoadFromDiskDebounced = asyncDebounce( + async () => { + try { + await this.fsLoadFromDisk(); + } catch {} + }, + 50, + { + leading: false, + trailing: true, + }, + ); + private fsFileWatcher?: any; private fsInitFileWatcher = async () => { if (this.fs == null) { throw Error("this.fs must be defined"); } - console.log("watching for changes"); + // console.log("watching for changes"); // use this.fs interface to watch path for changes. - this.fsFileWatcher = await this.fs.watch(this.path); + this.fsFileWatcher = await this.fs.watch(this.path, { unique: true }); (async () => { for await (const { eventType } of this.fsFileWatcher) { - console.log("got change", eventType); + // console.log("got change", eventType); if (eventType == "change" || eventType == "rename") { - await this.fsLoadFromDisk(); + this.fsLoadFromDiskDebounced(); } if (eventType == "rename") { this.fsFileWatcher.close(); @@ -3782,7 +3797,7 @@ export class SyncDoc extends EventEmitter { return; } } - console.log("done watching"); + //console.log("done watching"); })(); }; From e64eee33aa7a8201cb84b7e98b4b5333e47f9576 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 20 Jul 2025 23:54:38 +0000 Subject: [PATCH 046/798] sync-doc: implement a pretty nice approach to filesystem watching for sync editing --- .../conat/test/sync-doc/watch-file.test.ts | 37 ++++++++++++++++++- src/packages/conat/files/watch.ts | 34 ++++++++++++++--- src/packages/sync/editor/generic/sync-doc.ts | 9 +++-- 3 files changed, 68 insertions(+), 12 deletions(-) diff --git a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts index fc345b0947..a1ad595960 100644 --- a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts +++ b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts @@ -44,10 +44,11 @@ describe("basic watching of file on disk happens automatically", () => { path, service: server.service, }); + await once(s2, "ready"); let c = 0, c2 = 0; - s.on("after-change", () => c++); - s2.on("after-change", () => c2++); + s.on("handle-file-change", () => c++); + s2.on("handle-file-change", () => c2++); await fs.writeFile(path, "version3"); await wait({ @@ -59,6 +60,38 @@ describe("basic watching of file on disk happens automatically", () => { expect(s2.to_str()).toEqual("version3"); expect(c + c2).toBe(1); }); + + it("file watching must still work if either client is closed", async () => { + s.close(); + await fs.writeFile(path, "version4"); + await wait({ + until: () => { + return s2.to_str() == "version4"; + }, + }); + expect(s2.to_str()).toEqual("version4"); + }); + + let client3, s3; + it("add a third client and close client2 and have file watching still work", async () => { + client3 = connect(); + s3 = client3.sync.string({ + project_id, + path, + service: server.service, + }); + await once(s3, "ready"); + s2.close(); + + await fs.writeFile(path, "version5"); + + await wait({ + until: () => { + return s3.to_str() == "version5"; + }, + }); + expect(s3.to_str()).toEqual("version5"); + }); }); /* diff --git a/src/packages/conat/files/watch.ts b/src/packages/conat/files/watch.ts index 942d3116b5..03a05c57ae 100644 --- a/src/packages/conat/files/watch.ts +++ b/src/packages/conat/files/watch.ts @@ -43,15 +43,37 @@ export function watchServer({ const server: ConatSocketServer = client.socket.listen(subject); logger.debug("server: listening on ", { subject }); - //const unique; + const unique: { [path: string]: ServerSocket[] } = {}; async function handleUnique({ mesg, socket, path, options }) { - const w = await watch(path, options); + let w: any = undefined; + socket.once("closed", () => { - w.close(); + // when this socket closes, remove it from recipient list + unique[path] = unique[path]?.filter((x) => x.id != socket.id); + if (unique[path] != null && unique[path].length == 0) { + // nobody listening + w?.close(); + w = undefined; + delete unique[path]; + } }); - await mesg.respond(); - for await (const event of w) { - socket.write(event); + + if (unique[path] == null) { + // set it up + unique[path] = [socket]; + w = await watch(path, options); + await mesg.respond(); + for await (const event of w) { + for (const s of unique[path]) { + if (s.state == "ready") { + s.write(event); + break; + } + } + } + } else { + unique[path].push(socket); + await mesg.respond(); } } diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 79c65b3739..3d89526e1d 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -61,6 +61,8 @@ const CURSOR_THROTTLE_NATS_MS = 150; // Ignore file changes for this long after save to disk. const RECENT_SAVE_TO_DISK_MS = 2000; +const WATCH_DEBOUNCE = 250; + const PARALLEL_INIT = true; import { @@ -3019,7 +3021,6 @@ export class SyncDoc extends EventEmitter { size = contents.length; this.from_str(contents); } catch (err) { - console.log(err); if (err.code == "ENOENT") { dbg("file no longer exists -- setting to blank"); size = 0; @@ -3237,7 +3238,6 @@ export class SyncDoc extends EventEmitter { throw Error("bug"); } const value = this.to_str(); - console.log("fs.writeFile", this.path); await this.fs.writeFile(this.path, value); }; @@ -3766,10 +3766,11 @@ export class SyncDoc extends EventEmitter { private fsLoadFromDiskDebounced = asyncDebounce( async () => { try { + this.emit("handle-file-change"); await this.fsLoadFromDisk(); } catch {} }, - 50, + WATCH_DEBOUNCE, { leading: false, trailing: true, @@ -3786,7 +3787,7 @@ export class SyncDoc extends EventEmitter { this.fsFileWatcher = await this.fs.watch(this.path, { unique: true }); (async () => { for await (const { eventType } of this.fsFileWatcher) { - // console.log("got change", eventType); + //console.log("got change", eventType); if (eventType == "change" || eventType == "rename") { this.fsLoadFromDiskDebounced(); } From c4c96322ade03914ff75b62ace463e50a643ea48 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 21 Jul 2025 01:07:59 +0000 Subject: [PATCH 047/798] syncdoc: working but hacky approach to saving to disk without causing a load --- .../conat/test/sync-doc/watch-file.test.ts | 28 ++++++++++++++++++- src/packages/conat/files/watch.ts | 28 ++++++++++++++++++- src/packages/sync/editor/generic/sync-doc.ts | 4 +++ 3 files changed, 58 insertions(+), 2 deletions(-) diff --git a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts index a1ad595960..42da5aeb10 100644 --- a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts +++ b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts @@ -1,4 +1,13 @@ -import { before, after, uuid, connect, server, once, wait } from "./setup"; +import { + before, + after, + uuid, + connect, + server, + once, + wait, + delay, +} from "./setup"; beforeAll(before); afterAll(after); @@ -36,6 +45,23 @@ describe("basic watching of file on disk happens automatically", () => { }); }); + it("change file on disk should not trigger a load from disk", async () => { + const orig = s.fsLoadFromDiskDebounced; + let c = 0; + s.fsLoadFromDiskDebounced = () => { + c += 1; + }; + s.from_str("a different value"); + await s.save_to_disk(); + expect(c).toBe(0); + await delay(100); + expect(c).toBe(0); + s.fsLoadFromDiskDebounced = orig; + // disable the ignore that happens as part of save_to_disk, + // or the tests below won't work + await s.fsFileWatcher?.ignore(0); + }); + let client2, s2; it("file watching also works if there are multiple clients, with only one handling the change", async () => { client2 = connect(); diff --git a/src/packages/conat/files/watch.ts b/src/packages/conat/files/watch.ts index 03a05c57ae..6032f2b023 100644 --- a/src/packages/conat/files/watch.ts +++ b/src/packages/conat/files/watch.ts @@ -64,6 +64,17 @@ export function watchServer({ w = await watch(path, options); await mesg.respond(); for await (const event of w) { + const now = Date.now(); + let doIgnore = false; + for (const s of unique[path]) { + if (s.ignoreUntil != null && s.ignoreUntil > now) { + doIgnore = true; + break; + } + } + if (doIgnore) { + continue; + } for (const s of unique[path]) { if (s.state == "ready") { s.write(event); @@ -84,6 +95,9 @@ export function watchServer({ }); await mesg.respond(); for await (const event of w) { + if ((socket.ignoreUntil ?? 0) >= Date.now()) { + continue; + } socket.write(event); } } @@ -95,12 +109,18 @@ export function watchServer({ }); let initialized = false; socket.on("request", async (mesg) => { + const data = mesg.data; + if (data.ignore != null) { + socket.ignoreUntil = Date.now() + data.ignore; + await mesg.respond(null, { noThrow: true }); + return; + } try { if (initialized) { throw Error("already initialized"); } initialized = true; - const { path, options } = mesg.data; + const { path, options } = data; logger.debug("got request", { path, options }); if (options?.unique) { await handleUnique({ mesg, socket, path, options }); @@ -141,5 +161,11 @@ export async function watchClient({ }); // tell it what to watch await socket.request({ path, options }); + + // ignore events for ignore ms. + iter.ignore = async (ignore: number) => { + await socket.request({ ignore }); + }; + return iter; } diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 3d89526e1d..286f3d3507 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -3238,6 +3238,10 @@ export class SyncDoc extends EventEmitter { throw Error("bug"); } const value = this.to_str(); + // tell watcher not to fire any change events for a little time, + // so no clients waste resources loading in response to us saving + // to disk. + await this.fsFileWatcher?.ignore(2000); await this.fs.writeFile(this.path, value); }; From 6b52a5a4240b46d99f9a53da52c690734dc99292 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 21 Jul 2025 01:20:48 +0000 Subject: [PATCH 048/798] file watch ignore when saving -- more ts friendly implementation --- src/packages/conat/files/fs.ts | 9 ++++++-- src/packages/conat/files/watch.ts | 38 ++++++++++++++++++++++--------- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index a06baa02bb..80b96ecccd 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -1,6 +1,10 @@ import { type Client } from "@cocalc/conat/core/client"; import { conat } from "@cocalc/conat/client"; -import { watchServer, watchClient } from "@cocalc/conat/files/watch"; +import { + watchServer, + watchClient, + type WatchIterator, +} from "@cocalc/conat/files/watch"; export const DEFAULT_FILE_SERVICE = "fs"; export interface Filesystem { @@ -30,7 +34,7 @@ export interface Filesystem { ) => Promise; writeFile: (path: string, data: string | Buffer) => Promise; // todo: typing - watch: (path: string, options?) => Promise; + watch: (path: string, options?) => Promise; } interface IStats { @@ -183,6 +187,7 @@ export async function fsServer({ service, fs, client }: Options) { async writeFile(path: string, data: string | Buffer) { await (await fs(this.subject)).writeFile(path, data); }, + // @ts-ignore async watch() { const subject = this.subject!; if (watches[subject] != null) { diff --git a/src/packages/conat/files/watch.ts b/src/packages/conat/files/watch.ts index 6032f2b023..3372da2279 100644 --- a/src/packages/conat/files/watch.ts +++ b/src/packages/conat/files/watch.ts @@ -44,7 +44,8 @@ export function watchServer({ logger.debug("server: listening on ", { subject }); const unique: { [path: string]: ServerSocket[] } = {}; - async function handleUnique({ mesg, socket, path, options }) { + const ignores: { [path: string]: { ignoreUntil: number }[] } = {}; + async function handleUnique({ mesg, socket, path, options, ignore }) { let w: any = undefined; socket.once("closed", () => { @@ -55,19 +56,21 @@ export function watchServer({ w?.close(); w = undefined; delete unique[path]; + delete ignores[path]; } }); if (unique[path] == null) { // set it up unique[path] = [socket]; + ignores[path] = [ignore]; w = await watch(path, options); await mesg.respond(); for await (const event of w) { const now = Date.now(); let doIgnore = false; - for (const s of unique[path]) { - if (s.ignoreUntil != null && s.ignoreUntil > now) { + for (const { ignoreUntil } of ignores[path]) { + if (ignoreUntil > now) { doIgnore = true; break; } @@ -84,18 +87,19 @@ export function watchServer({ } } else { unique[path].push(socket); + ignores[path].push(ignore); await mesg.respond(); } } - async function handleNonUnique({ mesg, socket, path, options }) { + async function handleNonUnique({ mesg, socket, path, options, ignore }) { const w = await watch(path, options); socket.once("closed", () => { w.close(); }); await mesg.respond(); for await (const event of w) { - if ((socket.ignoreUntil ?? 0) >= Date.now()) { + if (ignore.ignoreUntil >= Date.now()) { continue; } socket.write(event); @@ -108,10 +112,11 @@ export function watchServer({ subject: socket.subject, }); let initialized = false; + const ignore = { ignoreUntil: 0 }; socket.on("request", async (mesg) => { const data = mesg.data; if (data.ignore != null) { - socket.ignoreUntil = Date.now() + data.ignore; + ignore.ignoreUntil = data.ignore > 0 ? Date.now() + data.ignore : 0; await mesg.respond(null, { noThrow: true }); return; } @@ -123,9 +128,9 @@ export function watchServer({ const { path, options } = data; logger.debug("got request", { path, options }); if (options?.unique) { - await handleUnique({ mesg, socket, path, options }); + await handleUnique({ mesg, socket, path, options, ignore }); } else { - await handleNonUnique({ mesg, socket, path, options }); + await handleNonUnique({ mesg, socket, path, options, ignore }); } } catch (err) { mesg.respondSync(null, { @@ -138,6 +143,15 @@ export function watchServer({ return server; } +export type WatchIterator = EventIterator & { + ignore?: (ignore: number) => Promise; +}; + +export interface ChangeEvent { + eventType: "change" | "rename"; + filename: string; +} + export async function watchClient({ client, subject, @@ -148,7 +162,7 @@ export async function watchClient({ subject: string; path: string; options?: WatchOptions; -}) { +}): Promise { const socket = await client.socket.connect(subject); const iter = new EventIterator(socket, "data", { map: (args) => args[0], @@ -162,10 +176,12 @@ export async function watchClient({ // tell it what to watch await socket.request({ path, options }); + const iter2 = iter as WatchIterator; + // ignore events for ignore ms. - iter.ignore = async (ignore: number) => { + iter2.ignore = async (ignore: number) => { await socket.request({ ignore }); }; - return iter; + return iter2; } From e39cd45380c1be3b14ee47d761b38369f7c5f58d Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 21 Jul 2025 01:24:06 +0000 Subject: [PATCH 049/798] ... --- src/packages/sync/editor/generic/sync-doc.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 286f3d3507..99caf07f8e 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -3242,6 +3242,7 @@ export class SyncDoc extends EventEmitter { // so no clients waste resources loading in response to us saving // to disk. await this.fsFileWatcher?.ignore(2000); + if(this.isClosed()) return; await this.fs.writeFile(this.path, value); }; From 87fb723c0837e393b0d881a3d243909095103987 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 21 Jul 2025 04:06:35 +0000 Subject: [PATCH 050/798] sync-doc: implement extremely simple version of "has unsaved changes" for saved-to-disk - this isn't as "amazing" as the full on hash-on-disk comparison one, but it's vastly simpler and very fast, and should be as good for most realistic purposes. It's probably much closer to what people expect anyways and has the advantage of rarely being wrong. --- .../conat/test/sync-doc/watch-file.test.ts | 51 ++++++++++++++++++- src/packages/conat/files/watch.ts | 22 ++++---- src/packages/sync/editor/generic/sync-doc.ts | 27 ++++++++-- 3 files changed, 85 insertions(+), 15 deletions(-) diff --git a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts index 42da5aeb10..2a1acb48ea 100644 --- a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts +++ b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts @@ -7,6 +7,7 @@ import { once, wait, delay, + waitUntilSynced, } from "./setup"; beforeAll(before); @@ -17,7 +18,7 @@ describe("basic watching of file on disk happens automatically", () => { const path = "a.txt"; let client, s, fs; - it("creates two clients with noAutosave enabled", async () => { + it("creates client", async () => { client = connect(); fs = client.fs({ project_id, service: server.service }); await fs.writeFile(path, "init"); @@ -120,6 +121,54 @@ describe("basic watching of file on disk happens automatically", () => { }); }); +describe.only("has unsaved changes", () => { + const project_id = uuid(); + let s1, s2, client1, client2; + + it("creates two clients", async () => { + client1 = connect(); + client2 = connect(); + s1 = client1.sync.string({ + project_id, + path: "a.txt", + service: server.service, + }); + await once(s1, "ready"); + // definitely has unsaved changes, since it doesn't even exist + expect(s1.has_unsaved_changes()).toBe(true); + + s2 = client2.sync.string({ + project_id, + path: "a.txt", + service: server.service, + }); + await once(s2, "ready"); + expect(s1.to_str()).toBe(""); + expect(s2.to_str()).toBe(""); + expect(s1 === s2).toBe(false); + expect(s2.has_unsaved_changes()).toBe(true); + }); + + it("save empty file to disk -- now no unsaved changes", async () => { + await s1.save_to_disk(); + expect(s1.has_unsaved_changes()).toBe(false); + // but s2 doesn't know anything + expect(s2.has_unsaved_changes()).toBe(true); + }); + + it("make a change via s2 and save", async () => { + s2.from_str("i am s2"); + await s2.save_to_disk(); + expect(s2.has_unsaved_changes()).toBe(false); + }); + + it("as soon as s1 learns that there was a change to the file on disk, it doesn't know", async () => { + await waitUntilSynced([s1, s2]); + expect(s1.has_unsaved_changes()).toBe(true); + expect(s1.to_str()).toEqual("i am s2"); + }); +}); + /* watching of file with multiple clients diff --git a/src/packages/conat/files/watch.ts b/src/packages/conat/files/watch.ts index 3372da2279..531dc82d9f 100644 --- a/src/packages/conat/files/watch.ts +++ b/src/packages/conat/files/watch.ts @@ -25,9 +25,9 @@ export interface WatchOptions { maxQueue?: number; overflow?: "ignore" | "throw"; - // if more than one client is actively watching the same path and has unique set, only one - // will receive updates. Also, if there are multiple clients with unique set, the options - // of all but the first are ignored. + // if more than one client is actively watching the same path and has unique set, all but one may receive + // the extra field ignore:true in the update. Also, if there are multiple clients with unique set, the + // other options of all but the first are ignored. unique?: boolean; } @@ -68,20 +68,22 @@ export function watchServer({ await mesg.respond(); for await (const event of w) { const now = Date.now(); - let doIgnore = false; + let ignore = false; for (const { ignoreUntil } of ignores[path]) { if (ignoreUntil > now) { - doIgnore = true; + // every client is told to ignore this change, i.e., not load based on it happening + ignore = true; break; } } - if (doIgnore) { - continue; - } for (const s of unique[path]) { if (s.state == "ready") { - s.write(event); - break; + if (ignore) { + s.write({ ...event, ignore: true }); + } else { + s.write(event); + ignore = true; + } } } } diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 99caf07f8e..20f9bc1da3 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -1784,6 +1784,7 @@ export class SyncDoc extends EventEmitter { try { stats = await this.fs.stat(this.path); } catch (err) { + this.lastDiskValue = undefined; // nonexistent or don't know if (err.code == "ENOENT") { // path does not exist -- nothing further to do return false; @@ -3017,6 +3018,7 @@ export class SyncDoc extends EventEmitter { let contents; try { contents = await this.fs.readFile(this.path, "utf8"); + this.lastDiskValue = contents; dbg("file exists"); size = contents.length; this.from_str(contents); @@ -3139,6 +3141,9 @@ export class SyncDoc extends EventEmitter { if (this.state !== "ready") { return; } + if (this.fs != null) { + return this.fsHasUnsavedChanges(); + } const dbg = this.dbg("has_unsaved_changes"); try { return this.hash_of_saved_version() !== this.hash_of_live_version(); @@ -3227,6 +3232,11 @@ export class SyncDoc extends EventEmitter { return true; }; + private lastDiskValue: string | undefined = undefined; + fsHasUnsavedChanges = (): boolean => { + return this.lastDiskValue != this.to_str(); + }; + fsSaveToDisk = async () => { const dbg = this.dbg("fsSaveToDisk"); if (this.client.is_deleted(this.path, this.project_id)) { @@ -3242,8 +3252,9 @@ export class SyncDoc extends EventEmitter { // so no clients waste resources loading in response to us saving // to disk. await this.fsFileWatcher?.ignore(2000); - if(this.isClosed()) return; + if (this.isClosed()) return; await this.fs.writeFile(this.path, value); + this.lastDiskValue = value; }; /* Initiates a save of file to disk, then waits for the @@ -3259,9 +3270,14 @@ export class SyncDoc extends EventEmitter { // properly. return; } + if (this.fs != null) { - return await this.fsSaveToDisk(); + this.commit(); + await this.fsSaveToDisk(); + this.update_has_unsaved_changes(); + return; } + const dbg = this.dbg("save_to_disk"); if (this.client.is_deleted(this.path, this.project_id)) { dbg("not saving to disk because deleted"); @@ -3791,12 +3807,15 @@ export class SyncDoc extends EventEmitter { // use this.fs interface to watch path for changes. this.fsFileWatcher = await this.fs.watch(this.path, { unique: true }); (async () => { - for await (const { eventType } of this.fsFileWatcher) { + for await (const { eventType, ignore } of this.fsFileWatcher) { + // we don't know what's on disk anymore, + this.lastDiskValue = undefined; //console.log("got change", eventType); - if (eventType == "change" || eventType == "rename") { + if (!ignore) { this.fsLoadFromDiskDebounced(); } if (eventType == "rename") { + // always have to recreate in case of a rename this.fsFileWatcher.close(); // start a new watcher since file descriptor changed this.fsInitFileWatcher(); From 611ee19dd5bb3147a08ef051fae20b6c3f440b7f Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 21 Jul 2025 23:09:50 +0000 Subject: [PATCH 051/798] rewrite code formatting to not use backend state or service -- just a simple stateless api call - this will make it easier/transparent to just have a backend code formatter, e.g., maybe as part of conat-api, which is important for making editing fast/easier without having to start a project --- .../conat/test/sync-doc/watch-file.test.ts | 11 ---- src/packages/conat/core/client.ts | 1 + src/packages/conat/project/api/editor.ts | 2 +- src/packages/frontend/client/project.ts | 7 ++- .../frame-editors/code-editor/actions.ts | 54 ++++++++++--------- src/packages/project/conat/open-files.ts | 9 ++++ .../project/formatters/format.test.ts | 36 +++++++++++++ src/packages/project/formatters/index.ts | 31 +++++------ .../project/formatters/prettier-lib.ts | 15 ------ src/packages/project/package.json | 2 +- src/packages/sync/editor/db/sync.ts | 5 +- src/packages/sync/editor/generic/util.ts | 1 + 12 files changed, 100 insertions(+), 74 deletions(-) create mode 100644 src/packages/project/formatters/format.test.ts delete mode 100644 src/packages/project/formatters/prettier-lib.ts diff --git a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts index 2a1acb48ea..e1f3cb830f 100644 --- a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts +++ b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts @@ -168,14 +168,3 @@ describe.only("has unsaved changes", () => { expect(s1.to_str()).toEqual("i am s2"); }); }); - -/* -watching of file with multiple clients - --- only one does the actual file load - --- when one writes file to disk, another doesn't try to load it - -(various ways to do that: sticky fs server would mean only one is -writing backend can ignore the resulting change event) -*/ diff --git a/src/packages/conat/core/client.ts b/src/packages/conat/core/client.ts index bfa92256f6..58bc516679 100644 --- a/src/packages/conat/core/client.ts +++ b/src/packages/conat/core/client.ts @@ -273,6 +273,7 @@ export const MAX_INTEREST_TIMEOUT = 90_000; const DEFAULT_WAIT_FOR_INTEREST_TIMEOUT = 30_000; +// WARNING: do NOT change MSGPACK_ENCODER_OPTIONS unless you know what you're doing! const MSGPACK_ENCODER_OPTIONS = { // ignoreUndefined is critical so database queries work properly, and // also we have a lot of api calls with tons of wasted undefined values. diff --git a/src/packages/conat/project/api/editor.ts b/src/packages/conat/project/api/editor.ts index cd70dc973d..37e9d27b5a 100644 --- a/src/packages/conat/project/api/editor.ts +++ b/src/packages/conat/project/api/editor.ts @@ -46,7 +46,7 @@ export interface Editor { jupyterKernels: (opts?: { noCache?: boolean }) => Promise; - // returns a patch to transform str into formatted form. + // returns formatted version of str. formatterString: (opts: { str: string; options: FormatterOptions; diff --git a/src/packages/frontend/client/project.ts b/src/packages/frontend/client/project.ts index 79ed25713f..6d621d4d84 100644 --- a/src/packages/frontend/client/project.ts +++ b/src/packages/frontend/client/project.ts @@ -50,8 +50,11 @@ export class ProjectClient { this.client = client; } - private conatApi = (project_id: string) => { - return this.client.conat_client.projectApi({ project_id }); + conatApi = (project_id: string, compute_server_id = 0) => { + return this.client.conat_client.projectApi({ + project_id, + compute_server_id, + }); }; // This can write small text files in one message. diff --git a/src/packages/frontend/frame-editors/code-editor/actions.ts b/src/packages/frontend/frame-editors/code-editor/actions.ts index 17fc8b7c8a..ded1541198 100644 --- a/src/packages/frontend/frame-editors/code-editor/actions.ts +++ b/src/packages/frontend/frame-editors/code-editor/actions.ts @@ -52,11 +52,11 @@ import { } from "@cocalc/frontend/misc/local-storage"; import { AvailableFeatures } from "@cocalc/frontend/project_configuration"; import { SyncDB } from "@cocalc/sync/editor/db"; -import { apply_patch } from "@cocalc/sync/editor/generic/util"; +import { apply_patch, make_patch } from "@cocalc/sync/editor/generic/util"; import type { SyncString } from "@cocalc/sync/editor/string/sync"; import { once } from "@cocalc/util/async-utils"; import { - Config as FormatterConfig, + Options as FormatterOptions, Exts as FormatterExts, Syntax as FormatterSyntax, Tool as FormatterTool, @@ -89,7 +89,6 @@ import { SetMap, } from "../frame-tree/types"; import { - formatter, get_default_font_size, log_error, syncdb2, @@ -112,6 +111,7 @@ import { misspelled_words } from "./spell-check"; import { log_opened_time } from "@cocalc/frontend/project/open-file"; import { ensure_project_running } from "@cocalc/frontend/project/project-start-warning"; import { alert_message } from "@cocalc/frontend/alerts"; +import { webapp_client } from "@cocalc/frontend/webapp-client"; interface gutterMarkerParams { line: number; @@ -2259,10 +2259,6 @@ export class Actions< const cm = this._get_cm(id); if (!cm) return; - if (!(await this.ensure_latest_changes_are_saved())) { - return; - } - // Important: this function may be called even if there is no format support, // because it can be called via a keyboard shortcut. That's why we gracefully // handle this case -- see https://github.com/sagemathinc/cocalc/issues/4180 @@ -2270,36 +2266,44 @@ export class Actions< if (s == null) { return; } - // TODO: Using any here since TypeMap is just not working right... - if (!this.has_format_support(id, s.get("available_features"))) { - return; - } // Definitely have format support cm.focus(); const ext = filename_extension(this.path).toLowerCase() as FormatterExts; const syntax: FormatterSyntax = ext2syntax[ext]; - const config: FormatterConfig = { - syntax, + const parser = syntax2tool[syntax]; + if (!parser || !this.has_format_support(id, s.get("available_features"))) { + return; + } + const options: FormatterOptions = { + parser, tabWidth: cm.getOption("tabSize") as number, useTabs: cm.getOption("indentWithTabs") as boolean, lastChanged: this._syncstring.last_changed(), }; - this.set_status("Running code formatter..."); + const api = webapp_client.project_client.conatApi(this.project_id); + const str = cm.getValue(); + try { - const patch = await formatter(this.project_id, this.path, config); - if (patch != null) { - // Apply the patch. - // NOTE: old backends that haven't restarted just return {status:'ok'} - // and directly make the change. Delete this comment in a month or so. - // See https://github.com/sagemathinc/cocalc/issues/4335 - this.set_syncstring_to_codemirror(); - const new_val = apply_patch(patch, this._syncstring.to_str())[0]; - this._syncstring.from_str(new_val); - this._syncstring.commit(); - this.set_codemirror_to_syncstring(); + this.set_status("Running code formatter..."); + let formatted = await api.editor.formatterString({ + str, + options, + path: this.path, + }); + if (formatted == str) { + // nothing to do + return; + } + const str2 = cm.getValue(); + if (str2 != str) { + // user made edits *during* formatting, so we "3-way merge" it in, rather + // than breaking what they did: + const patch = make_patch(str, formatted); + formatted = apply_patch(patch, str2)[0]; } + cm.setValueNoJump(formatted); this.setFormatError(""); } catch (err) { this.setFormatError(`${err}`, this._syncstring?.to_str()); diff --git a/src/packages/project/conat/open-files.ts b/src/packages/project/conat/open-files.ts index 10c96d1ede..e5114c6993 100644 --- a/src/packages/project/conat/open-files.ts +++ b/src/packages/project/conat/open-files.ts @@ -221,6 +221,12 @@ function computeServerId(path: string): number { return computeServers?.get(path) ?? 0; } +function hasBackendState(path) { + return ( + path.endsWith("." + JUPYTER_SYNCDB_EXTENSIONS) || path.endsWith(".sagews") + ); +} + async function handleChange({ path, time, @@ -229,6 +235,9 @@ async function handleChange({ doctype, id, }: OpenFileEntry & { id?: number }) { + if (!hasBackendState(path)) { + return; + } try { if (id == null) { id = computeServerId(path); diff --git a/src/packages/project/formatters/format.test.ts b/src/packages/project/formatters/format.test.ts new file mode 100644 index 0000000000..e538da02af --- /dev/null +++ b/src/packages/project/formatters/format.test.ts @@ -0,0 +1,36 @@ +import { run_formatter_string as formatString } from "./index"; + +describe("format some strings", () => { + it("formats markdown with math", async () => { + const s = await formatString({ + str: "# foo\n\n- $\\int x^2$\n- blah", + options: { parser: "markdown" }, + }); + expect(s).toEqual("# foo\n\n- $\\int x^2$\n- blah\n"); + }); + + it("formats some python", async () => { + const s = await formatString({ + str: "def f( n = 0):\n print( n )", + options: { parser: "python" }, + }); + expect(s).toEqual("def f(n=0):\n print(n)\n"); + }); + + it("format some typescript", async () => { + const s = await formatString({ + str: "function f( n = 0) { console.log( n ) }", + options: { parser: "typescript" }, + }); + expect(s).toEqual("function f(n = 0) {\n console.log(n);\n}\n"); + }); + + it("formatting invalid typescript throws an error", async () => { + await expect(async () => { + await formatString({ + str: "function f( n = 0) { console.log( n ) ", + options: { parser: "typescript" }, + }); + }).rejects.toThrow("'}' expected"); + }); +}); diff --git a/src/packages/project/formatters/index.ts b/src/packages/project/formatters/index.ts index e9bff1e08c..ca2c26434e 100644 --- a/src/packages/project/formatters/index.ts +++ b/src/packages/project/formatters/index.ts @@ -13,8 +13,6 @@ Also, by doing this on the backend we don't add 5MB (!) to the webpack frontend something that is not supported on the frontend anyway. */ -declare let require: any; - import { make_patch } from "@cocalc/sync/editor/generic/util"; import { math_escape, math_unescape } from "@cocalc/util/markdown-utils"; import { filename_extension } from "@cocalc/util/misc"; @@ -30,7 +28,6 @@ import { xml_format } from "./xml-format"; // mathjax-utils is from upstream project Jupyter import { once } from "@cocalc/util/async-utils"; import { remove_math, replace_math } from "@cocalc/util/mathjax-utils"; -import { get_prettier } from "./prettier-lib"; import type { Syntax as FormatterSyntax, Config, @@ -91,20 +88,14 @@ export async function run_formatter({ } } const doc = syncstring.get_doc(); - let formatted, math, input0; + let formatted, input0; let input = (input0 = doc.to_str()); - if (options.parser === "markdown") { - [input, math] = remove_math(math_escape(input)); - } try { formatted = await run_formatter_string({ path, str: input, options }); } catch (err) { logger.debug(`run_formatter error: ${err.message}`); return { status: "error", phase: "format", error: err.message }; } - if (options.parser === "markdown") { - formatted = math_unescape(replace_math(formatted, math)); - } // NOTE: the code used to make the change here on the backend. // See https://github.com/sagemathinc/cocalc/issues/4335 for why // that leads to confusion. @@ -121,9 +112,13 @@ export async function run_formatter_string({ options: Options; path?: string; // only used for CLANG }): Promise { - let formatted; - const input = str; + let formatted, math; + let input = str; logger.debug(`run_formatter options.parser: "${options.parser}"`); + if (options.parser === "markdown") { + [input, math] = remove_math(math_escape(input)); + } + switch (options.parser) { case "latex": case "latexindent": @@ -163,12 +158,12 @@ export async function run_formatter_string({ formatted = await rust_format(input, options, logger); break; default: - const prettier = get_prettier(); - if (prettier != null) { - formatted = prettier.format(input, options); - } else { - throw Error("Could not load 'prettier'"); - } + const prettier = await import("prettier"); + formatted = await prettier.format(input, options as any); + } + + if (options.parser === "markdown") { + formatted = math_unescape(replace_math(formatted, math)); } return formatted; } diff --git a/src/packages/project/formatters/prettier-lib.ts b/src/packages/project/formatters/prettier-lib.ts deleted file mode 100644 index 413569cb42..0000000000 --- a/src/packages/project/formatters/prettier-lib.ts +++ /dev/null @@ -1,15 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -// The whole purpose of this is to only load prettier if we really need it – this saves a few MB of project memory usage - -let instance: { format: Function } | null = null; - -export function get_prettier() { - if (instance == null) { - instance = require("prettier"); - } - return instance; -} diff --git a/src/packages/project/package.json b/src/packages/project/package.json index 19f3fda579..320802b629 100644 --- a/src/packages/project/package.json +++ b/src/packages/project/package.json @@ -68,7 +68,7 @@ "start": "NODE_OPTIONS='--trace-warnings --unhandled-rejections=strict --enable-source-maps' pnpm cocalc-project", "build": "../node_modules/.bin/tsc --build", "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput", - "test": "COCALC_PROJECT_ID=812abe34-a382-4bd1-9071-29b6f4334f03 COCALC_USERNAME=user pnpm exec jest", + "test": "COCALC_PROJECT_ID=812abe34-a382-4bd1-9071-29b6f4334f03 COCALC_USERNAME=user NODE_OPTIONS='--experimental-vm-modules' pnpm exec jest", "depcheck": "pnpx depcheck", "prepublishOnly": "pnpm test", "clean": "rm -rf dist" diff --git a/src/packages/sync/editor/db/sync.ts b/src/packages/sync/editor/db/sync.ts index b8db32366c..2ba806511c 100644 --- a/src/packages/sync/editor/db/sync.ts +++ b/src/packages/sync/editor/db/sync.ts @@ -10,6 +10,9 @@ import { Document, DocType } from "../generic/types"; export interface SyncDBOpts0 extends SyncOpts0 { primary_keys: string[]; string_cols?: string[]; + // format = what format to store the underlying file using: json or msgpack + // The default is json unless otherwise specified. + format?: "json" | "msgpack"; } export interface SyncDBOpts extends SyncDBOpts0 { @@ -37,7 +40,7 @@ export class SyncDB extends SyncDoc { super(opts1 as SyncOpts); } - get_one(arg?) : any { + get_one(arg?): any { // I know it is really of type DBDocument. return (this.get_doc() as DBDocument).get_one(arg); } diff --git a/src/packages/sync/editor/generic/util.ts b/src/packages/sync/editor/generic/util.ts index fd666b09a1..2c66ee9729 100644 --- a/src/packages/sync/editor/generic/util.ts +++ b/src/packages/sync/editor/generic/util.ts @@ -68,6 +68,7 @@ export function make_patch(s0: string, s1: string): CompressedPatch { } // apply a compressed patch to a string. +// Returns the result *and* whether or not the patch applied cleanly. export function apply_patch( patch: CompressedPatch, s: string, From dd945012bc53ec0d739bcdb82eb7286615278094 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 22 Jul 2025 00:30:11 +0000 Subject: [PATCH 052/798] delete all the syncstring-based formatter code (just format strings); implement simple approach to file deletion detection --- src/packages/conat/service/formatter.ts | 46 ------------------------- src/packages/project/conat/formatter.ts | 25 -------------- 2 files changed, 71 deletions(-) delete mode 100644 src/packages/conat/service/formatter.ts delete mode 100644 src/packages/project/conat/formatter.ts diff --git a/src/packages/conat/service/formatter.ts b/src/packages/conat/service/formatter.ts deleted file mode 100644 index 4fa5b5a490..0000000000 --- a/src/packages/conat/service/formatter.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* -Formatting services in a project. -*/ - -import { createServiceClient, createServiceHandler } from "./typed"; - -import type { - Options as FormatterOptions, - FormatResult, -} from "@cocalc/util/code-formatter"; - -// TODO: we may change it to NOT take compute server and have this listening from -// project and all compute servers... and have only the one with the file open -// actually reply. -interface FormatterApi { - formatter: (opts: { - path: string; - options: FormatterOptions; - }) => Promise; -} - -export function formatterClient({ compute_server_id = 0, project_id }) { - return createServiceClient({ - project_id, - compute_server_id, - service: "formatter", - }); -} - -export async function createFormatterService({ - compute_server_id = 0, - project_id, - impl, -}: { - project_id: string; - compute_server_id?: number; - impl: FormatterApi; -}) { - return await createServiceHandler({ - project_id, - compute_server_id, - service: "formatter", - description: "Code formatter API", - impl, - }); -} diff --git a/src/packages/project/conat/formatter.ts b/src/packages/project/conat/formatter.ts deleted file mode 100644 index aa593e89bc..0000000000 --- a/src/packages/project/conat/formatter.ts +++ /dev/null @@ -1,25 +0,0 @@ -/* -File formatting service. -*/ - -import { run_formatter, type Options } from "../formatters"; -import { createFormatterService as create } from "@cocalc/conat/service/formatter"; -import { compute_server_id, project_id } from "@cocalc/project/data"; - -interface Message { - path: string; - options: Options; -} - -export async function createFormatterService({ openSyncDocs }) { - const impl = { - formatter: async (opts: Message) => { - const syncstring = openSyncDocs[opts.path]; - if (syncstring == null) { - throw Error(`"${opts.path}" is not opened`); - } - return await run_formatter({ ...opts, syncstring }); - }, - }; - return await create({ compute_server_id, project_id, impl }); -} From fee5a14b58b0e37e4e96088cf4df479b5133f85f Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 22 Jul 2025 00:43:20 +0000 Subject: [PATCH 053/798] surpress an antd react19 warning --- src/packages/static/src/rspack.config.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/packages/static/src/rspack.config.ts b/src/packages/static/src/rspack.config.ts index c79f2282ba..dd973ef7d5 100644 --- a/src/packages/static/src/rspack.config.ts +++ b/src/packages/static/src/rspack.config.ts @@ -168,7 +168,10 @@ export default function getConfig({ middleware }: Options = {}): Configuration { const config: Configuration = { // this makes things 10x slower: //cache: RSPACK_DEV_SERVER || PRODMODE ? false : true, - ignoreWarnings: [/Failed to parse source map/], + ignoreWarnings: [ + /Failed to parse source map/, + /formItemNode = ReactDOM.findDOMNode/, + ], devtool: PRODMODE ? undefined : "eval-cheap-module-source-map", mode: PRODMODE ? ("production" as "production") From 581414681eb2bb126e2d1d3a347ff667ec530de3 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 22 Jul 2025 00:43:38 +0000 Subject: [PATCH 054/798] remove old formatter; implement file delete --- src/packages/conat/project/api/editor.ts | 4 +- .../frame-editors/code-editor/actions.ts | 28 ++++--- .../frontend/frame-editors/generic/client.ts | 26 ------ .../frontend/project/websocket/api.ts | 24 +----- src/packages/project/browser-websocket/api.ts | 8 +- src/packages/project/conat/api/editor.ts | 2 +- src/packages/project/conat/open-files.ts | 12 +-- .../project/formatters/format.test.ts | 2 +- src/packages/project/formatters/index.ts | 79 +------------------ src/packages/sync/editor/generic/sync-doc.ts | 51 ++++++++++++ 10 files changed, 80 insertions(+), 156 deletions(-) diff --git a/src/packages/conat/project/api/editor.ts b/src/packages/conat/project/api/editor.ts index 37e9d27b5a..9db2b45471 100644 --- a/src/packages/conat/project/api/editor.ts +++ b/src/packages/conat/project/api/editor.ts @@ -10,7 +10,7 @@ export const editor = { jupyterRunNotebook: true, jupyterKernelLogo: true, jupyterKernels: true, - formatterString: true, + formatString: true, printSageWS: true, createTerminalService: true, }; @@ -47,7 +47,7 @@ export interface Editor { jupyterKernels: (opts?: { noCache?: boolean }) => Promise; // returns formatted version of str. - formatterString: (opts: { + formatString: (opts: { str: string; options: FormatterOptions; path?: string; // only used for CLANG diff --git a/src/packages/frontend/frame-editors/code-editor/actions.ts b/src/packages/frontend/frame-editors/code-editor/actions.ts index ded1541198..28f025e153 100644 --- a/src/packages/frontend/frame-editors/code-editor/actions.ts +++ b/src/packages/frontend/frame-editors/code-editor/actions.ts @@ -350,6 +350,12 @@ export class Actions< throw Error(`invalid doctype="${this.doctype}"`); } + this._syncstring.once("deleted", () => { + // the file was deleted + this._syncstring.close(); + this._get_project_actions().close_file(this.path); + }); + this._syncstring.once("ready", (err) => { if (this.doctype != "none") { // doctype = 'none' must be handled elsewhere, e.g., terminals. @@ -2287,23 +2293,21 @@ export class Actions< try { this.set_status("Running code formatter..."); - let formatted = await api.editor.formatterString({ + let formatted = await api.editor.formatString({ str, options, path: this.path, }); - if (formatted == str) { - // nothing to do - return; - } - const str2 = cm.getValue(); - if (str2 != str) { - // user made edits *during* formatting, so we "3-way merge" it in, rather - // than breaking what they did: - const patch = make_patch(str, formatted); - formatted = apply_patch(patch, str2)[0]; + if (formatted != str) { + const str2 = cm.getValue(); + if (str2 != str) { + // user made edits *during* formatting, so we "3-way merge" it in, rather + // than breaking what they did: + const patch = make_patch(str, formatted); + formatted = apply_patch(patch, str2)[0]; + } + cm.setValueNoJump(formatted); } - cm.setValueNoJump(formatted); this.setFormatError(""); } catch (err) { this.setFormatError(`${err}`, this._syncstring?.to_str()); diff --git a/src/packages/frontend/frame-editors/generic/client.ts b/src/packages/frontend/frame-editors/generic/client.ts index 67d143f53a..ef26b16dfb 100644 --- a/src/packages/frontend/frame-editors/generic/client.ts +++ b/src/packages/frontend/frame-editors/generic/client.ts @@ -10,9 +10,7 @@ Typescript async/await rewrite of @cocalc/util/client.coffee... import { Map } from "immutable"; import { redux } from "@cocalc/frontend/app-framework"; import { webapp_client } from "@cocalc/frontend/webapp-client"; -import { CompressedPatch } from "@cocalc/sync/editor/generic/types"; import { callback2 } from "@cocalc/util/async-utils"; -import { Config as FormatterConfig } from "@cocalc/util/code-formatter"; import { FakeSyncstring } from "./syncstring-fake"; import { type UserSearchResult as User } from "@cocalc/util/db-schema/accounts"; export { type User }; @@ -154,30 +152,6 @@ export async function write_text_file_to_project( await webapp_client.project_client.write_text_file(opts); } -export async function formatter( - project_id: string, - path: string, - config: FormatterConfig, -): Promise { - const api = await webapp_client.project_client.api(project_id); - const resp = await api.formatter(path, config); - - if (resp.status === "error") { - const loc = resp.error?.loc; - if (loc && loc.start) { - throw Error( - `Syntax error prevented formatting code (possibly on line ${loc.start.line} column ${loc.start.column}) -- fix and run again.`, - ); - } else if (resp.error) { - throw Error(resp.error); - } else { - throw Error("Syntax error prevented formatting code."); - } - } else { - return resp.patch; - } -} - export function log_error(error: string | object): void { webapp_client.tracking_client.log_error(error); } diff --git a/src/packages/frontend/project/websocket/api.ts b/src/packages/frontend/project/websocket/api.ts index 00ed94d1a6..31dced4922 100644 --- a/src/packages/frontend/project/websocket/api.ts +++ b/src/packages/frontend/project/websocket/api.ts @@ -19,7 +19,6 @@ import { import type { Config as FormatterConfig, Options as FormatterOptions, - FormatResult, } from "@cocalc/util/code-formatter"; import { syntax2tool } from "@cocalc/util/code-formatter"; import { DirectoryListingEntry } from "@cocalc/util/types"; @@ -35,7 +34,6 @@ import type { ExecuteCodeOutput, ExecuteCodeOptions, } from "@cocalc/util/types/execute-code"; -import { formatterClient } from "@cocalc/conat/service/formatter"; import { syncFsClientClient } from "@cocalc/conat/service/syncfs-client"; const log = (...args) => { @@ -259,33 +257,15 @@ export class API { } }; - // Returns { status: "ok", patch:... the patch} or - // { status: "error", phase: "format", error: err.message }. - // We return a patch rather than the entire file, since often - // the file is very large, but the formatting is tiny. This is purely - // a data compression technique. - formatter = async ( - path: string, - config: FormatterConfig, - compute_server_id?: number, - ): Promise => { - const options: FormatterOptions = this.check_formatter_available(config); - const client = formatterClient({ - project_id: this.project_id, - compute_server_id: compute_server_id ?? this.getComputeServerId(path), - }); - return await client.formatter({ path, options }); - }; - formatter_string = async ( str: string, config: FormatterConfig, timeout: number = 15000, compute_server_id?: number, - ): Promise => { + ): Promise => { const options: FormatterOptions = this.check_formatter_available(config); const api = this.getApi({ compute_server_id, timeout }); - return await api.editor.formatterString({ str, options }); + return await api.editor.formatString({ str, options }); }; exec = async (opts: ExecuteCodeOptions): Promise => { diff --git a/src/packages/project/browser-websocket/api.ts b/src/packages/project/browser-websocket/api.ts index 8ca6e9387b..edccfdb87f 100644 --- a/src/packages/project/browser-websocket/api.ts +++ b/src/packages/project/browser-websocket/api.ts @@ -16,7 +16,7 @@ import { getClient } from "@cocalc/project/client"; import { get_configuration } from "../configuration"; -import { run_formatter, run_formatter_string } from "../formatters"; +import { formatString } from "../formatters"; import { nbconvert as jupyter_nbconvert } from "../jupyter/convert"; import { jupyter_strip_notebook } from "@cocalc/jupyter/nbgrader/jupyter-parse"; import { jupyter_run_notebook } from "@cocalc/jupyter/nbgrader/jupyter-run"; @@ -131,12 +131,8 @@ export async function handleApiCall({ return await canonical_paths(data.paths); case "configuration": return await get_configuration(data.aspect, data.no_cache); - case "prettier": // deprecated - case "formatter": - return await run_formatter(data); - case "prettier_string": // deprecated case "formatter_string": - return await run_formatter_string(data); + return await formatString(data); case "exec": if (data.opts == null) { throw Error("opts must not be null"); diff --git a/src/packages/project/conat/api/editor.ts b/src/packages/project/conat/api/editor.ts index 32358e614d..c8fde5365a 100644 --- a/src/packages/project/conat/api/editor.ts +++ b/src/packages/project/conat/api/editor.ts @@ -1,7 +1,7 @@ export { jupyter_strip_notebook as jupyterStripNotebook } from "@cocalc/jupyter/nbgrader/jupyter-parse"; export { jupyter_run_notebook as jupyterRunNotebook } from "@cocalc/jupyter/nbgrader/jupyter-run"; export { nbconvert as jupyterNbconvert } from "../../jupyter/convert"; -export { run_formatter_string as formatterString } from "../../formatters"; +export { formatString } from "../../formatters"; export { logo as jupyterKernelLogo } from "@cocalc/jupyter/kernel/logo"; export { get_kernel_data as jupyterKernels } from "@cocalc/jupyter/kernel/kernel-data"; export { newFile } from "@cocalc/backend/misc/new-file"; diff --git a/src/packages/project/conat/open-files.ts b/src/packages/project/conat/open-files.ts index e5114c6993..11dab2fe09 100644 --- a/src/packages/project/conat/open-files.ts +++ b/src/packages/project/conat/open-files.ts @@ -17,7 +17,7 @@ DEBUG_CONSOLE=yes DEBUG=cocalc:debug:project:conat:* node x = await require("@cocalc/project/conat/open-files").init(); Object.keys(x) -[ 'openFiles', 'openDocs', 'formatter', 'terminate', 'computeServers', 'cc' ] +[ 'openFiles', 'openDocs', 'terminate', 'computeServers', 'cc' ] > x.openFiles.getAll(); @@ -67,7 +67,7 @@ doing this! Then: Welcome to Node.js v20.19.0. Type ".help" for more information. > x = await require("@cocalc/project/conat/open-files").init(); Object.keys(x) -[ 'openFiles', 'openDocs', 'formatter', 'terminate', 'computeServers' ] +[ 'openFiles', 'openDocs', 'terminate', 'computeServers' ] > @@ -89,7 +89,6 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { delay } from "awaiting"; import { initJupyterRedux, removeJupyterRedux } from "@cocalc/jupyter/kernel"; import { filename_extension, original_path } from "@cocalc/util/misc"; -import { createFormatterService } from "./formatter"; import { type ConatService } from "@cocalc/conat/service/service"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import { map as awaitMap } from "awaiting"; @@ -124,7 +123,6 @@ const FILE_DELETION_GRACE_PERIOD = 2000; const FILE_DELETION_INITIAL_DELAY = 15000; let openFiles: OpenFiles | null = null; -let formatter: any = null; const openDocs: { [path: string]: SyncDoc | ConatService } = {}; let computeServers: ComputeServerManager | null = null; const openTimes: { [path: string]: number } = {}; @@ -185,13 +183,10 @@ export async function init() { } }); - formatter = await createFormatterService({ openSyncDocs: openDocs }); - // useful for development return { openFiles, openDocs, - formatter, terminate, computeServers, cc: connectToConat(), @@ -206,9 +201,6 @@ export function terminate() { openFiles?.close(); openFiles = null; - formatter?.close(); - formatter = null; - computeServers?.close(); computeServers = null; } diff --git a/src/packages/project/formatters/format.test.ts b/src/packages/project/formatters/format.test.ts index e538da02af..34ce38cca8 100644 --- a/src/packages/project/formatters/format.test.ts +++ b/src/packages/project/formatters/format.test.ts @@ -1,4 +1,4 @@ -import { run_formatter_string as formatString } from "./index"; +import { formatString } from "./index"; describe("format some strings", () => { it("formats markdown with math", async () => { diff --git a/src/packages/project/formatters/index.ts b/src/packages/project/formatters/index.ts index ca2c26434e..62fdcaa16a 100644 --- a/src/packages/project/formatters/index.ts +++ b/src/packages/project/formatters/index.ts @@ -4,16 +4,9 @@ */ /* -Use a formatter like prettier to reformat a syncstring. - -This very nicely use the in-memory node module to prettyify code, by simply modifying the syncstring -on the backend. This avoids having to send the whole file back and forth, worrying about multiple users -and their cursors, file state etc. -- it just merges in the prettification at a point in time. -Also, by doing this on the backend we don't add 5MB (!) to the webpack frontend bundle, to install -something that is not supported on the frontend anyway. +Use a formatter like prettier to format a string of code. */ -import { make_patch } from "@cocalc/sync/editor/generic/util"; import { math_escape, math_unescape } from "@cocalc/util/markdown-utils"; import { filename_extension } from "@cocalc/util/misc"; import { bib_format } from "./bib-format"; @@ -26,90 +19,24 @@ import { r_format } from "./r-format"; import { rust_format } from "./rust-format"; import { xml_format } from "./xml-format"; // mathjax-utils is from upstream project Jupyter -import { once } from "@cocalc/util/async-utils"; import { remove_math, replace_math } from "@cocalc/util/mathjax-utils"; import type { Syntax as FormatterSyntax, Config, Options, - FormatResult, } from "@cocalc/util/code-formatter"; export type { Config, Options, FormatterSyntax }; import { getLogger } from "@cocalc/backend/logger"; -import { getClient } from "@cocalc/project/client"; - -// don't wait too long, since the entire api call likely times out after 5s. -const MAX_WAIT_FOR_SYNC = 3000; const logger = getLogger("project:formatters"); -export async function run_formatter({ - path, - options, - syncstring, -}: { - path: string; - options: Options; - syncstring?; -}): Promise { - const client = getClient(); - // What we do is edit the syncstring with the given path to be "prettier" if possible... - if (syncstring == null) { - syncstring = client.syncdoc({ path }); - } - if (syncstring == null || syncstring.get_state() == "closed") { - return { - status: "error", - error: "document not fully opened", - phase: "format", - }; - } - if (syncstring.get_state() != "ready") { - await once(syncstring, "ready"); - } - if (options.lastChanged) { - // wait within reason until syncstring's last change is this new. - // (It's not a huge problem if this fails for some reason.) - const start = Date.now(); - const waitUntil = new Date(options.lastChanged); - while ( - Date.now() - start < MAX_WAIT_FOR_SYNC && - syncstring.last_changed() < waitUntil - ) { - try { - await once( - syncstring, - "change", - MAX_WAIT_FOR_SYNC - (Date.now() - start), - ); - } catch { - break; - } - } - } - const doc = syncstring.get_doc(); - let formatted, input0; - let input = (input0 = doc.to_str()); - try { - formatted = await run_formatter_string({ path, str: input, options }); - } catch (err) { - logger.debug(`run_formatter error: ${err.message}`); - return { status: "error", phase: "format", error: err.message }; - } - // NOTE: the code used to make the change here on the backend. - // See https://github.com/sagemathinc/cocalc/issues/4335 for why - // that leads to confusion. - const patch = make_patch(input0, formatted); - return { status: "ok", patch }; -} - -export async function run_formatter_string({ +export async function formatString({ options, str, path, }: { str: string; - options: Options; + options: Options; // e.g., {parser:'python'} path?: string; // only used for CLANG }): Promise { let formatted, math; diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 20f9bc1da3..ac91aa7a3b 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -61,6 +61,10 @@ const CURSOR_THROTTLE_NATS_MS = 150; // Ignore file changes for this long after save to disk. const RECENT_SAVE_TO_DISK_MS = 2000; +// If file does not exist for this long, then we close. +const CLOSE_WHEN_DELETED_MS = 2000; +const CLOSE_CHECK_INTERVAL_MS = 500; + const WATCH_DEBOUNCE = 250; const PARALLEL_INIT = true; @@ -3756,6 +3760,9 @@ export class SyncDoc extends EventEmitter { }, 60000); private initInterestLoop = async () => { + if (this.fs != null) { + return; + } if (!this.client.is_browser()) { // only browser clients -- so actual humans return; @@ -3819,6 +3826,8 @@ export class SyncDoc extends EventEmitter { this.fsFileWatcher.close(); // start a new watcher since file descriptor changed this.fsInitFileWatcher(); + // also check if file was deleted, in which case we'll just close + this.fsCloseIfFileDeleted(); return; } } @@ -3830,6 +3839,48 @@ export class SyncDoc extends EventEmitter { this.fsFileWatcher?.close(); delete this.fsFileWatcher; }; + + // returns true if file definitely exists right now, + // false if it definitely does not, and throws exception otherwise, + // e.g., network error. + private fsFileExists = async (): Promise => { + if (this.fs == null) { + throw Error("bug -- fs must be defined"); + } + try { + await this.fs.stat(this.path); + return true; + } catch (err) { + if (err.code == "ENOENT") { + // file not there now. + return false; + } + throw err; + } + }; + + private fsCloseIfFileDeleted = async () => { + if (this.fs == null) { + throw Error("bug -- fs must be defined"); + } + const start = Date.now(); + while (Date.now() - start < CLOSE_WHEN_DELETED_MS) { + try { + if (await this.fsFileExists()) { + // file definitely exists right now. + return; + } + // file definitely does NOT exist right now. + } catch { + // network not working or project off -- no way to know. + return; + } + await delay(CLOSE_CHECK_INTERVAL_MS); + } + // file still doesn't exist -- consider it deleted -- browsers + // should close the tab and possibly notify user. + this.emit("deleted"); + }; } function isCompletePatchStream(dstream) { From 476e9c35bc4ed1ea54ee59b86e562ff84e43358f Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 22 Jul 2025 01:15:09 +0000 Subject: [PATCH 055/798] fix building latex on save --- src/packages/sync/editor/generic/sync-doc.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index ac91aa7a3b..aafea609b9 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -3257,6 +3257,7 @@ export class SyncDoc extends EventEmitter { // to disk. await this.fsFileWatcher?.ignore(2000); if (this.isClosed()) return; + this.last_save_to_disk_time = new Date(); await this.fs.writeFile(this.path, value); this.lastDiskValue = value; }; From dffe85edc0e469ec64a6bd8dea084d48f0de9e8b Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 22 Jul 2025 01:25:44 +0000 Subject: [PATCH 056/798] sync: eliminate non-conate syncdb and syncstring usage in the frontend --- src/packages/frontend/chat/register.ts | 2 +- src/packages/frontend/course/sync.ts | 2 +- src/packages/frontend/frame-editors/generic/client.ts | 7 ------- src/packages/frontend/syncdoc.coffee | 4 ++-- 4 files changed, 4 insertions(+), 11 deletions(-) diff --git a/src/packages/frontend/chat/register.ts b/src/packages/frontend/chat/register.ts index 7ff96d84ca..c28991c167 100644 --- a/src/packages/frontend/chat/register.ts +++ b/src/packages/frontend/chat/register.ts @@ -26,7 +26,7 @@ export function initChat(project_id: string, path: string): ChatActions { redux.getProjectActions(project_id)?.setNotDeleted(path); } - const syncdb = webapp_client.sync_client.sync_db({ + const syncdb = webapp_client.conat_client.conat().sync.db({ project_id, path, primary_keys: ["date", "sender_id", "event"], diff --git a/src/packages/frontend/course/sync.ts b/src/packages/frontend/course/sync.ts index 69a470e1f8..6fbfe3df11 100644 --- a/src/packages/frontend/course/sync.ts +++ b/src/packages/frontend/course/sync.ts @@ -31,7 +31,7 @@ export function create_sync_db( const path = store.get("course_filename"); actions.setState({ loading: true }); - const syncdb = webapp_client.sync_client.sync_db({ + const syncdb = webapp_client.conat_client.conat().sync.db({ project_id, path, primary_keys: ["table", "handout_id", "student_id", "assignment_id"], diff --git a/src/packages/frontend/frame-editors/generic/client.ts b/src/packages/frontend/frame-editors/generic/client.ts index ef26b16dfb..33a8159f04 100644 --- a/src/packages/frontend/frame-editors/generic/client.ts +++ b/src/packages/frontend/frame-editors/generic/client.ts @@ -194,9 +194,6 @@ interface SyncstringOpts2 { export function syncstring2(opts: SyncstringOpts2): SyncString { return webapp_client.conat_client.conat().sync.string(opts); - // const opts1: any = opts; - // opts1.client = webapp_client; - // return webapp_client.sync_client.sync_string(opts1); } export interface SyncDBOpts { @@ -215,9 +212,6 @@ export interface SyncDBOpts { export function syncdb(opts: SyncDBOpts): any { return webapp_client.conat_client.conat().sync.db(opts); - - // const opts1: any = opts; - // return webapp_client.sync_db(opts1); } import type { SyncDB } from "@cocalc/sync/editor/db/sync"; @@ -229,7 +223,6 @@ export function syncdb2(opts: SyncDBOpts): SyncDB { const opts1: any = opts; opts1.client = webapp_client; return webapp_client.conat_client.conat().sync.db(opts1); - // return webapp_client.sync_client.sync_db(opts1); } interface QueryOpts { diff --git a/src/packages/frontend/syncdoc.coffee b/src/packages/frontend/syncdoc.coffee index ae8ab1995c..535a38c5c6 100644 --- a/src/packages/frontend/syncdoc.coffee +++ b/src/packages/frontend/syncdoc.coffee @@ -66,7 +66,7 @@ class SynchronizedString extends AbstractSynchronizedDoc @project_id = @opts.project_id @filename = @opts.filename @connect = @_connect - @_syncstring = webapp_client.sync_client.sync_string + @_syncstring = webapp_client.conat_client.conat().sync.string project_id : @project_id path : @filename cursors : opts.cursors @@ -209,7 +209,7 @@ class SynchronizedDocument2 extends SynchronizedDocument @filename = '.smc/root' + @filename id = require('@cocalc/util/schema').client_db.sha1(@project_id, @filename) - @_syncstring = webapp_client.sync_client.sync_string + @_syncstring = webapp_client.conat_client.conat().sync.string id : id project_id : @project_id path : @filename From 91b79d3469b7a03d33e276fa9e997d452b43ab35 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 22 Jul 2025 14:26:48 +0000 Subject: [PATCH 057/798] sync-doc: add unit test of file deletion --- .../conat/test/sync-doc/delete.test.ts | 53 +++++++++++++++ src/packages/conat/files/watch.ts | 9 ++- src/packages/conat/socket/client.ts | 2 +- src/packages/conat/sync-doc/sync-client.ts | 7 ++ src/packages/conat/sync-doc/syncdb.ts | 2 +- src/packages/conat/sync-doc/syncstring.ts | 9 ++- src/packages/sync/editor/generic/sync-doc.ts | 65 +++++++++++++++---- src/packages/util/async-utils.ts | 14 ++++ 8 files changed, 141 insertions(+), 20 deletions(-) create mode 100644 src/packages/backend/conat/test/sync-doc/delete.test.ts diff --git a/src/packages/backend/conat/test/sync-doc/delete.test.ts b/src/packages/backend/conat/test/sync-doc/delete.test.ts new file mode 100644 index 0000000000..c34b57d0b0 --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/delete.test.ts @@ -0,0 +1,53 @@ +import { + before, + after, + uuid, + connect, + server, + once, + delay, + waitUntilSynced, +} from "./setup"; + +beforeAll(before); +afterAll(after); + +describe("deleting a file that is open as a syncdoc", () => { + const project_id = uuid(); + const path = "a.txt"; + let client1, client2, s1, s2, fs; + const deletedThreshold = 50; // make test faster + + it("creates two clients editing 'a.txt'", async () => { + client1 = connect(); + client2 = connect(); + fs = client1.fs({ project_id, service: server.service }); + await fs.writeFile(path, "my existing file"); + s1 = client1.sync.string({ + project_id, + path, + fs, + deletedThreshold, + }); + + await once(s1, "ready"); + + s2 = client2.sync.string({ + project_id, + path, + service: server.service, + deletedThreshold, + }); + await once(s2, "ready"); + }); + + it("delete 'a.txt' from disk and both clients emit 'deleted' event in about ${deletedThreshold}ms", async () => { + const start = Date.now(); + const d1 = once(s1, "deleted"); + const d2 = once(s2, "deleted"); + await fs.unlink("a.txt"); + await d1; + await d2; + expect(Date.now() - start).toBeLessThan(deletedThreshold + 1000); + }); +}); diff --git a/src/packages/conat/files/watch.ts b/src/packages/conat/files/watch.ts index 531dc82d9f..fea473a8a8 100644 --- a/src/packages/conat/files/watch.ts +++ b/src/packages/conat/files/watch.ts @@ -8,7 +8,6 @@ import { type ServerSocket, } from "@cocalc/conat/socket"; import { EventIterator } from "@cocalc/util/event-iterator"; - import { getLogger } from "@cocalc/conat/client"; const logger = getLogger("conat:files:watch"); @@ -165,7 +164,8 @@ export async function watchClient({ path: string; options?: WatchOptions; }): Promise { - const socket = await client.socket.connect(subject); + await client.waitForInterest(subject); + const socket = client.socket.connect(subject); const iter = new EventIterator(socket, "data", { map: (args) => args[0], onEnd: () => { @@ -176,7 +176,10 @@ export async function watchClient({ iter.end(); }); // tell it what to watch - await socket.request({ path, options }); + await socket.request({ + path, + options, + }); const iter2 = iter as WatchIterator; diff --git a/src/packages/conat/socket/client.ts b/src/packages/conat/socket/client.ts index bb107bdaeb..db8598a0e2 100644 --- a/src/packages/conat/socket/client.ts +++ b/src/packages/conat/socket/client.ts @@ -225,7 +225,7 @@ export class ConatSocketClient extends ConatSocketBase { if (this.state == "closed") { throw Error("closed"); } - // console.log("sending request from client ", { subject, data, options }); + //console.log("sending request from client ", { subject, data, options }); return await this.client.request(subject, data, options); }; diff --git a/src/packages/conat/sync-doc/sync-client.ts b/src/packages/conat/sync-doc/sync-client.ts index 9609dce5a3..6c284b2421 100644 --- a/src/packages/conat/sync-doc/sync-client.ts +++ b/src/packages/conat/sync-doc/sync-client.ts @@ -18,8 +18,15 @@ export class SyncClient extends EventEmitter implements Client0 { throw Error("client must be specified"); } this.client = client; + this.client.once("closed", this.close); } + close = () => { + this.emit("closed"); + // @ts-ignore + delete this.client; + }; + is_project = (): boolean => false; is_browser = (): boolean => true; is_compute_server = (): boolean => false; diff --git a/src/packages/conat/sync-doc/syncdb.ts b/src/packages/conat/sync-doc/syncdb.ts index a4437d7459..b67add9dea 100644 --- a/src/packages/conat/sync-doc/syncdb.ts +++ b/src/packages/conat/sync-doc/syncdb.ts @@ -11,7 +11,7 @@ export interface SyncDBOptions extends Omit { export type { SyncDB }; export function syncdb({ client, service, ...opts }: SyncDBOptions): SyncDB { - const fs = client.fs({ service, project_id: opts.project_id }); + const fs = opts.fs ?? client.fs({ service, project_id: opts.project_id }); const syncClient = new SyncClient(client); return new SyncDB({ ...opts, fs, client: syncClient }); } diff --git a/src/packages/conat/sync-doc/syncstring.ts b/src/packages/conat/sync-doc/syncstring.ts index 5fb6d2e2fb..37cf1185af 100644 --- a/src/packages/conat/sync-doc/syncstring.ts +++ b/src/packages/conat/sync-doc/syncstring.ts @@ -13,9 +13,12 @@ export interface SyncStringOptions extends Omit { export type { SyncString }; -export function syncstring({ client, service, ...opts }: SyncStringOptions): SyncString { - const fs = client.fs({ service, project_id: opts.project_id }); +export function syncstring({ + client, + service, + ...opts +}: SyncStringOptions): SyncString { + const fs = opts.fs ?? client.fs({ service, project_id: opts.project_id }); const syncClient = new SyncClient(client); return new SyncString({ ...opts, fs, client: syncClient }); } - diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index aafea609b9..3063ce80a9 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -61,9 +61,9 @@ const CURSOR_THROTTLE_NATS_MS = 150; // Ignore file changes for this long after save to disk. const RECENT_SAVE_TO_DISK_MS = 2000; -// If file does not exist for this long, then we close. -const CLOSE_WHEN_DELETED_MS = 2000; -const CLOSE_CHECK_INTERVAL_MS = 500; +// If file does not exist for this long, then syncdoc emits a 'deleted' event. +const DELETED_THRESHOLD = 2000; +const DELETED_CHECK_INTERVAL = 750; const WATCH_DEBOUNCE = 250; @@ -165,6 +165,10 @@ export interface SyncOpts0 { // if true, do not implicitly save on commit. This is very // useful for unit testing to easily simulate offline state. noAutosave?: boolean; + + // optional timeout for how long to wait from when a file is + // deleted until emiting a 'deleted' event. + deletedThreshold?: number; } export interface SyncOpts extends SyncOpts0 { @@ -282,8 +286,11 @@ export class SyncDoc extends EventEmitter { private noAutosave?: boolean; + private deletedThreshold?: number; + constructor(opts: SyncOpts) { super(); + if (opts.string_id === undefined) { this.string_id = schema.client_db.sha1(opts.project_id, opts.path); } else { @@ -305,12 +312,15 @@ export class SyncDoc extends EventEmitter { "ephemeral", "fs", "noAutosave", + "deletedThreshold", ]) { if (opts[field] != undefined) { this[field] = opts[field]; } } + this.client.once("closed", this.close); + this.legacy = new LegacyHistory({ project_id: this.project_id, path: this.path, @@ -395,6 +405,7 @@ export class SyncDoc extends EventEmitter { }, { start: 3000, max: 15000, decay: 1.3 }, ); + if (this.isClosed()) return; // Success -- everything initialized with no issues. this.set_state("ready"); @@ -3813,9 +3824,16 @@ export class SyncDoc extends EventEmitter { } // console.log("watching for changes"); // use this.fs interface to watch path for changes. - this.fsFileWatcher = await this.fs.watch(this.path, { unique: true }); + try { + this.fsFileWatcher = await this.fs.watch(this.path, { unique: true }); + } catch (err) { + if (this.isClosed()) return; + throw err; + } + if (this.isClosed()) return; (async () => { for await (const { eventType, ignore } of this.fsFileWatcher) { + if (this.isClosed()) return; // we don't know what's on disk anymore, this.lastDiskValue = undefined; //console.log("got change", eventType); @@ -3823,12 +3841,28 @@ export class SyncDoc extends EventEmitter { this.fsLoadFromDiskDebounced(); } if (eventType == "rename") { + // check if file was deleted + this.fsCloseIfFileDeleted(); // always have to recreate in case of a rename this.fsFileWatcher.close(); // start a new watcher since file descriptor changed - this.fsInitFileWatcher(); - // also check if file was deleted, in which case we'll just close - this.fsCloseIfFileDeleted(); + await until( + async () => { + if (this.isClosed()) return true; + try { + await this.fsInitFileWatcher(); + return true; + } catch (err) { + // console.warn( + // "sync-doc WARNING: issue creating file watch", + // this.path, + // err, + // ); + return false; + } + }, + { min: 3000 }, + ); return; } } @@ -3865,7 +3899,8 @@ export class SyncDoc extends EventEmitter { throw Error("bug -- fs must be defined"); } const start = Date.now(); - while (Date.now() - start < CLOSE_WHEN_DELETED_MS) { + const threshold = this.deletedThreshold ?? DELETED_THRESHOLD; + while (true) { try { if (await this.fsFileExists()) { // file definitely exists right now. @@ -3876,11 +3911,17 @@ export class SyncDoc extends EventEmitter { // network not working or project off -- no way to know. return; } - await delay(CLOSE_CHECK_INTERVAL_MS); + const elapsed = Date.now() - start; + if (elapsed > threshold) { + // out of time to appear again, and definitely concluded + // it does not exist above + // file still doesn't exist -- consider it deleted -- browsers + // should close the tab and possibly notify user. + this.emit("deleted"); + return; + } + await delay(Math.min(DELETED_CHECK_INTERVAL, threshold - elapsed)); } - // file still doesn't exist -- consider it deleted -- browsers - // should close the tab and possibly notify user. - this.emit("deleted"); }; } diff --git a/src/packages/util/async-utils.ts b/src/packages/util/async-utils.ts index f0fed46ef4..ac40a124e1 100644 --- a/src/packages/util/async-utils.ts +++ b/src/packages/util/async-utils.ts @@ -168,6 +168,12 @@ export class TimeoutError extends Error { } } +function captureStackWithoutPrinting() { + const obj = {} as any; + Error.captureStackTrace(obj, captureStackWithoutPrinting); + return obj.stack; +} + /* Wait for an event emitter to emit any event at all once. Returns array of args emitted by that event. If timeout_ms is 0 (the default) this can wait an unbounded @@ -178,12 +184,14 @@ export class TimeoutError extends Error { If the obj throws 'closed' before the event is emitted, then this throws an error, since clearly event can never be emitted. */ +const DEBUG_ONCE = false; // log a better stack trace in some cases export async function once( obj: EventEmitter, event: string, timeout_ms: number | undefined = 0, ): Promise { if (obj == null) throw Error("once -- obj is undefined"); + const stack = DEBUG_ONCE ? captureStackWithoutPrinting() : undefined; if (timeout_ms == null) { // clients might explicitly pass in undefined, but below we expect 0 to mean "no timeout" timeout_ms = 0; @@ -207,11 +215,17 @@ export async function once( function onClosed() { cleanup(); + if (DEBUG_ONCE) { + console.log(stack); + } reject(new TimeoutError(`once: "${event}" not emitted before "closed"`)); } function onTimeout() { cleanup(); + if (DEBUG_ONCE) { + console.log(stack); + } reject( new TimeoutError( `once: timeout of ${timeout_ms}ms waiting for "${event}"`, From eac0e2fda4f734a7bcf89349b182048571dd8a8a Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 22 Jul 2025 15:28:54 +0000 Subject: [PATCH 058/798] fix broken test and some ts --- .../backend/conat/test/sync-doc/delete.test.ts | 11 +---------- src/packages/conat/files/watch.ts | 1 - 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/src/packages/backend/conat/test/sync-doc/delete.test.ts b/src/packages/backend/conat/test/sync-doc/delete.test.ts index c34b57d0b0..823fba9932 100644 --- a/src/packages/backend/conat/test/sync-doc/delete.test.ts +++ b/src/packages/backend/conat/test/sync-doc/delete.test.ts @@ -1,13 +1,4 @@ -import { - before, - after, - uuid, - connect, - server, - once, - delay, - waitUntilSynced, -} from "./setup"; +import { before, after, uuid, connect, server, once } from "./setup"; beforeAll(before); afterAll(after); diff --git a/src/packages/conat/files/watch.ts b/src/packages/conat/files/watch.ts index fea473a8a8..0e9027b33b 100644 --- a/src/packages/conat/files/watch.ts +++ b/src/packages/conat/files/watch.ts @@ -164,7 +164,6 @@ export async function watchClient({ path: string; options?: WatchOptions; }): Promise { - await client.waitForInterest(subject); const socket = client.socket.connect(subject); const iter = new EventIterator(socket, "data", { map: (args) => args[0], From 0120c3ec798a251761653831cb2821210320ca1d Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 22 Jul 2025 17:53:55 +0000 Subject: [PATCH 059/798] sync-doc: improving behavior based on unit testing --- .../backend/conat/files/test/watch.test.ts | 16 +- .../conat/test/sync-doc/conflict.test.ts | 2 +- .../conat/test/sync-doc/delete.test.ts | 79 ++++- .../conat/test/sync-doc/watch-file.test.ts | 4 +- src/packages/conat/core/client.ts | 7 + src/packages/conat/files/watch.ts | 11 +- src/packages/frontend/conat/client.ts | 2 +- src/packages/sync/editor/generic/sync-doc.ts | 320 ++++-------------- 8 files changed, 176 insertions(+), 265 deletions(-) diff --git a/src/packages/backend/conat/files/test/watch.test.ts b/src/packages/backend/conat/files/test/watch.test.ts index 0c3e61c376..3cefad5843 100644 --- a/src/packages/backend/conat/files/test/watch.test.ts +++ b/src/packages/backend/conat/files/test/watch.test.ts @@ -34,7 +34,7 @@ describe("basic core of the async path watch functionality", () => { }); let w; - it("create a watcher client", async () => { + it("create a watcher client for 'a.txt'", async () => { w = await watchClient({ client, subject: "foo", path: "a.txt" }); }); @@ -58,4 +58,18 @@ describe("basic core of the async path watch functionality", () => { await wait({ until: () => Object.keys(server.sockets).length == 0 }); expect(Object.keys(server.sockets).length).toEqual(0); }); + + it("trying to watch file that does not exist throws error", async () => { + await expect(async () => { + await watchClient({ client, subject: "foo", path: "b.txt" }); + }).rejects.toThrow( + "Error: ENOENT: no such file or directory, watch 'b.txt'", + ); + + try { + await watchClient({ client, subject: "foo", path: "b.txt" }); + } catch (err) { + expect(err.code).toEqual("ENOENT"); + } + }); }); diff --git a/src/packages/backend/conat/test/sync-doc/conflict.test.ts b/src/packages/backend/conat/test/sync-doc/conflict.test.ts index 34797cc53e..739b33da7e 100644 --- a/src/packages/backend/conat/test/sync-doc/conflict.test.ts +++ b/src/packages/backend/conat/test/sync-doc/conflict.test.ts @@ -215,7 +215,7 @@ describe("do the example in the blog post 'Lies I was Told About Collaborative E }); const numHeads = 15; -describe.only(`create editing conflict with ${numHeads} heads`, () => { +describe(`create editing conflict with ${numHeads} heads`, () => { const project_id = uuid(); let docs: any[] = [], clients: any[] = []; diff --git a/src/packages/backend/conat/test/sync-doc/delete.test.ts b/src/packages/backend/conat/test/sync-doc/delete.test.ts index 823fba9932..0d1854be74 100644 --- a/src/packages/backend/conat/test/sync-doc/delete.test.ts +++ b/src/packages/backend/conat/test/sync-doc/delete.test.ts @@ -1,4 +1,4 @@ -import { before, after, uuid, connect, server, once } from "./setup"; +import { before, after, uuid, connect, server, once, delay } from "./setup"; beforeAll(before); afterAll(after); @@ -8,6 +8,7 @@ describe("deleting a file that is open as a syncdoc", () => { const path = "a.txt"; let client1, client2, s1, s2, fs; const deletedThreshold = 50; // make test faster + const watchRecreateWait = 100; it("creates two clients editing 'a.txt'", async () => { client1 = connect(); @@ -19,6 +20,7 @@ describe("deleting a file that is open as a syncdoc", () => { path, fs, deletedThreshold, + watchRecreateWait, }); await once(s1, "ready"); @@ -28,11 +30,12 @@ describe("deleting a file that is open as a syncdoc", () => { path, service: server.service, deletedThreshold, + watchRecreateWait, }); await once(s2, "ready"); }); - it("delete 'a.txt' from disk and both clients emit 'deleted' event in about ${deletedThreshold}ms", async () => { + it(`delete 'a.txt' from disk and both clients emit 'deleted' event in about ${deletedThreshold}ms`, async () => { const start = Date.now(); const d1 = once(s1, "deleted"); const d2 = once(s2, "deleted"); @@ -41,4 +44,76 @@ describe("deleting a file that is open as a syncdoc", () => { await d2; expect(Date.now() - start).toBeLessThan(deletedThreshold + 1000); }); + + it("clients still work (clients can ignore 'deleted' if they want)", async () => { + expect(s1.isClosed()).toBe(false); + expect(s2.isClosed()).toBe(false); + s1.from_str("back"); + const d1 = once(s1, "watching"); + const d2 = once(s2, "watching"); + await s1.save_to_disk(); + expect(await fs.readFile("a.txt", "utf8")).toEqual("back"); + await d1; + await d2; + }); + + it(`deleting 'a.txt' again -- still triggers deleted events`, async () => { + const start = Date.now(); + const d1 = once(s1, "deleted"); + const d2 = once(s2, "deleted"); + await fs.unlink("a.txt"); + await d1; + await d2; + expect(Date.now() - start).toBeLessThan(deletedThreshold + 1000); + }); + + // it("disconnect one client, delete file, then reconnect client", async () => { + // console.log(1); + // client2.disconnect(); + // const d1 = once(s1, "deleted"); + // const d2 = once(s2, "deleted"); + // console.log(2); + // await fs.unlink("a.txt"); + // console.log(3); + // client2.connect(); + // console.log(4); + // await d1; + // console.log(5); + // await d2; + // expect(Date.now() - start).toBeLessThan(deletedThreshold + 1000); + // }); +}); + +describe("deleting a file then recreate it quickly does NOT trigger a 'deleted' event", () => { + const project_id = uuid(); + const path = "a.txt"; + let client1, s1, fs; + const deletedThreshold = 250; + + it("creates two clients editing 'a.txt'", async () => { + client1 = connect(); + fs = client1.fs({ project_id, service: server.service }); + await fs.writeFile(path, "my existing file"); + s1 = client1.sync.string({ + project_id, + path, + fs, + service: server.service, + deletedThreshold, + }); + + await once(s1, "ready"); + }); + + it(`delete 'a.txt' from disk and both clients emit 'deleted' event in about ${deletedThreshold}ms`, async () => { + let c1 = 0; + s1.once("deleted", () => { + c1++; + }); + await fs.unlink("a.txt"); + await delay(deletedThreshold - 100); + await fs.writeFile(path, "I'm back!"); + await delay(deletedThreshold); + expect(c1).toBe(0); + }); }); diff --git a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts index e1f3cb830f..85d11a5d1f 100644 --- a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts +++ b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts @@ -121,11 +121,11 @@ describe("basic watching of file on disk happens automatically", () => { }); }); -describe.only("has unsaved changes", () => { +describe("has unsaved changes", () => { const project_id = uuid(); let s1, s2, client1, client2; - it("creates two clients", async () => { + it("creates two clients and opens a new file (does not exist on disk yet)", async () => { client1 = connect(); client2 = connect(); s1 = client1.sync.string({ diff --git a/src/packages/conat/core/client.ts b/src/packages/conat/core/client.ts index 58bc516679..e11b2ec0a7 100644 --- a/src/packages/conat/core/client.ts +++ b/src/packages/conat/core/client.ts @@ -573,6 +573,10 @@ export class Client extends EventEmitter { setTimeout(() => this.conn.io.disconnect(), 1); }; + connect = () => { + this.conn.io.connect(); + }; + isConnected = () => this.state == "connected"; isSignedIn = () => !!(this.info?.user && !this.info?.user?.error); @@ -1992,5 +1996,8 @@ export function headerToError(headers) { err[field] = headers.error_attrs[field]; } } + if (err['code'] === undefined && headers.code) { + err['code'] = headers.code; + } return err; } diff --git a/src/packages/conat/files/watch.ts b/src/packages/conat/files/watch.ts index 0e9027b33b..4c1beade5b 100644 --- a/src/packages/conat/files/watch.ts +++ b/src/packages/conat/files/watch.ts @@ -2,7 +2,10 @@ Remotely proxying a fs.watch AsyncIterator over a Conat Socket. */ -import { type Client as ConatClient } from "@cocalc/conat/core/client"; +import { + type Client as ConatClient, + headerToError, +} from "@cocalc/conat/core/client"; import { type ConatSocketServer, type ServerSocket, @@ -173,12 +176,16 @@ export async function watchClient({ }); socket.on("closed", () => { iter.end(); + delete iter2.ignore; }); // tell it what to watch - await socket.request({ + const resp = await socket.request({ path, options, }); + if (resp.headers?.error) { + throw headerToError(resp.headers); + } const iter2 = iter as WatchIterator; diff --git a/src/packages/frontend/conat/client.ts b/src/packages/frontend/conat/client.ts index dedb2d4533..674aa05ac5 100644 --- a/src/packages/frontend/conat/client.ts +++ b/src/packages/frontend/conat/client.ts @@ -262,7 +262,7 @@ export class ConatClient extends EventEmitter { console.log( `Connecting to ${this._conatClient?.options.address}: attempts ${attempts}`, ); - this._conatClient?.conn.io.connect(); + this._conatClient?.connect(); return false; }, { min: 3000, max: 15000 }, diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 3063ce80a9..73cf96966a 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -64,11 +64,10 @@ const RECENT_SAVE_TO_DISK_MS = 2000; // If file does not exist for this long, then syncdoc emits a 'deleted' event. const DELETED_THRESHOLD = 2000; const DELETED_CHECK_INTERVAL = 750; +const WATCH_RECREATE_WAIT = 3000; const WATCH_DEBOUNCE = 250; -const PARALLEL_INIT = true; - import { COMPUTE_THRESH_MS, COMPUTER_SERVER_CURSOR_TYPE, @@ -169,6 +168,10 @@ export interface SyncOpts0 { // optional timeout for how long to wait from when a file is // deleted until emiting a 'deleted' event. deletedThreshold?: number; + // how long to wait before trying to recreate a watch -- this mainly + // matters in cases when the file is deleted and the client ignores + // the 'deleted' event. + watchRecreateWait?: number; } export interface SyncOpts extends SyncOpts0 { @@ -287,6 +290,7 @@ export class SyncDoc extends EventEmitter { private noAutosave?: boolean; private deletedThreshold?: number; + private watchRecreateWait?: number; constructor(opts: SyncOpts) { super(); @@ -313,6 +317,7 @@ export class SyncDoc extends EventEmitter { "fs", "noAutosave", "deletedThreshold", + "watchRecreateWait", ]) { if (opts[field] != undefined) { this[field] = opts[field]; @@ -1037,26 +1042,6 @@ export class SyncDoc extends EventEmitter { return t; }; - /* The project calls set_initialized once it has checked for - the file on disk; this way the frontend knows that the - syncstring has been initialized in the database, and also - if there was an error doing the check. - */ - private set_initialized = async ( - error: string, - read_only: boolean, - size: number, - ): Promise => { - this.assert_table_is_ready("syncstring"); - this.dbg("set_initialized")({ error, read_only, size }); - const init = { time: this.client.server_time(), size, error }; - await this.set_syncstring_table({ - init, - read_only, - last_active: this.client.server_time(), - }); - }; - /* List of logical timestamps of the versions of this string in the sync table that we opened to start editing (so starts with what was the most recent snapshot when we started). The list of timestamps @@ -1502,153 +1487,32 @@ export class SyncDoc extends EventEmitter { this.assert_not_closed( "initAll -- before init patch_list, cursors, evaluator, ipywidgets", ); - if (PARALLEL_INIT) { - await Promise.all([ - this.init_patch_list(), - this.init_cursors(), - this.init_evaluator(), - this.init_ipywidgets(), - this.initFileWatcher(), - ]); - this.assert_not_closed( - "initAll -- successful init patch_list, cursors, evaluator, and ipywidgets", - ); - } else { - await this.init_patch_list(); - this.assert_not_closed("initAll -- successful init_patch_list"); - await this.init_cursors(); - this.assert_not_closed("initAll -- successful init_patch_cursors"); - await this.init_evaluator(); - this.assert_not_closed("initAll -- successful init_evaluator"); - await this.init_ipywidgets(); - this.assert_not_closed("initAll -- successful init_ipywidgets"); - } + await Promise.all([ + this.init_patch_list(), + this.init_cursors(), + this.init_evaluator(), + this.init_ipywidgets(), + this.initFileWatcher(), + ]); + this.assert_not_closed( + "initAll -- successful init patch_list, cursors, evaluator, and ipywidgets", + ); this.init_table_close_handlers(); this.assert_not_closed("initAll -- successful init_table_close_handlers"); log("file_use_interval"); this.init_file_use_interval(); - if (this.fs != null) { - await this.fsLoadFromDisk(); - } else { - if (await this.isFileServer()) { - log("load_from_disk"); - // This sets initialized, which is needed to be fully ready. - // We keep trying this load from disk until sync-doc is closed - // or it succeeds. It may fail if, e.g., the file is too - // large or is not readable by the user. They are informed to - // fix the problem... and once they do (and wait up to 10s), - // this will finish. - // if (!this.client.is_browser() && !this.client.is_project()) { - // // FAKE DELAY!!! Just to simulate flakiness / slow network!!!! - // await delay(3000); - // } - await retry_until_success({ - f: this.init_load_from_disk, - max_delay: 10000, - desc: "syncdoc -- load_from_disk", - }); - log("done loading from disk"); - } else { - if (this.patch_list!.count() == 0) { - await Promise.race([ - this.waitUntilFullyReady(), - once(this.patch_list!, "change"), - ]); - } - } - } - this.assert_not_closed("initAll -- load from disk"); - this.emit("init"); + await this.fsLoadFromDiskIfNewer(); + + this.emit("init"); this.assert_not_closed("initAll -- after waiting until fully ready"); - if (await this.isFileServer()) { - log("init file autosave"); - this.init_file_autosave(); - } this.update_has_unsaved_changes(); log("done"); }; - private init_error = (): string | undefined => { - let x; - try { - x = this.syncstring_table.get_one(); - } catch (_err) { - // if the table hasn't been initialized yet, - // it can't be in error state. - return undefined; - } - return x?.get("init")?.get("error"); - }; - - // wait until the syncstring table is ready to be - // used (so extracted from archive, etc.), - private waitUntilFullyReady = async (): Promise => { - this.assert_not_closed("wait_until_fully_ready"); - const dbg = this.dbg("wait_until_fully_ready"); - dbg(); - - if (this.client.is_browser() && this.init_error()) { - // init is set and is in error state. Give the backend a few seconds - // to try to fix this error before giving up. The browser client - // can close and open the file to retry this (as instructed). - try { - await this.syncstring_table.wait(() => !this.init_error(), 5); - } catch (err) { - // fine -- let the code below deal with this problem... - } - } - - let init; - const is_init = (t: SyncTable) => { - this.assert_not_closed("is_init"); - const tbl = t.get_one(); - if (tbl == null) { - dbg("null"); - return false; - } - init = tbl.get("init")?.toJS(); - return init != null; - }; - dbg("waiting for init..."); - await this.syncstring_table.wait(is_init, 0); - dbg("init done"); - if (init.error) { - throw Error(init.error); - } - assertDefined(this.patch_list); - if (init.size == null) { - // don't crash but warn at least. - console.warn("SYNC BUG -- init.size must be defined", { init }); - } - if ( - !this.client.is_project() && - this.patch_list.count() === 0 && - init.size - ) { - dbg("waiting for patches for nontrivial file"); - // normally this only happens in a later event loop, - // so force it now. - dbg("handling patch update queue since", this.patch_list.count()); - await this.handle_patch_update_queue(true); - assertDefined(this.patch_list); - dbg("done handling, now ", this.patch_list.count()); - if (this.patch_list.count() === 0) { - // wait for a change -- i.e., project loading the file from - // disk and making available... Because init.size > 0, we know that - // there must be SOMETHING in the patches table once initialization is done. - // This is the root cause of https://github.com/sagemathinc/cocalc/issues/2382 - await once(this.patches_table, "change"); - dbg("got patches_table change"); - await this.handle_patch_update_queue(true); - dbg("handled update queue"); - } - } - }; - private assert_table_is_ready = (table: string): void => { const t = this[table + "_table"]; // not using string template only because it breaks codemirror! if (t == null || t.get_state() != "connected") { @@ -1780,17 +1644,6 @@ export class SyncDoc extends EventEmitter { this.set_read_only(read_only); }; - private init_load_from_disk = async (): Promise => { - if (this.state == "closed") { - // stop trying, no error -- this is assumed - // in a retry_until_success elsewhere. - return; - } - if (await this.load_from_disk_if_newer()) { - throw Error("failed to load from disk"); - } - }; - private fsLoadFromDiskIfNewer = async (): Promise => { // [ ] TODO: readonly handling... if (this.fs == null) throw Error("bug"); @@ -1828,54 +1681,6 @@ export class SyncDoc extends EventEmitter { return false; }; - private load_from_disk_if_newer = async (): Promise => { - if (this.fs != null) { - return await this.fsLoadFromDiskIfNewer(); - } - if (this.client.path_exists == null) { - throw Error("legacy clients must define path_exists"); - } - const last_changed = new Date(this.last_changed()); - const firstLoad = this.versions().length == 0; - const dbg = this.dbg("load_from_disk_if_newer"); - let is_read_only: boolean = false; - let size: number = 0; - let error: string = ""; - try { - dbg("check if path exists"); - if (await callback2(this.client.path_exists, { path: this.path })) { - // the path exists - dbg("path exists -- stat file"); - const stats = await callback2(this.client.path_stat, { - path: this.path, - }); - if (firstLoad || stats.ctime > last_changed) { - dbg( - `disk file changed more recently than edits (or first load), so loading, ${stats.ctime} > ${last_changed}; firstLoad=${firstLoad}`, - ); - size = await this.readFile(); - if (firstLoad) { - dbg("emitting first-load event"); - // this event is emited the first time the document is ever loaded from disk. - this.emit("first-load"); - } - dbg("loaded"); - } else { - dbg("stick with database version"); - } - dbg("checking if read only"); - is_read_only = await this.file_is_read_only(); - dbg("read_only", is_read_only); - } - } catch (err) { - error = `${err}`; - } - - await this.set_initialized(error, is_read_only, size); - dbg("done"); - return !!error; - }; - private patch_table_query = (cutoff?: number) => { const query = { string_id: this.string_id, @@ -3266,7 +3071,12 @@ export class SyncDoc extends EventEmitter { // tell watcher not to fire any change events for a little time, // so no clients waste resources loading in response to us saving // to disk. - await this.fsFileWatcher?.ignore(2000); + try { + await this.fsFileWatcher?.ignore(2000); + } catch { + // not a big problem if we can't ignore (e.g., this happens potentially + // after deleting the file or if file doesn't exist) + } if (this.isClosed()) return; this.last_save_to_disk_time = new Date(); await this.fs.writeFile(this.path, value); @@ -3822,51 +3632,48 @@ export class SyncDoc extends EventEmitter { if (this.fs == null) { throw Error("this.fs must be defined"); } - // console.log("watching for changes"); - // use this.fs interface to watch path for changes. + // use this.fs interface to watch path for changes -- we try once: try { this.fsFileWatcher = await this.fs.watch(this.path, { unique: true }); - } catch (err) { - if (this.isClosed()) return; - throw err; - } + } catch {} if (this.isClosed()) return; + + // not closed -- so if above succeeds we start watching. + // if not, we loop waiting for file to be created so we can watch it (async () => { - for await (const { eventType, ignore } of this.fsFileWatcher) { - if (this.isClosed()) return; - // we don't know what's on disk anymore, - this.lastDiskValue = undefined; - //console.log("got change", eventType); - if (!ignore) { - this.fsLoadFromDiskDebounced(); - } - if (eventType == "rename") { - // check if file was deleted - this.fsCloseIfFileDeleted(); - // always have to recreate in case of a rename - this.fsFileWatcher.close(); - // start a new watcher since file descriptor changed - await until( - async () => { - if (this.isClosed()) return true; - try { - await this.fsInitFileWatcher(); - return true; - } catch (err) { - // console.warn( - // "sync-doc WARNING: issue creating file watch", - // this.path, - // err, - // ); - return false; - } - }, - { min: 3000 }, - ); - return; + if (this.fsFileWatcher != null) { + this.emit("watching"); + for await (const { eventType, ignore } of this.fsFileWatcher) { + if (this.isClosed()) return; + // we don't know what's on disk anymore, + this.lastDiskValue = undefined; + //console.log("got change", eventType); + if (!ignore) { + this.fsLoadFromDiskDebounced(); + } + if (eventType == "rename") { + break; + } } - } - //console.log("done watching"); + // check if file was deleted + this.fsCloseIfFileDeleted(); + this.fsFileWatcher?.close(); + delete this.fsFileWatcher; + } + // start a new watcher since file descriptor probably changed or maybe file deleted + await delay(this.watchRecreateWait ?? WATCH_RECREATE_WAIT); + await until( + async () => { + if (this.isClosed()) return true; + try { + await this.fsInitFileWatcher(); + return true; + } catch { + return false; + } + }, + { min: this.watchRecreateWait ?? WATCH_RECREATE_WAIT }, + ); })(); }; @@ -3895,6 +3702,7 @@ export class SyncDoc extends EventEmitter { }; private fsCloseIfFileDeleted = async () => { + if (this.isClosed()) return; if (this.fs == null) { throw Error("bug -- fs must be defined"); } From 156bef6d01e3fdbdc1394ec63ec49a6a56c70c4f Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 22 Jul 2025 18:48:08 +0000 Subject: [PATCH 060/798] syncdoc: a little bit of org and opt; frontend -- delay spinners and progress --- .../frontend/components/fake-progress.tsx | 1 - src/packages/frontend/components/loading.tsx | 14 +- .../frame-editors/frame-tree/editor.tsx | 4 +- .../frontend/project/new/new-file-page.tsx | 4 +- src/packages/sync/editor/generic/sync-doc.ts | 122 ++---------------- 5 files changed, 24 insertions(+), 121 deletions(-) diff --git a/src/packages/frontend/components/fake-progress.tsx b/src/packages/frontend/components/fake-progress.tsx index d634e14ad5..1b0f10ef85 100644 --- a/src/packages/frontend/components/fake-progress.tsx +++ b/src/packages/frontend/components/fake-progress.tsx @@ -23,7 +23,6 @@ export default function FakeProgress({ time }) { return ( null} percent={percent} strokeColor={{ "0%": "#108ee9", "100%": "#87d068" }} diff --git a/src/packages/frontend/components/loading.tsx b/src/packages/frontend/components/loading.tsx index 8e0dac0769..5b1578de9b 100644 --- a/src/packages/frontend/components/loading.tsx +++ b/src/packages/frontend/components/loading.tsx @@ -19,9 +19,9 @@ export const Estimate = null; // webpack + TS es2020 modules need this interface Props { style?: CSSProperties; text?: string; - estimate?: Estimate; + estimate?: Estimate | number; theme?: "medium" | undefined; - delay?: number; // if given, don't show anything until after delay milliseconds. The component could easily unmount by then, and hence never annoyingly flicker on screen. + delay?: number; // (default:1000) don't show anything until after delay milliseconds. The component could easily unmount by then, and hence never annoyingly flicker on screen. transparent?: boolean; } @@ -40,7 +40,7 @@ export function Loading({ text, estimate, theme, - delay, + delay = 1000, transparent = false, }: Props) { const intl = useIntl(); @@ -64,7 +64,13 @@ export function Loading({ {estimate != undefined && (
- +
)} diff --git a/src/packages/frontend/frame-editors/frame-tree/editor.tsx b/src/packages/frontend/frame-editors/frame-tree/editor.tsx index 3cd75739f8..a689300108 100644 --- a/src/packages/frontend/frame-editors/frame-tree/editor.tsx +++ b/src/packages/frontend/frame-editors/frame-tree/editor.tsx @@ -18,7 +18,7 @@ import { import { ErrorDisplay, Loading, - LoadingEstimate, + type LoadingEstimate, } from "@cocalc/frontend/components"; import { AvailableFeatures } from "@cocalc/frontend/project_configuration"; import { is_different } from "@cocalc/util/misc"; @@ -194,7 +194,7 @@ const FrameTreeEditor: React.FC = React.memo( if (is_loaded) return; return (
- +
); } diff --git a/src/packages/frontend/project/new/new-file-page.tsx b/src/packages/frontend/project/new/new-file-page.tsx index 08df80c968..015c97b069 100644 --- a/src/packages/frontend/project/new/new-file-page.tsx +++ b/src/packages/frontend/project/new/new-file-page.tsx @@ -6,7 +6,6 @@ import { Button, Input, Modal, Space } from "antd"; import { useEffect, useRef, useState } from "react"; import { defineMessage, FormattedMessage, useIntl } from "react-intl"; - import { default_filename } from "@cocalc/frontend/account"; import { Alert, Col, Row } from "@cocalc/frontend/antd-bootstrap"; import { @@ -25,7 +24,6 @@ import { SettingBox, Tip, } from "@cocalc/frontend/components"; -import FakeProgress from "@cocalc/frontend/components/fake-progress"; import ComputeServer from "@cocalc/frontend/compute/inline"; import { filenameIcon } from "@cocalc/frontend/file-associations"; import { FileUpload, UploadLink } from "@cocalc/frontend/file-upload"; @@ -430,7 +428,7 @@ export default function NewFilePage(props: Props) { footer={<>} >
- +
diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 73cf96966a..6a477b6768 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -119,7 +119,6 @@ import type { Patch, } from "./types"; import { isTestClient, patch_cmp } from "./util"; -import { CONAT_OPEN_FILE_TOUCH_INTERVAL } from "@cocalc/util/conat"; import mergeDeep from "@cocalc/util/immutable-deep-merge"; import { JUPYTER_SYNCDB_EXTENSIONS } from "@cocalc/util/jupyter/names"; import { LegacyHistory } from "./legacy"; @@ -399,7 +398,7 @@ export class SyncDoc extends EventEmitter { } const m = `WARNING: problem initializing ${this.path} -- ${err}`; log(m); - if (DEBUG) { + if (DEBUG || true) { console.trace(err); } // log always @@ -1200,59 +1199,6 @@ export class SyncDoc extends EventEmitter { this.ipywidgets_state?.close(); }; - // TODO: We **have** to do this on the client, since the backend - // **security model** for accessing the patches table only - // knows the string_id, but not the project_id/path. Thus - // there is no way currently to know whether or not the client - // has access to the patches, and hence the patches table - // query fails. This costs significant time -- a roundtrip - // and write to the database -- whenever the user opens a file. - // This fix should be to change the patches schema somehow - // to have the user also provide the project_id and path, thus - // proving they have access to the sha1 hash (string_id), but - // don't actually use the project_id and path as columns in - // the table. This requires some new idea I guess of virtual - // fields.... - // Also, this also establishes the correct doctype. - - // Since this MUST succeed before doing anything else. This is critical - // because the patches table can't be opened anywhere if the syncstring - // object doesn't exist, due to how our security works, *AND* that the - // patches table uses the string_id, which is a SHA1 hash. - private ensure_syncstring_exists_in_db = async (): Promise => { - const dbg = this.dbg("ensure_syncstring_exists_in_db"); - if (this.useConat) { - dbg("skipping -- no database"); - return; - } - - if (!this.client.is_connected()) { - dbg("wait until connected...", this.client.is_connected()); - await once(this.client, "connected"); - } - - if (this.client.is_browser() && !this.client.is_signed_in()) { - // the browser has to sign in, unlike the project (and compute servers) - await once(this.client, "signed_in"); - } - - if (this.state == ("closed" as State)) return; - - dbg("do syncstring write query..."); - - await callback2(this.client.query, { - query: { - syncstrings: { - string_id: this.string_id, - project_id: this.project_id, - path: this.path, - doctype: JSON.stringify(this.doctype), - }, - }, - }); - dbg("wrote syncstring to db - done."); - }; - private synctable = async ( query, options: any[], @@ -1445,15 +1391,10 @@ export class SyncDoc extends EventEmitter { dbg("getting table..."); this.syncstring_table = await this.synctable(query, []); - if (this.ephemeral && this.client.is_project()) { - await this.set_syncstring_table({ - doctype: JSON.stringify(this.doctype), - }); - } else { - dbg("handling the first update..."); - this.handle_syncstring_update(); - } + dbg("handling the first update..."); + this.handle_syncstring_update(); this.syncstring_table.on("change", this.handle_syncstring_update); + this.syncstring_table.on("change", this.update_has_unsaved_changes); }; // Used for internal debug logging @@ -1468,26 +1409,18 @@ export class SyncDoc extends EventEmitter { }; private initAll = async (): Promise => { + //const t0 = Date.now(); if (this.state !== "init") { throw Error("connect can only be called in init state"); } const log = this.dbg("initAll"); - log("update interest"); - this.initInterestLoop(); - - log("ensure syncstring exists"); - this.assert_not_closed("initAll -- before ensuring syncstring exists"); - await this.ensure_syncstring_exists_in_db(); - - await this.init_syncstring_table(); - this.assert_not_closed("initAll -- successful init_syncstring_table"); - log("patch_list, cursors, evaluator, ipywidgets"); this.assert_not_closed( "initAll -- before init patch_list, cursors, evaluator, ipywidgets", ); await Promise.all([ + this.init_syncstring_table(), this.init_patch_list(), this.init_cursors(), this.init_evaluator(), @@ -1511,6 +1444,7 @@ export class SyncDoc extends EventEmitter { this.update_has_unsaved_changes(); log("done"); + //console.log("initAll: done", Date.now() - t0); }; private assert_table_is_ready = (table: string): void => { @@ -1662,9 +1596,9 @@ export class SyncDoc extends EventEmitter { } } dbg("path exists"); - const lastChanged = new Date(this.last_changed()); + const lastChanged = this.last_changed(); const firstLoad = this.versions().length == 0; - if (firstLoad || stats.ctime > lastChanged) { + if (firstLoad || stats.mtime.valueOf() > lastChanged) { dbg( `disk file changed more recently than edits, so loading ${stats.ctime} > ${lastChanged}; firstLoad=${firstLoad}`, ); @@ -1747,10 +1681,6 @@ export class SyncDoc extends EventEmitter { update_has_unsaved_changes(); }); - this.syncstring_table.on("change", () => { - update_has_unsaved_changes(); - }); - dbg("adding all known patches"); patch_list.add(this.get_patches()); @@ -3080,6 +3010,8 @@ export class SyncDoc extends EventEmitter { if (this.isClosed()) return; this.last_save_to_disk_time = new Date(); await this.fs.writeFile(this.path, value); + const lastChanged = this.last_changed(); + await this.fs.utimes(this.path, lastChanged / 1000, lastChanged / 1000); this.lastDiskValue = value; }; @@ -3581,38 +3513,6 @@ export class SyncDoc extends EventEmitter { } }, 60000); - private initInterestLoop = async () => { - if (this.fs != null) { - return; - } - if (!this.client.is_browser()) { - // only browser clients -- so actual humans - return; - } - const touch = async () => { - if (this.state == "closed" || this.client?.touchOpenFile == null) return; - await this.client.touchOpenFile({ - path: this.path, - project_id: this.project_id, - doctype: this.doctype, - }); - }; - // then every CONAT_OPEN_FILE_TOUCH_INTERVAL (30 seconds). - await until( - async () => { - if (this.state == "closed") { - return true; - } - await touch(); - return false; - }, - { - start: CONAT_OPEN_FILE_TOUCH_INTERVAL, - max: CONAT_OPEN_FILE_TOUCH_INTERVAL, - }, - ); - }; - private fsLoadFromDiskDebounced = asyncDebounce( async () => { try { From c78bc6266aba1bfddfdd92b2cfe0b974da23f61a Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 22 Jul 2025 19:01:20 +0000 Subject: [PATCH 061/798] preload background file tabs --- src/packages/frontend/project/open-file.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/packages/frontend/project/open-file.ts b/src/packages/frontend/project/open-file.ts index 2f9a49912b..c905f6914b 100644 --- a/src/packages/frontend/project/open-file.ts +++ b/src/packages/frontend/project/open-file.ts @@ -28,6 +28,14 @@ import { syncdbPath as ipynbSyncdbPath } from "@cocalc/util/jupyter/names"; import { termPath } from "@cocalc/util/terminal/names"; import { excludeFromComputeServer } from "@cocalc/frontend/file-associations"; +// if true, PRELOAD_BACKGROUND_TABS makes it so all tabs have their file editing +// preloaded, even background tabs. This can make the UI much more responsive, +// since after refreshing your browser or opening a project that had tabs open, +// all files are ready to edit instantly. It uses more browser memory (of course), +// and increases server load. Most users have very few files open at once, +// so this is probably a major win for power users and has little impact on load. +const PRELOAD_BACKGROUND_TABS = true; + export interface OpenFileOpts { path: string; ext?: string; // if given, use editor for this extension instead of whatever extension path has. @@ -339,6 +347,10 @@ export async function open_file( return; } + if (PRELOAD_BACKGROUND_TABS) { + await actions.initFileRedux(opts.path); + } + if (opts.foreground) { actions.foreground_project(opts.change_history); const tab = path_to_tab(opts.path); From 687f713bb4b2872f5569845614cbe0a2f95a14ee Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 22 Jul 2025 20:56:34 +0000 Subject: [PATCH 062/798] syncdoc -- delete the old approach and embrace this.fs --- .../conat/test/sync-doc/watch-file.test.ts | 9 +- src/packages/sync/editor/generic/sync-doc.ts | 948 +----------------- 2 files changed, 54 insertions(+), 903 deletions(-) diff --git a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts index 85d11a5d1f..8f1238dbc0 100644 --- a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts +++ b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts @@ -47,9 +47,9 @@ describe("basic watching of file on disk happens automatically", () => { }); it("change file on disk should not trigger a load from disk", async () => { - const orig = s.fsLoadFromDiskDebounced; + const orig = s.readFileDebounced; let c = 0; - s.fsLoadFromDiskDebounced = () => { + s.readFileDebounced = () => { c += 1; }; s.from_str("a different value"); @@ -57,10 +57,10 @@ describe("basic watching of file on disk happens automatically", () => { expect(c).toBe(0); await delay(100); expect(c).toBe(0); - s.fsLoadFromDiskDebounced = orig; + s.readFileDebounced = orig; // disable the ignore that happens as part of save_to_disk, // or the tests below won't work - await s.fsFileWatcher?.ignore(0); + await s.fileWatcher?.ignore(0); }); let client2, s2; @@ -78,6 +78,7 @@ describe("basic watching of file on disk happens automatically", () => { s2.on("handle-file-change", () => c2++); await fs.writeFile(path, "version3"); + expect(await fs.readFile(path, "utf8")).toEqual("version3"); await wait({ until: () => { return s2.to_str() == "version3" && s.to_str() == "version3"; diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 6a477b6768..68e780ef0a 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -19,47 +19,13 @@ EVENTS: - ... TODO */ -const USE_CONAT = true; - -/* OFFLINE_THRESH_S - If the client becomes disconnected from - the backend for more than this long then---on reconnect---do - extra work to ensure that all snapshots are up to date (in - case snapshots were made when we were offline), and mark the - sent field of patches that weren't saved. I.e., we rebase - all offline changes. */ -// const OFFLINE_THRESH_S = 5 * 60; // 5 minutes. - -/* How often the local hub will autosave this file to disk if - it has it open and there are unsaved changes. This is very - important since it ensures that a user that edits a file but - doesn't click "Save" and closes their browser (right after - their edits have gone to the database), still has their - file saved to disk soon. This is important, e.g., for homework - getting collected and not missing the last few changes. It turns - out this is what people expect. - Set to 0 to disable. (But don't do that.) */ -const FILE_SERVER_AUTOSAVE_S = 45; -// const FILE_SERVER_AUTOSAVE_S = 5; - // How big of files we allow users to open using syncstrings. const MAX_FILE_SIZE_MB = 32; -// How frequently to check if file is or is not read only. -// The filesystem watcher is NOT sufficient for this, because -// it is NOT triggered on permissions changes. Thus we must -// poll for read only status periodically, unfortunately. -const READ_ONLY_CHECK_INTERVAL_MS = 7500; - // This parameter determines throttling when broadcasting cursor position // updates. Make this larger to reduce bandwidth at the expense of making // cursors less responsive. -const CURSOR_THROTTLE_MS = 750; - -// NATS is much faster and can handle load, and cursors only uses pub/sub -const CURSOR_THROTTLE_NATS_MS = 150; - -// Ignore file changes for this long after save to disk. -const RECENT_SAVE_TO_DISK_MS = 2000; +const CURSOR_THROTTLE_MS = 150; // If file does not exist for this long, then syncdoc emits a 'deleted' event. const DELETED_THRESHOLD = 2000; @@ -72,7 +38,6 @@ import { COMPUTE_THRESH_MS, COMPUTER_SERVER_CURSOR_TYPE, decodeUUIDtoNum, - SYNCDB_PARAMS as COMPUTE_SERVE_MANAGER_SYNCDB_PARAMS, } from "@cocalc/util/compute/manager"; import { DEFAULT_SNAPSHOT_INTERVAL } from "@cocalc/util/db-schema/syncstring-schema"; @@ -85,16 +50,13 @@ import { callback2, cancel_scheduled, once, - retry_until_success, until, asyncDebounce, } from "@cocalc/util/async-utils"; import { wait } from "@cocalc/util/async-wait"; import { - auxFileToOriginal, assertDefined, close, - endswith, field_cmp, filename_extension, hash_string, @@ -115,7 +77,6 @@ import type { CompressedPatch, DocType, Document, - FileWatcher, Patch, } from "./types"; import { isTestClient, patch_cmp } from "./util"; @@ -203,12 +164,6 @@ export class SyncDoc extends EventEmitter { // Throttling of incoming upstream patches from project to client. private patch_interval: number = 250; - // This is what's actually output by setInterval -- it's - // not an amount of time. - private fileserver_autosave_timer: number = 0; - - private read_only_timer: number = 0; - // throttling of change events -- e.g., is useful for course // editor where we have hundreds of changes and the UI gets // overloaded unless we throttle and group them. @@ -253,22 +208,14 @@ export class SyncDoc extends EventEmitter { private settings: Map = Map(); - private syncstring_save_state: string = ""; - // patches that this client made during this editing session. private my_patches: { [time: string]: XPatch } = {}; - private watch_path?: string; - private file_watcher?: FileWatcher; - private handle_patch_update_queue_running: boolean; private patch_update_queue: string[] = []; private undo_state: UndoState | undefined; - private save_to_disk_start_ctime: number | undefined; - private save_to_disk_end_ctime: number | undefined; - private persistent: boolean = false; private last_has_unsaved_changes?: boolean = undefined; @@ -278,9 +225,6 @@ export class SyncDoc extends EventEmitter { private sync_is_disabled: boolean = false; private delay_sync_timer: any; - // static because we want exactly one across all docs! - private static computeServerManagerDoc?: SyncDoc; - private useConat: boolean; legacy: LegacyHistory; @@ -334,7 +278,7 @@ export class SyncDoc extends EventEmitter { // NOTE: Do not use conat in test mode, since there we use a minimal // "fake" client that does all communication internally and doesn't // use conat. We also use this for the messages composer. - this.useConat = USE_CONAT && !isTestClient(opts.client); + this.useConat = !isTestClient(opts.client); if (this.ephemeral) { // So the doctype written to the database reflects the // ephemeral state. Here ephemeral determines whether @@ -416,164 +360,6 @@ export class SyncDoc extends EventEmitter { this.emit_change(); // from nothing to something. }; - // True if this client is responsible for managing - // the state of this document with respect to - // the file system. By default, the project is responsible, - // but it could be something else (e.g., a compute server!). It's - // important that whatever algorithm determines this, it is - // a function of state that is eventually consistent. - // IMPORTANT: whether or not we are the file server can - // change over time, so if you call isFileServer and - // set something up (e.g., autosave or a watcher), based - // on the result, you need to clear it when the state - // changes. See the function handleComputeServerManagerChange. - private isFileServer = reuseInFlight(async () => { - if (this.state == "closed") return; - if (this.client == null || this.client.is_browser()) { - // browser is never the file server (yet), and doesn't need to do - // anything related to watching for changes in state. - // Someday via webassembly or browsers making users files availabl, - // etc., we will have this. Not today. - return false; - } - const computeServerManagerDoc = this.getComputeServerManagerDoc(); - const log = this.dbg("isFileServer"); - if (computeServerManagerDoc == null) { - log("not using compute server manager for this doc"); - return this.client.is_project(); - } - - const state = computeServerManagerDoc.get_state(); - log("compute server manager doc state: ", state); - if (state == "closed") { - log("compute server manager is closed"); - // something really messed up - return this.client.is_project(); - } - if (state != "ready") { - try { - log( - "waiting for compute server manager doc to be ready; current state=", - state, - ); - await once(computeServerManagerDoc, "ready", 15000); - log("compute server manager is ready"); - } catch (err) { - log( - "WARNING -- failed to initialize computeServerManagerDoc -- err=", - err, - ); - return this.client.is_project(); - } - } - - // id of who the user *wants* to be the file server. - const path = this.getFileServerPath(); - const fileServerId = - computeServerManagerDoc.get_one({ path })?.get("id") ?? 0; - if (this.client.is_project()) { - log( - "we are project, so we are fileserver if fileServerId=0 and it is ", - fileServerId, - ); - return fileServerId == 0; - } - // at this point we have to be a compute server - const computeServerId = decodeUUIDtoNum(this.client.client_id()); - // this is usually true -- but might not be if we are switching - // directly from one compute server to another. - log("we are compute server and ", { fileServerId, computeServerId }); - return fileServerId == computeServerId; - }); - - private getFileServerPath = () => { - if (this.path?.endsWith("." + JUPYTER_SYNCDB_EXTENSIONS)) { - // treating jupyter as a weird special case here. - return auxFileToOriginal(this.path); - } - return this.path; - }; - - private getComputeServerManagerDoc = () => { - if (this.path == COMPUTE_SERVE_MANAGER_SYNCDB_PARAMS.path) { - // don't want to recursively explode! - return null; - } - if (SyncDoc.computeServerManagerDoc == null) { - if (this.client.is_project()) { - // @ts-ignore: TODO! - SyncDoc.computeServerManagerDoc = this.client.syncdoc({ - path: COMPUTE_SERVE_MANAGER_SYNCDB_PARAMS.path, - }); - } else { - // @ts-ignore: TODO! - SyncDoc.computeServerManagerDoc = this.client.sync_client.sync_db({ - project_id: this.project_id, - ...COMPUTE_SERVE_MANAGER_SYNCDB_PARAMS, - }); - } - if ( - SyncDoc.computeServerManagerDoc != null && - !this.client.is_browser() - ) { - // start watching for state changes - SyncDoc.computeServerManagerDoc.on( - "change", - this.handleComputeServerManagerChange, - ); - } - } - return SyncDoc.computeServerManagerDoc; - }; - - private handleComputeServerManagerChange = async (keys) => { - if (SyncDoc.computeServerManagerDoc == null) { - return; - } - let relevant = false; - for (const key of keys ?? []) { - if (key.get("path") == this.path) { - relevant = true; - break; - } - } - if (!relevant) { - return; - } - const path = this.getFileServerPath(); - const fileServerId = - SyncDoc.computeServerManagerDoc.get_one({ path })?.get("id") ?? 0; - const ourId = this.client.is_project() - ? 0 - : decodeUUIDtoNum(this.client.client_id()); - // we are considering ourself the file server already if we have - // either a watcher or autosave on. - const thinkWeAreFileServer = - this.file_watcher != null || this.fileserver_autosave_timer; - const weAreFileServer = fileServerId == ourId; - if (thinkWeAreFileServer != weAreFileServer) { - // life has changed! Let's adapt. - if (thinkWeAreFileServer) { - // we were acting as the file server, but now we are not. - await this.save_to_disk_filesystem_owner(); - // Stop doing things we are no longer supposed to do. - clearInterval(this.fileserver_autosave_timer as any); - this.fileserver_autosave_timer = 0; - // stop watching filesystem - await this.update_watch_path(); - } else { - // load our state from the disk - await this.readFile(); - // we were not acting as the file server, but now we need. Let's - // step up to the plate. - // start watching filesystem - await this.update_watch_path(this.path); - // enable autosave - await this.init_file_autosave(); - } - } - }; - // Return id of ACTIVE remote compute server, if one is connected and pinging, or 0 // if none is connected. This is used by Jupyter to determine who // should evaluate code. @@ -627,7 +413,7 @@ export class SyncDoc extends EventEmitter { locs: any, side_effect: boolean = false, ) => { - if (this.state != "ready") { + if (!this.isReady()) { return; } if (this.cursors_table == null) { @@ -680,7 +466,7 @@ export class SyncDoc extends EventEmitter { set_cursor_locs: typeof this.setCursorLocsNoThrottle = throttle( this.setCursorLocsNoThrottle, - USE_CONAT ? CURSOR_THROTTLE_NATS_MS : CURSOR_THROTTLE_MS, + CURSOR_THROTTLE_MS, { leading: true, trailing: true, @@ -729,6 +515,7 @@ export class SyncDoc extends EventEmitter { }; isClosed = () => (this.state ?? "closed") == "closed"; + isReady = () => this.state == "ready"; private set_state = (state: State): void => { this.state = state; @@ -978,42 +765,6 @@ export class SyncDoc extends EventEmitter { return this.undo_state; }; - private save_to_disk_autosave = async (): Promise => { - if (this.state !== "ready") { - return; - } - const dbg = this.dbg("save_to_disk_autosave"); - dbg(); - try { - await this.save_to_disk(); - } catch (err) { - dbg(`failed -- ${err}`); - } - }; - - /* Make it so the local hub project will automatically save - the file to disk periodically. */ - private init_file_autosave = async () => { - // Do not autosave sagews until we resolve - // https://github.com/sagemathinc/cocalc/issues/974 - // Similarly, do not autosave ipynb because of - // https://github.com/sagemathinc/cocalc/issues/5216 - if ( - !FILE_SERVER_AUTOSAVE_S || - !(await this.isFileServer()) || - this.fileserver_autosave_timer || - endswith(this.path, ".sagews") || - endswith(this.path, "." + JUPYTER_SYNCDB_EXTENSIONS) - ) { - return; - } - - // Explicit cast due to node vs browser typings. - this.fileserver_autosave_timer = ( - setInterval(this.save_to_disk_autosave, FILE_SERVER_AUTOSAVE_S * 1000) - ); - }; - // account_id of the user who made the edit at // the given point in time. account_id = (time: number): string => { @@ -1119,10 +870,6 @@ export class SyncDoc extends EventEmitter { const dbg = this.dbg("close"); dbg("close"); - SyncDoc.computeServerManagerDoc?.removeListener( - "change", - this.handleComputeServerManagerChange, - ); // // SYNC STUFF // @@ -1152,25 +899,13 @@ export class SyncDoc extends EventEmitter { cancel_scheduled(this.emit_change); } - if (this.fileserver_autosave_timer) { - clearInterval(this.fileserver_autosave_timer as any); - this.fileserver_autosave_timer = 0; - } - - if (this.read_only_timer) { - clearInterval(this.read_only_timer as any); - this.read_only_timer = 0; - } - this.patch_update_queue = []; // Stop watching for file changes. It's important to // do this *before* all the await's below, since // this syncdoc can't do anything in response to a // a file change in its current state. - this.update_watch_path(); // no input = closes it, if open - - this.fsCloseFileWatcher(); + this.closeFileWatcher(); if (this.patch_list != null) { // not async -- just a data structure in memory @@ -1437,7 +1172,7 @@ export class SyncDoc extends EventEmitter { log("file_use_interval"); this.init_file_use_interval(); - await this.fsLoadFromDiskIfNewer(); + await this.loadFromDiskIfNewer(); this.emit("init"); this.assert_not_closed("initAll -- after waiting until fully ready"); @@ -1457,7 +1192,7 @@ export class SyncDoc extends EventEmitter { }; assert_is_ready = (desc: string): void => { - if (this.state != "ready") { + if (!this.isReady()) { throw Error(`must be ready -- ${desc}`); } }; @@ -1531,57 +1266,10 @@ export class SyncDoc extends EventEmitter { await Promise.all(v); }; - private pathExistsAndIsReadOnly = async (path): Promise => { - if (this.client.path_exists == null) { - throw Error("legacy clients must define path_exists"); - } - if (this.client.path_access == null) { - throw Error("legacy clients must define path_access"); - } - - try { - await callback2(this.client.path_access, { - path, - mode: "w", - }); - // clearly exists and is NOT read only: - return false; - } catch (err) { - // either it doesn't exist or it is read only - if (await callback2(this.client.path_exists, { path })) { - // it exists, so is read only and exists - return true; - } - // doesn't exist - return false; - } - }; - - private file_is_read_only = async (): Promise => { - if (await this.pathExistsAndIsReadOnly(this.path)) { - return true; - } - const path = this.getFileServerPath(); - if (path != this.path) { - if (await this.pathExistsAndIsReadOnly(path)) { - return true; - } - } - return false; - }; - - private update_if_file_is_read_only = async (): Promise => { - const read_only = await this.file_is_read_only(); - if (this.state == "closed") { - return; - } - this.set_read_only(read_only); - }; - - private fsLoadFromDiskIfNewer = async (): Promise => { + private loadFromDiskIfNewer = async (): Promise => { // [ ] TODO: readonly handling... if (this.fs == null) throw Error("bug"); - const dbg = this.dbg("fsLoadFromDiskIfNewer"); + const dbg = this.dbg("loadFromDiskIfNewer"); let stats; try { stats = await this.fs.stat(this.path); @@ -1602,7 +1290,7 @@ export class SyncDoc extends EventEmitter { dbg( `disk file changed more recently than edits, so loading ${stats.ctime} > ${lastChanged}; firstLoad=${firstLoad}`, ); - await this.fsLoadFromDisk(); + await this.readFile(); if (firstLoad) { dbg("emitting first-load event"); // this event is emited the first time the document is ever loaded from disk. @@ -2008,12 +1696,12 @@ export class SyncDoc extends EventEmitter { break; } } - if (this.state != "ready") { + if (!this.isReady()) { // above async waits could have resulted in state change. return; } await this.handle_patch_update_queue(true); - if (this.state != "ready") { + if (!this.isReady()) { return; } @@ -2174,7 +1862,7 @@ export class SyncDoc extends EventEmitter { this.patch_list.add([obj]); this.patches_table.set(obj); await this.patches_table.save(); - if (this.state != "ready") { + if (!this.isReady()) { return; } @@ -2449,77 +2137,6 @@ export class SyncDoc extends EventEmitter { return this.last_save_to_disk_time; }; - private handle_syncstring_save_state = async ( - state: string, - time: Date, - ): Promise => { - // Called when the save state changes. - - /* this.syncstring_save_state is used to make it possible to emit a - 'save-to-disk' event, whenever the state changes - to indicate a save completed. - - NOTE: it is intentional that this.syncstring_save_state is not defined - the first time this function is called, so that save-to-disk - with last save time gets emitted on initial load (which, e.g., triggers - latex compilation properly in case of a .tex file). - */ - if (state === "done" && this.syncstring_save_state !== "done") { - this.last_save_to_disk_time = time; - this.emit("save-to-disk", time); - } - const dbg = this.dbg("handle_syncstring_save_state"); - dbg( - `state='${state}', this.syncstring_save_state='${this.syncstring_save_state}', this.state='${this.state}'`, - ); - if ( - this.state === "ready" && - (await this.isFileServer()) && - this.syncstring_save_state !== "requested" && - state === "requested" - ) { - this.syncstring_save_state = state; // only used in the if above - dbg("requesting save to disk -- calling save_to_disk"); - // state just changed to requesting a save to disk... - // so we do it (unless of course syncstring is still - // being initialized). - try { - // Uncomment the following to test simulating a - // random failure in save_to_disk: - // if (Math.random() < 0.5) throw Error("CHAOS MONKEY!"); // FOR TESTING ONLY. - await this.save_to_disk(); - } catch (err) { - // CRITICAL: we must unset this.syncstring_save_state (and set the save state); - // otherwise, it stays as "requested" and this if statement would never get - // run again, thus completely breaking saving this doc to disk. - // It is normal behavior that *sometimes* this.save_to_disk might - // throw an exception, e.g., if the file is temporarily deleted - // or save it called before everything is initialized, or file - // is temporarily set readonly, or maybe there is a file system error. - // Of course, the finally below will also take care of this. However, - // it's nice to record the error here. - this.syncstring_save_state = "done"; - await this.set_save({ state: "done", error: `${err}` }); - dbg(`ERROR saving to disk in handle_syncstring_save_state-- ${err}`); - } finally { - // No matter what, after the above code is run, - // the save state in the table better be "done". - // We triple check that here, though of course - // we believe the logic in save_to_disk and above - // should always accomplish this. - dbg("had to set the state to done in finally block"); - if ( - this.state === "ready" && - (this.syncstring_save_state != "done" || - this.syncstring_table_get_one().getIn(["save", "state"]) != "done") - ) { - this.syncstring_save_state = "done"; - await this.set_save({ state: "done", error: "" }); - } - } - } - }; - private handle_syncstring_update = async (): Promise => { if (this.state === "closed") { return; @@ -2530,10 +2147,6 @@ export class SyncDoc extends EventEmitter { const data = this.syncstring_table_get_one(); const x: any = data != null ? data.toJS() : undefined; - if (x != null && x.save != null) { - this.handle_syncstring_save_state(x.save.state, x.save.time); - } - dbg(JSON.stringify(x)); if (x == null || x.users == null) { dbg("new_document"); @@ -2622,147 +2235,9 @@ export class SyncDoc extends EventEmitter { this.emit("metadata-change"); }; - private initFileWatcher = async (): Promise => { - if (this.fs != null) { - return await this.fsInitFileWatcher(); - } - - if (!(await this.isFileServer())) { - // ensures we are NOT watching anything - await this.update_watch_path(); - return; - } - - // If path isn't being properly watched, make it so. - if (this.watch_path !== this.path) { - await this.update_watch_path(this.path); - } - - await this.pending_save_to_disk(); - }; - - private pending_save_to_disk = async (): Promise => { - this.assert_table_is_ready("syncstring"); - if (!(await this.isFileServer())) { - return; - } - - const x = this.syncstring_table.get_one(); - // Check if there is a pending save-to-disk that is needed. - if (x != null && x.getIn(["save", "state"]) === "requested") { - try { - await this.save_to_disk(); - } catch (err) { - const dbg = this.dbg("pending_save_to_disk"); - dbg(`ERROR saving to disk in pending_save_to_disk -- ${err}`); - } - } - }; - - private update_watch_path = async (path?: string): Promise => { - if (this.fs != null) { - return; - } - const dbg = this.dbg("update_watch_path"); - if (this.file_watcher != null) { - // clean up - dbg("close"); - this.file_watcher.close(); - delete this.file_watcher; - delete this.watch_path; - } - if (path != null && this.client.is_deleted(path, this.project_id)) { - dbg(`not setting up watching since "${path}" is explicitly deleted`); - return; - } - if (path == null) { - dbg("not opening another watcher since path is null"); - this.watch_path = path; - return; - } - if (this.watch_path != null) { - // this case is impossible since we deleted it above if it is was defined. - dbg("watch_path already defined"); - return; - } - dbg("opening watcher..."); - if (this.state === "closed") { - throw Error("must not be closed"); - } - if (this.client.path_exists == null) { - throw Error("legacy clients must define path_exists"); - } - this.watch_path = path; - try { - if (!(await callback2(this.client.path_exists, { path }))) { - if (this.client.is_deleted(path, this.project_id)) { - dbg(`not setting up watching since "${path}" is explicitly deleted`); - return; - } - // path does not exist - dbg( - `write '${path}' to disk from syncstring in-memory database version`, - ); - const data = this.to_str(); - await callback2(this.client.write_file, { path, data }); - dbg(`wrote '${path}' to disk`); - } - } catch (err) { - // This can happen, e.g, if path is read only. - dbg(`could NOT write '${path}' to disk -- ${err}`); - await this.update_if_file_is_read_only(); - // In this case, can't really setup a file watcher. - return; - } - - dbg("now requesting to watch file"); - this.file_watcher = this.client.watch_file({ path }); - this.file_watcher.on("change", this.handle_file_watcher_change); - this.file_watcher.on("delete", this.handle_file_watcher_delete); - this.setupReadOnlyTimer(); - }; - - private setupReadOnlyTimer = () => { - if (this.read_only_timer) { - clearInterval(this.read_only_timer as any); - this.read_only_timer = 0; - } - this.read_only_timer = ( - setInterval(this.update_if_file_is_read_only, READ_ONLY_CHECK_INTERVAL_MS) - ); - }; - - private handle_file_watcher_change = async (ctime: Date): Promise => { - const dbg = this.dbg("handle_file_watcher_change"); - const time: number = ctime.valueOf(); - dbg( - `file_watcher: change, ctime=${time}, this.save_to_disk_start_ctime=${this.save_to_disk_start_ctime}, this.save_to_disk_end_ctime=${this.save_to_disk_end_ctime}`, - ); - if ( - this.save_to_disk_start_ctime == null || - (this.save_to_disk_end_ctime != null && - time - this.save_to_disk_end_ctime >= RECENT_SAVE_TO_DISK_MS) - ) { - // Either we never saved to disk, or the last attempt - // to save was at least RECENT_SAVE_TO_DISK_MS ago, and it finished, - // so definitely this change event was not caused by it. - dbg("load_from_disk since no recent save to disk"); - await this.readFile(); - return; - } - }; - - private handle_file_watcher_delete = async (): Promise => { - this.assert_is_ready("handle_file_watcher_delete"); - const dbg = this.dbg("handle_file_watcher_delete"); - dbg("delete: set_deleted and closing"); - await this.client.set_deleted(this.path, this.project_id); - this.close(); - }; - - private fsLoadFromDisk = async (): Promise => { + readFile = reuseInFlight(async (): Promise => { if (this.fs == null) throw Error("bug"); - const dbg = this.dbg("fsLoadFromDisk"); + const dbg = this.dbg("readFile"); let size: number; let contents; @@ -2786,98 +2261,15 @@ export class SyncDoc extends EventEmitter { await this.save(); this.emit("after-change"); return size; - }; - - readFile = reuseInFlight(async (): Promise => { - if (this.fs != null) { - return await this.fsLoadFromDisk(); - } - if (this.client.path_exists == null) { - throw Error("legacy clients must define path_exists"); - } - const path = this.path; - const dbg = this.dbg("load_from_disk"); - dbg(); - const exists: boolean = await callback2(this.client.path_exists, { path }); - let size: number; - if (!exists) { - dbg("file no longer exists -- setting to blank"); - size = 0; - this.from_str(""); - } else { - dbg("file exists"); - await this.update_if_file_is_read_only(); - - const data = await callback2(this.client.path_read, { - path, - maxsize_MB: MAX_FILE_SIZE_MB, - }); - - size = data.length; - dbg(`got it -- length=${size}`); - this.from_str(data); - this.commit(); - // we also know that this is the version on disk, so we update the hash - await this.set_save({ - state: "done", - error: "", - hash: hash_string(data), - }); - } - // save new version to database, which we just set via from_str. - await this.save(); - return size; }); - private set_save = async (save: { - state: string; - error: string; - hash?: number; - expected_hash?: number; - time?: number; - }): Promise => { - this.assert_table_is_ready("syncstring"); - // set timestamp of when the save happened; this can be useful - // for coordinating running code, etc.... and is just generally useful. - const cur = this.syncstring_table_get_one().toJS()?.save; - if (cur != null) { - if ( - cur.state == save.state && - cur.error == save.error && - cur.hash == (save.hash ?? cur.hash) && - cur.expected_hash == (save.expected_hash ?? cur.expected_hash) && - cur.time == (save.time ?? cur.time) - ) { - // no genuine change, so no point in wasting cycles on updating. - return; - } - } - if (!save.time) { - save.time = Date.now(); - } - await this.set_syncstring_table({ save }); - }; - - private set_read_only = async (read_only: boolean): Promise => { - this.assert_table_is_ready("syncstring"); - await this.set_syncstring_table({ read_only }); - }; - is_read_only = (): boolean => { - this.assert_table_is_ready("syncstring"); + // [ ] TODO return this.syncstring_table_get_one().get("read_only"); }; wait_until_read_only_known = async (): Promise => { - await this.wait_until_ready(); - function read_only_defined(t: SyncTable): boolean { - const x = t.get_one(); - if (x == null) { - return false; - } - return x.get("read_only") != null; - } - await this.syncstring_table.wait(read_only_defined, 5 * 60); + // [ ] TODO }; /* Returns true if the current live version of this document has @@ -2888,30 +2280,15 @@ export class SyncDoc extends EventEmitter { commited to the database yet. Returns *undefined* if initialization not even done yet. */ has_unsaved_changes = (): boolean | undefined => { - if (this.state !== "ready") { - return; - } - if (this.fs != null) { - return this.fsHasUnsavedChanges(); - } - const dbg = this.dbg("has_unsaved_changes"); - try { - return this.hash_of_saved_version() !== this.hash_of_live_version(); - } catch (err) { - dbg( - "exception computing hash_of_saved_version and hash_of_live_version", - err, - ); - // This could happen, e.g. when syncstring_table isn't connected - // in some edge case. Better to just say we don't know then crash - // everything. See https://github.com/sagemathinc/cocalc/issues/3577 + if (!this.isReady()) { return; } + return this.hasUnsavedChanges(); }; // Returns hash of last version saved to disk (as far as we know). hash_of_saved_version = (): number | undefined => { - if (this.state !== "ready") { + if (!this.isReady()) { return; } return this.syncstring_table_get_one().getIn(["save", "hash"]) as @@ -2924,7 +2301,7 @@ export class SyncDoc extends EventEmitter { (TODO: write faster version of this for syncdb, which avoids converting to a string, which is a waste of time.) */ hash_of_live_version = (): number | undefined => { - if (this.state !== "ready") { + if (!this.isReady()) { return; } return hash_string(this.doc.to_str()); @@ -2937,7 +2314,7 @@ export class SyncDoc extends EventEmitter { the user to close their browser. */ has_uncommitted_changes = (): boolean => { - if (this.state !== "ready") { + if (!this.isReady()) { return false; } return this.patches_table.has_uncommitted_changes(); @@ -2983,12 +2360,12 @@ export class SyncDoc extends EventEmitter { }; private lastDiskValue: string | undefined = undefined; - fsHasUnsavedChanges = (): boolean => { + private hasUnsavedChanges = (): boolean => { return this.lastDiskValue != this.to_str(); }; - fsSaveToDisk = async () => { - const dbg = this.dbg("fsSaveToDisk"); + writeFile = async () => { + const dbg = this.dbg("writeFile"); if (this.client.is_deleted(this.path, this.project_id)) { dbg("not saving to disk because deleted"); return; @@ -2998,11 +2375,11 @@ export class SyncDoc extends EventEmitter { throw Error("bug"); } const value = this.to_str(); - // tell watcher not to fire any change events for a little time, + // include {ignore:true} with events for this long, // so no clients waste resources loading in response to us saving // to disk. try { - await this.fsFileWatcher?.ignore(2000); + await this.fileWatcher?.ignore(2000); } catch { // not a big problem if we can't ignore (e.g., this happens potentially // after deleting the file or if file doesn't exist) @@ -3018,7 +2395,7 @@ export class SyncDoc extends EventEmitter { /* Initiates a save of file to disk, then waits for the state to change. */ save_to_disk = reuseInFlight(async (): Promise => { - if (this.state != "ready") { + if (!this.isReady()) { // We just make save_to_disk a successful // no operation, if the document is either // closed or hasn't finished opening, since @@ -3029,67 +2406,8 @@ export class SyncDoc extends EventEmitter { return; } - if (this.fs != null) { - this.commit(); - await this.fsSaveToDisk(); - this.update_has_unsaved_changes(); - return; - } - - const dbg = this.dbg("save_to_disk"); - if (this.client.is_deleted(this.path, this.project_id)) { - dbg("not saving to disk because deleted"); - await this.set_save({ state: "done", error: "" }); - return; - } - - // Make sure to include changes to the live document. - // A side effect of save if we didn't do this is potentially - // discarding them, which is obviously not good. this.commit(); - - dbg("initiating the save"); - if (!this.has_unsaved_changes()) { - dbg("no unsaved changes, so don't save"); - // CRITICAL: this optimization is assumed by - // autosave, etc. - await this.set_save({ state: "done", error: "" }); - return; - } - - if (this.is_read_only()) { - dbg("read only, so can't save to disk"); - // save should fail if file is read only and there are changes - throw Error("can't save readonly file with changes to disk"); - } - - // First make sure any changes are saved to the database. - // One subtle case where this matters is that loading a file - // with \r's into codemirror changes them to \n... - if (!(await this.isFileServer())) { - dbg("browser client -- sending any changes over network"); - await this.save(); - dbg("save done; now do actual save to the *disk*."); - this.assert_is_ready("save_to_disk - after save"); - } - - try { - await this.save_to_disk_aux(); - } catch (err) { - if (this.state != "ready") return; - const error = `save to disk failed -- ${err}`; - dbg(error); - if (await this.isFileServer()) { - this.set_save({ error, state: "done" }); - } - } - if (this.state != "ready") return; - - if (!(await this.isFileServer())) { - dbg("now wait for the save to disk to finish"); - this.assert_is_ready("save_to_disk - waiting to finish"); - await this.wait_for_save_to_disk_done(); - } + await this.writeFile(); this.update_has_unsaved_changes(); }); @@ -3107,7 +2425,7 @@ export class SyncDoc extends EventEmitter { }; private update_has_unsaved_changes = (): void => { - if (this.state != "ready") { + if (!this.isReady()) { // This can happen, since this is called by a debounced function. // Make it a no-op in case we're not ready. // See https://github.com/sagemathinc/cocalc/issues/3577 @@ -3120,174 +2438,6 @@ export class SyncDoc extends EventEmitter { } }; - // wait for save.state to change state. - private wait_for_save_to_disk_done = async (): Promise => { - const dbg = this.dbg("wait_for_save_to_disk_done"); - dbg(); - function until(table): boolean { - const done = table.get_one().getIn(["save", "state"]) === "done"; - dbg("checking... done=", done); - return done; - } - - let last_err: string | undefined = undefined; - const f = async () => { - dbg("f"); - if ( - this.state != "ready" || - this.client.is_deleted(this.path, this.project_id) - ) { - dbg("not ready or deleted - no longer trying to save."); - return; - } - try { - dbg("waiting until done..."); - await this.syncstring_table.wait(until, 15); - } catch (err) { - dbg("timed out after 15s"); - throw Error("timed out"); - } - if ( - this.state != "ready" || - this.client.is_deleted(this.path, this.project_id) - ) { - dbg("not ready or deleted - no longer trying to save."); - return; - } - const err = this.syncstring_table_get_one().getIn(["save", "error"]) as - | string - | undefined; - if (err) { - dbg("error", err); - last_err = err; - throw Error(err); - } - dbg("done, with no error."); - last_err = undefined; - return; - }; - await retry_until_success({ - f, - max_tries: 8, - desc: "wait_for_save_to_disk_done", - }); - if ( - this.state != "ready" || - this.client.is_deleted(this.path, this.project_id) - ) { - return; - } - if (last_err && typeof this.client.log_error != null) { - this.client.log_error?.({ - string_id: this.string_id, - path: this.path, - project_id: this.project_id, - error: `Error saving file -- ${last_err}`, - }); - } - }; - - /* Auxiliary function 2 for saving to disk: - If this is associated with - a project and has a filename. - A user (web browsers) sets the save state to requested. - The project sets the state to saving, does the save - to disk, then sets the state to done. - */ - private save_to_disk_aux = async (): Promise => { - this.assert_is_ready("save_to_disk_aux"); - - if (!(await this.isFileServer())) { - return await this.save_to_disk_non_filesystem_owner(); - } - - try { - return await this.save_to_disk_filesystem_owner(); - } catch (err) { - this.emit("save_to_disk_filesystem_owner", err); - throw err; - } - }; - - private save_to_disk_non_filesystem_owner = async (): Promise => { - this.assert_is_ready("save_to_disk_non_filesystem_owner"); - - if (!this.has_unsaved_changes()) { - /* Browser client has no unsaved changes, - so don't need to save -- - CRITICAL: this optimization is assumed by autosave. - */ - return; - } - const x = this.syncstring_table.get_one(); - if (x != null && x.getIn(["save", "state"]) === "requested") { - // Nothing to do -- save already requested, which is - // all the browser client has to do. - return; - } - - // string version of this doc - const data: string = this.to_str(); - const expected_hash = hash_string(data); - await this.set_save({ state: "requested", error: "", expected_hash }); - }; - - private save_to_disk_filesystem_owner = async (): Promise => { - this.assert_is_ready("save_to_disk_filesystem_owner"); - const dbg = this.dbg("save_to_disk_filesystem_owner"); - - // check if on-disk version is same as in memory, in - // which case no save is needed. - const data = this.to_str(); // string version of this doc - const hash = hash_string(data); - dbg("hash = ", hash); - - /* - // TODO: put this consistency check back in (?). - const expected_hash = this.syncstring_table - .get_one() - .getIn(["save", "expected_hash"]); - */ - - if (hash === this.hash_of_saved_version()) { - // No actual save to disk needed; still we better - // record this fact in table in case it - // isn't already recorded - this.set_save({ state: "done", error: "", hash }); - return; - } - - const path = this.path; - if (!path) { - const err = "cannot save without path"; - this.set_save({ state: "done", error: err }); - throw Error(err); - } - - dbg("project - write to disk file", path); - // set window to slightly earlier to account for clock - // imprecision. - // Over an sshfs mount, all stats info is **rounded down - // to the nearest second**, which this also takes care of. - this.save_to_disk_start_ctime = Date.now() - 1500; - this.save_to_disk_end_ctime = undefined; - try { - await callback2(this.client.write_file, { path, data }); - this.assert_is_ready("save_to_disk_filesystem_owner -- after write_file"); - const stat = await callback2(this.client.path_stat, { path }); - this.assert_is_ready("save_to_disk_filesystem_owner -- after path_state"); - this.save_to_disk_end_ctime = stat.ctime.valueOf() + 1500; - this.set_save({ - state: "done", - error: "", - hash: hash_string(data), - }); - } catch (err) { - this.set_save({ state: "done", error: JSON.stringify(err) }); - throw err; - } - }; - /* When the underlying synctable that defines the state of the document changes due to new remote patches, this @@ -3513,11 +2663,11 @@ export class SyncDoc extends EventEmitter { } }, 60000); - private fsLoadFromDiskDebounced = asyncDebounce( + private readFileDebounced = asyncDebounce( async () => { try { this.emit("handle-file-change"); - await this.fsLoadFromDisk(); + await this.readFile(); } catch {} }, WATCH_DEBOUNCE, @@ -3527,38 +2677,38 @@ export class SyncDoc extends EventEmitter { }, ); - private fsFileWatcher?: any; - private fsInitFileWatcher = async () => { + private fileWatcher?: any; + private initFileWatcher = async () => { if (this.fs == null) { throw Error("this.fs must be defined"); } // use this.fs interface to watch path for changes -- we try once: try { - this.fsFileWatcher = await this.fs.watch(this.path, { unique: true }); + this.fileWatcher = await this.fs.watch(this.path, { unique: true }); } catch {} if (this.isClosed()) return; // not closed -- so if above succeeds we start watching. // if not, we loop waiting for file to be created so we can watch it (async () => { - if (this.fsFileWatcher != null) { + if (this.fileWatcher != null) { this.emit("watching"); - for await (const { eventType, ignore } of this.fsFileWatcher) { + for await (const { eventType, ignore } of this.fileWatcher) { if (this.isClosed()) return; // we don't know what's on disk anymore, this.lastDiskValue = undefined; //console.log("got change", eventType); if (!ignore) { - this.fsLoadFromDiskDebounced(); + this.readFileDebounced(); } if (eventType == "rename") { break; } } // check if file was deleted - this.fsCloseIfFileDeleted(); - this.fsFileWatcher?.close(); - delete this.fsFileWatcher; + this.closeIfFileDeleted(); + this.fileWatcher?.close(); + delete this.fileWatcher; } // start a new watcher since file descriptor probably changed or maybe file deleted await delay(this.watchRecreateWait ?? WATCH_RECREATE_WAIT); @@ -3566,7 +2716,7 @@ export class SyncDoc extends EventEmitter { async () => { if (this.isClosed()) return true; try { - await this.fsInitFileWatcher(); + await this.initFileWatcher(); return true; } catch { return false; @@ -3577,15 +2727,15 @@ export class SyncDoc extends EventEmitter { })(); }; - private fsCloseFileWatcher = () => { - this.fsFileWatcher?.close(); - delete this.fsFileWatcher; + private closeFileWatcher = () => { + this.fileWatcher?.close(); + delete this.fileWatcher; }; // returns true if file definitely exists right now, // false if it definitely does not, and throws exception otherwise, // e.g., network error. - private fsFileExists = async (): Promise => { + private fileExists = async (): Promise => { if (this.fs == null) { throw Error("bug -- fs must be defined"); } @@ -3601,7 +2751,7 @@ export class SyncDoc extends EventEmitter { } }; - private fsCloseIfFileDeleted = async () => { + private closeIfFileDeleted = async () => { if (this.isClosed()) return; if (this.fs == null) { throw Error("bug -- fs must be defined"); @@ -3610,7 +2760,7 @@ export class SyncDoc extends EventEmitter { const threshold = this.deletedThreshold ?? DELETED_THRESHOLD; while (true) { try { - if (await this.fsFileExists()) { + if (await this.fileExists()) { // file definitely exists right now. return; } From 1dc407e01a8d25dc17ca8116c3c0f59084908630 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 22 Jul 2025 22:47:12 +0000 Subject: [PATCH 063/798] add find command to fs api - this is one thing that node's fs does not have, but this is done in a highly flexible, but safe and minimal way. --- .../conat/files/test/local-path.test.ts | 10 ++- .../backend/files/sandbox/find.test.ts | 42 ++++++++++ src/packages/backend/files/sandbox/find.ts | 83 +++++++++++++++++++ src/packages/backend/files/sandbox/index.ts | 25 +++--- src/packages/conat/files/fs.ts | 12 +++ 5 files changed, 159 insertions(+), 13 deletions(-) create mode 100644 src/packages/backend/files/sandbox/find.test.ts create mode 100644 src/packages/backend/files/sandbox/find.ts diff --git a/src/packages/backend/conat/files/test/local-path.test.ts b/src/packages/backend/conat/files/test/local-path.test.ts index 30c8d04476..2c49add752 100644 --- a/src/packages/backend/conat/files/test/local-path.test.ts +++ b/src/packages/backend/conat/files/test/local-path.test.ts @@ -103,17 +103,25 @@ describe("use all the standard api functions of fs", () => { } }); + let fire; it("readdir works", async () => { await fs.mkdir("dirtest"); for (let i = 0; i < 5; i++) { await fs.writeFile(`dirtest/${i}`, `${i}`); } - const fire = "🔥.txt"; + fire = "🔥.txt"; await fs.writeFile(join("dirtest", fire), "this is ️‍🔥!"); const v = await fs.readdir("dirtest"); expect(v).toEqual(["0", "1", "2", "3", "4", fire]); }); + it("use the find command instead of readdir", async () => { + const { stdout } = await fs.find("dirtest", "%f\n"); + const v = stdout.toString().trim().split("\n"); + // output of find is NOT in alphabetical order: + expect(new Set(v)).toEqual(new Set(["0", "1", "2", "3", "4", fire])); + }); + it("realpath works", async () => { await fs.writeFile("file0", "file0"); await fs.symlink("file0", "file1"); diff --git a/src/packages/backend/files/sandbox/find.test.ts b/src/packages/backend/files/sandbox/find.test.ts new file mode 100644 index 0000000000..ca725e3569 --- /dev/null +++ b/src/packages/backend/files/sandbox/find.test.ts @@ -0,0 +1,42 @@ +import find from "./find"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let tempDir; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); +}); +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); + +describe("find files", () => { + it("directory starts empty", async () => { + const { stdout, truncated } = await find(tempDir, "%f\n"); + expect(stdout.length).toBe(0); + expect(truncated).toBe(false); + }); + + it("create a file and see it appears in find", async () => { + await writeFile(join(tempDir, "a.txt"), "hello"); + const { stdout, truncated } = await find(tempDir, "%f\n"); + expect(truncated).toBe(false); + expect(stdout.toString()).toEqual("a.txt\n"); + }); + + // this is NOT a great test, unfortunately. + const count = 10000; + it(`hopefully exceed the timeout by creating ${count} files`, async () => { + for (let i = 0; i < count; i++) { + await writeFile(join(tempDir, `${i}`), ""); + } + const t = Date.now(); + const { stdout, truncated } = await find(tempDir, "%f\n", 2); + expect(truncated).toBe(true); + expect(Date.now() - t).toBeGreaterThan(1); + + const { stdout: stdout2 } = await find(tempDir, "%f\n"); + expect(stdout2.length).toBeGreaterThan(stdout.length); + }); +}); diff --git a/src/packages/backend/files/sandbox/find.ts b/src/packages/backend/files/sandbox/find.ts new file mode 100644 index 0000000000..edc7047c37 --- /dev/null +++ b/src/packages/backend/files/sandbox/find.ts @@ -0,0 +1,83 @@ +import { spawn } from "node:child_process"; + +export default async function find( + path: string, + printf: string, + timeout?: number, +): Promise<{ + // the output as a Buffer (not utf8, since it could have arbitrary file names!) + stdout: Buffer; + // truncated is true if the timeout gets hit. + truncated: boolean; +}> { + if (!path) { + throw Error("path must be specified"); + } + if (!printf) { + throw Error("printf must be specified"); + } + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + let truncated = false; + + const args = [ + "-P", // Never follow symlinks (security) + path, // Search path + "-maxdepth", + "1", + "-mindepth", + "1", + "-printf", + printf, + ]; + + // Spawn find with minimal, fixed arguments + const child = spawn("find", args, { + stdio: ["ignore", "pipe", "pipe"], + env: {}, // Empty environment (security) + shell: false, // No shell interpretation (security) + }); + + let timer; + if (timeout) { + timer = setTimeout(() => { + if (!truncated) { + truncated = true; + child.kill("SIGTERM"); + } + }, timeout); + } else { + timer = null; + } + + child.stdout.on("data", (chunk: Buffer) => { + chunks.push(chunk); + }); + + let stderr = ""; + child.stderr.on("data", (chunk: Buffer) => { + stderr += chunk.toString(); + }); + + // Handle completion + child.on("error", (error) => { + if (timer) { + clearTimeout(timer); + } + reject(error); + }); + + child.on("exit", (code) => { + if (timer) { + clearTimeout(timer); + } + + if (code !== 0 && !truncated) { + reject(new Error(`find exited with code ${code}: ${stderr}`)); + return; + } + + resolve({ stdout: Buffer.concat(chunks), truncated }); + }); + }); +} diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index 27b4a800ad..979aca2a4c 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -64,12 +64,16 @@ import { } from "node:fs/promises"; import { watch } from "node:fs"; import { exists } from "@cocalc/backend/misc/async-utils-node"; -import { type DirectoryListingEntry } from "@cocalc/util/types"; -import getListing from "@cocalc/backend/get-listing"; import { join, resolve } from "path"; import { replace_all } from "@cocalc/util/misc"; import { EventIterator } from "@cocalc/util/event-iterator"; import { type WatchOptions } from "@cocalc/conat/files/watch"; +import find from "./find"; + +// max time a user find request can run -- this can cause excessive +// load on a server if there were a directory with a massive number of files, +// so must be limited. +const FIND_TIMEOUT = 3000; export class SandboxedFilesystem { // path should be the path to a FOLDER on the filesystem (not a file) @@ -151,6 +155,13 @@ export class SandboxedFilesystem { return await exists(await this.safeAbsPath(path)); }; + find = async ( + path: string, + printf: string, + ): Promise<{ stdout: Buffer; truncated: boolean }> => { + return await find(await this.safeAbsPath(path), printf, FIND_TIMEOUT); + }; + // hard link link = async (existingPath: string, newPath: string) => { return await link( @@ -159,16 +170,6 @@ export class SandboxedFilesystem { ); }; - ls = async ( - path: string, - { hidden, limit }: { hidden?: boolean; limit?: number } = {}, - ): Promise => { - return await getListing(await this.safeAbsPath(path), hidden, { - limit, - home: "/", - }); - }; - lstat = async (path: string) => { return await lstat(await this.safeAbsPath(path)); }; diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index 80b96ecccd..0f6746ebec 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -35,6 +35,15 @@ export interface Filesystem { writeFile: (path: string, data: string | Buffer) => Promise; // todo: typing watch: (path: string, options?) => Promise; + + // We add very little to the Filesystem api, but we have to add + // a sandboxed "find" command, since it is a 1-call way to get + // arbitrary directory listing info. + // find -P {path} -maxdepth 1 -mindepth 1 -printf {printf} + find: ( + path: string, + printf: string, + ) => Promise<{ stdout: Buffer; truncated: boolean }>; } interface IStats { @@ -129,6 +138,9 @@ export async function fsServer({ service, fs, client }: Options) { async exists(path: string) { return await (await fs(this.subject)).exists(path); }, + async find(path: string, printf: string) { + return await (await fs(this.subject)).find(path, printf); + }, async link(existingPath: string, newPath: string) { await (await fs(this.subject)).link(existingPath, newPath); }, From c4382823dcedc819f61496e0ecacc94e3cfd480d Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 22 Jul 2025 23:46:01 +0000 Subject: [PATCH 064/798] use reuseInFlight instead of 1000's of once listeners at once. - so we don't get that scary eventemitter warning. - I'm not technically sure this is actually better, but probably... --- src/packages/conat/core/cluster.ts | 3 +-- src/packages/conat/core/patterns.ts | 9 ++++++++- src/packages/conat/core/server.ts | 6 +++--- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/packages/conat/core/cluster.ts b/src/packages/conat/core/cluster.ts index 7db83c03c6..863f2cd943 100644 --- a/src/packages/conat/core/cluster.ts +++ b/src/packages/conat/core/cluster.ts @@ -7,7 +7,6 @@ import { type StickyUpdate, } from "@cocalc/conat/core/server"; import type { DStream } from "@cocalc/conat/sync/dstream"; -import { once } from "@cocalc/util/async-utils"; import { server as createPersistServer } from "@cocalc/conat/persist/server"; import { getLogger } from "@cocalc/conat/client"; import { hash_string } from "@cocalc/util/misc"; @@ -165,7 +164,7 @@ class ClusterLink { if (Date.now() - start >= timeout) { throw Error("timeout"); } - await once(this.interest, "change"); + await this.interest.waitForChange(); if ((this.state as any) == "closed" || signal?.aborted) { return false; } diff --git a/src/packages/conat/core/patterns.ts b/src/packages/conat/core/patterns.ts index 79eada9e5e..708df2afde 100644 --- a/src/packages/conat/core/patterns.ts +++ b/src/packages/conat/core/patterns.ts @@ -2,6 +2,8 @@ import { isEqual } from "lodash"; import { getLogger } from "@cocalc/conat/client"; import { EventEmitter } from "events"; import { hash_string } from "@cocalc/util/misc"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { once } from "@cocalc/util/async-utils"; type Index = { [pattern: string]: Index | string }; @@ -13,9 +15,14 @@ export class Patterns extends EventEmitter { constructor() { super(); - this.setMaxListeners(1000); + this.setMaxListeners(100); } + // wait until one single change event fires. Throws an error if this gets closed first. + waitForChange = reuseInFlight(async (timeout?) => { + await once(this, "change", timeout); + }); + close = () => { this.emit("closed"); this.patterns = {}; diff --git a/src/packages/conat/core/server.ts b/src/packages/conat/core/server.ts index dc87dcf191..e41135d2cf 100644 --- a/src/packages/conat/core/server.ts +++ b/src/packages/conat/core/server.ts @@ -53,7 +53,7 @@ import { import { Patterns } from "./patterns"; import { is_array } from "@cocalc/util/misc"; import { UsageMonitor } from "@cocalc/conat/monitor/usage"; -import { once, until } from "@cocalc/util/async-utils"; +import { until } from "@cocalc/util/async-utils"; import { clusterLink, type ClusterLink, @@ -1703,8 +1703,8 @@ export class ConatServer extends EventEmitter { throw Error("timeout"); } try { - // if signal is set only wait for the change for up to 1 second. - await once(this.interest, "change", signal != null ? 1000 : undefined); + // if signal is set, only wait for the change for up to 1 second. + await this.interest.waitForChange(signal != null ? 1000 : undefined); } catch { continue; } From 80224e933a472116189c39aec0d16d0007f385d3 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 00:17:27 +0000 Subject: [PATCH 065/798] fs sandbox: support for readonly and unsafe; ability to find files matching patterns --- .../conat/files/test/local-path.test.ts | 2 +- .../backend/files/sandbox/find.test.ts | 26 +++++- src/packages/backend/files/sandbox/find.ts | 88 ++++++++++++++++-- src/packages/backend/files/sandbox/index.ts | 89 +++++++++++++++++-- .../backend/files/sandbox/sandbox.test.ts | 88 +++++++++++++++++- src/packages/conat/files/fs.ts | 30 ++++++- 6 files changed, 304 insertions(+), 19 deletions(-) diff --git a/src/packages/backend/conat/files/test/local-path.test.ts b/src/packages/backend/conat/files/test/local-path.test.ts index 2c49add752..a05a7b297a 100644 --- a/src/packages/backend/conat/files/test/local-path.test.ts +++ b/src/packages/backend/conat/files/test/local-path.test.ts @@ -313,7 +313,7 @@ describe("use all the standard api functions of fs", () => { }); }); -describe("security: dangerous symlinks can't be followed", () => { +describe("SECURITY CHECKS: dangerous symlinks can't be followed", () => { let server; let tempDir; it("creates the simple fileserver service", async () => { diff --git a/src/packages/backend/files/sandbox/find.test.ts b/src/packages/backend/files/sandbox/find.test.ts index ca725e3569..f57ff4cd48 100644 --- a/src/packages/backend/files/sandbox/find.test.ts +++ b/src/packages/backend/files/sandbox/find.test.ts @@ -1,5 +1,5 @@ import find from "./find"; -import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -25,14 +25,34 @@ describe("find files", () => { expect(stdout.toString()).toEqual("a.txt\n"); }); + it("find files matching a given pattern", async () => { + await writeFile(join(tempDir, "pattern"), ""); + await mkdir(join(tempDir, "blue")); + await writeFile(join(tempDir, "blue", "Patton"), ""); + const { stdout } = await find(tempDir, "%f\n", { + expression: { type: "iname", pattern: "patt*" }, + }); + const v = stdout.toString().trim().split("\n"); + expect(new Set(v)).toEqual(new Set(["pattern"])); + }); + + it("find file in a subdirectory too", async () => { + const { stdout } = await find(tempDir, "%P\n", { + recursive: true, + expression: { type: "iname", pattern: "patt*" }, + }); + const w = stdout.toString().trim().split("\n"); + expect(new Set(w)).toEqual(new Set(["pattern", "blue/Patton"])); + }); + // this is NOT a great test, unfortunately. - const count = 10000; + const count = 5000; it(`hopefully exceed the timeout by creating ${count} files`, async () => { for (let i = 0; i < count; i++) { await writeFile(join(tempDir, `${i}`), ""); } const t = Date.now(); - const { stdout, truncated } = await find(tempDir, "%f\n", 2); + const { stdout, truncated } = await find(tempDir, "%f\n", { timeout: 0.1 }); expect(truncated).toBe(true); expect(Date.now() - t).toBeGreaterThan(1); diff --git a/src/packages/backend/files/sandbox/find.ts b/src/packages/backend/files/sandbox/find.ts index edc7047c37..90cd67c9bc 100644 --- a/src/packages/backend/files/sandbox/find.ts +++ b/src/packages/backend/files/sandbox/find.ts @@ -1,9 +1,11 @@ import { spawn } from "node:child_process"; +import type { FindOptions, FindExpression } from "@cocalc/conat/files/fs"; +export type { FindOptions, FindExpression }; export default async function find( path: string, printf: string, - timeout?: number, + { timeout = 0, recursive, expression }: FindOptions = {}, ): Promise<{ // the output as a Buffer (not utf8, since it could have arbitrary file names!) stdout: Buffer; @@ -23,13 +25,25 @@ export default async function find( const args = [ "-P", // Never follow symlinks (security) path, // Search path - "-maxdepth", - "1", "-mindepth", "1", - "-printf", - printf, ]; + if (!recursive) { + args.push("-maxdepth", "1"); + } + + // Add expression if provided + if (expression) { + try { + args.push(...buildFindArgs(expression)); + } catch (error) { + reject(error); + return; + } + } + args.push("-printf", printf); + + //console.log(`find ${args.join(" ")}`); // Spawn find with minimal, fixed arguments const child = spawn("find", args, { @@ -81,3 +95,67 @@ export default async function find( }); }); } + +function buildFindArgs(expr: FindExpression): string[] { + switch (expr.type) { + case "name": + // Validate pattern has no path separators + if (expr.pattern.includes("/")) { + throw new Error("Path separators not allowed in name patterns"); + } + return ["-name", expr.pattern]; + + case "iname": + if (expr.pattern.includes("/")) { + throw new Error("Path separators not allowed in name patterns"); + } + return ["-iname", expr.pattern]; + + case "type": + return ["-type", expr.value]; + + case "size": + // Validate size format (e.g., "10M", "1G", "500k") + if (!/^[0-9]+[kMGTP]?$/.test(expr.value)) { + throw new Error("Invalid size format"); + } + return ["-size", expr.operator + expr.value]; + + case "mtime": + if (!Number.isInteger(expr.days) || expr.days < 0) { + throw new Error("Invalid mtime days"); + } + return ["-mtime", expr.operator + expr.days]; + + case "newer": + // This is risky - would need to validate file path is within sandbox + if (expr.file.includes("..") || expr.file.startsWith("/")) { + throw new Error("Invalid reference file path"); + } + return ["-newer", expr.file]; + + case "and": + return [ + "(", + ...buildFindArgs(expr.left), + "-a", + ...buildFindArgs(expr.right), + ")", + ]; + + case "or": + return [ + "(", + ...buildFindArgs(expr.left), + "-o", + ...buildFindArgs(expr.right), + ")", + ]; + + case "not": + return ["!", ...buildFindArgs(expr.expr)]; + + default: + throw new Error("Unsupported expression type"); + } +} diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index 979aca2a4c..b65ba1ff3f 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -68,18 +68,44 @@ import { join, resolve } from "path"; import { replace_all } from "@cocalc/util/misc"; import { EventIterator } from "@cocalc/util/event-iterator"; import { type WatchOptions } from "@cocalc/conat/files/watch"; -import find from "./find"; +import find, { type FindOptions } from "./find"; // max time a user find request can run -- this can cause excessive // load on a server if there were a directory with a massive number of files, // so must be limited. -const FIND_TIMEOUT = 3000; +const MAX_FIND_TIMEOUT = 3000; + +interface Options { + // unsafeMode -- if true, assume security model where user is running this + // themself, e.g., in a project, so no security is needed at all. + unsafeMode?: boolean; + // readonly -- only allow operations that don't change files + readonly?: boolean; +} + +// If you add any methods below that are NOT for the public api +// be sure to exclude them here! +const INTERNAL_METHODS = new Set([ + "safeAbsPath", + "constructor", + "path", + "unsafeMode", + "readonly", + "assertWritable", +]); export class SandboxedFilesystem { - // path should be the path to a FOLDER on the filesystem (not a file) - constructor(public readonly path: string) { + public readonly unsafeMode: boolean; + public readonly readonly: boolean; + constructor( + // path should be the path to a FOLDER on the filesystem (not a file) + public readonly path: string, + { unsafeMode = false, readonly = false }: Options = {}, + ) { + this.unsafeMode = !!unsafeMode; + this.readonly = !!readonly; for (const f in this) { - if (f == "safeAbsPath" || f == "constructor" || f == "path") { + if (INTERNAL_METHODS.has(f)) { continue; } const orig = this[f]; @@ -99,12 +125,26 @@ export class SandboxedFilesystem { } } + private assertWritable = (path: string) => { + if (this.readonly) { + throw new SandboxError( + `EACCES: permission denied -- read only filesystem, open '${path}'`, + { errno: -13, code: "EACCES", syscall: "open", path }, + ); + } + }; + safeAbsPath = async (path: string): Promise => { if (typeof path != "string") { throw Error(`path must be a string but is of type ${typeof path}`); } // pathInSandbox is *definitely* a path in the sandbox: const pathInSandbox = join(this.path, resolve("/", path)); + + if (this.unsafeMode) { + // not secure -- just convenient. + return pathInSandbox; + } // However, there is still one threat, which is that it could // be a path to an existing link that goes out of the sandbox. So // we resolve to the realpath: @@ -128,10 +168,12 @@ export class SandboxedFilesystem { }; appendFile = async (path: string, data: string | Buffer, encoding?) => { + this.assertWritable(path); return await appendFile(await this.safeAbsPath(path), data, encoding); }; chmod = async (path: string, mode: string | number) => { + this.assertWritable(path); await chmod(await this.safeAbsPath(path), mode); }; @@ -140,10 +182,12 @@ export class SandboxedFilesystem { }; copyFile = async (src: string, dest: string) => { + this.assertWritable(dest); await copyFile(await this.safeAbsPath(src), await this.safeAbsPath(dest)); }; cp = async (src: string, dest: string, options?) => { + this.assertWritable(dest); await cp( await this.safeAbsPath(src), await this.safeAbsPath(dest), @@ -158,12 +202,22 @@ export class SandboxedFilesystem { find = async ( path: string, printf: string, + options?: FindOptions, ): Promise<{ stdout: Buffer; truncated: boolean }> => { - return await find(await this.safeAbsPath(path), printf, FIND_TIMEOUT); + options = { ...options }; + if ( + !this.unsafeMode && + (!options.timeout || options.timeout > MAX_FIND_TIMEOUT) + ) { + options.timeout = MAX_FIND_TIMEOUT; + } + + return await find(await this.safeAbsPath(path), printf, options); }; // hard link link = async (existingPath: string, newPath: string) => { + this.assertWritable(newPath); return await link( await this.safeAbsPath(existingPath), await this.safeAbsPath(newPath), @@ -175,6 +229,7 @@ export class SandboxedFilesystem { }; mkdir = async (path: string, options?) => { + this.assertWritable(path); await mkdir(await this.safeAbsPath(path), options); }; @@ -192,6 +247,7 @@ export class SandboxedFilesystem { }; rename = async (oldPath: string, newPath: string) => { + this.assertWritable(oldPath); await rename( await this.safeAbsPath(oldPath), await this.safeAbsPath(newPath), @@ -199,10 +255,12 @@ export class SandboxedFilesystem { }; rm = async (path: string, options?) => { + this.assertWritable(path); await rm(await this.safeAbsPath(path), options); }; rmdir = async (path: string, options?) => { + this.assertWritable(path); await rmdir(await this.safeAbsPath(path), options); }; @@ -211,6 +269,7 @@ export class SandboxedFilesystem { }; symlink = async (target: string, path: string) => { + this.assertWritable(target); return await symlink( await this.safeAbsPath(target), await this.safeAbsPath(path), @@ -218,10 +277,12 @@ export class SandboxedFilesystem { }; truncate = async (path: string, len?: number) => { + this.assertWritable(path); await truncate(await this.safeAbsPath(path), len); }; unlink = async (path: string) => { + this.assertWritable(path); await unlink(await this.safeAbsPath(path)); }; @@ -230,6 +291,7 @@ export class SandboxedFilesystem { atime: number | string | Date, mtime: number | string | Date, ) => { + this.assertWritable(path); await utimes(await this.safeAbsPath(path), atime, mtime); }; @@ -257,6 +319,21 @@ export class SandboxedFilesystem { }; writeFile = async (path: string, data: string | Buffer) => { + this.assertWritable(path); return await writeFile(await this.safeAbsPath(path), data); }; } + +export class SandboxError extends Error { + code: string; + errno: number; + syscall: string; + path: string; + constructor(mesg: string, { code, errno, syscall, path }) { + super(mesg); + this.code = code; + this.errno = errno; + this.syscall = syscall; + this.path = path; + } +} diff --git a/src/packages/backend/files/sandbox/sandbox.test.ts b/src/packages/backend/files/sandbox/sandbox.test.ts index ebabcb5dba..2676f5bbe4 100644 --- a/src/packages/backend/files/sandbox/sandbox.test.ts +++ b/src/packages/backend/files/sandbox/sandbox.test.ts @@ -1,5 +1,12 @@ import { SandboxedFilesystem } from "@cocalc/backend/files/sandbox"; -import { mkdtemp, mkdir, rm } from "node:fs/promises"; +import { + mkdtemp, + mkdir, + rm, + readFile, + symlink, + writeFile, +} from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "path"; @@ -16,6 +23,7 @@ describe("test using the filesystem sandbox to do a few standard things", () => await fs.writeFile("a", "hi"); const r = await fs.readFile("a", "utf8"); expect(r).toEqual("hi"); + expect(fs.unsafeMode).toBe(false); }); it("truncate file", async () => { @@ -166,6 +174,84 @@ describe("test watching a file and a folder in the sandbox", () => { }); }); +describe("unsafe mode sandbox", () => { + let fs; + it("creates and reads file", async () => { + await mkdir(join(tempDir, "test-unsafe")); + fs = new SandboxedFilesystem(join(tempDir, "test-unsafe"), { + unsafeMode: true, + }); + expect(fs.unsafeMode).toBe(true); + await fs.writeFile("a", "hi"); + const r = await fs.readFile("a", "utf8"); + expect(r).toEqual("hi"); + }); + + it("directly create a dangerous file that is a symlink outside of the sandbox -- this should work", async () => { + await writeFile(join(tempDir, "password"), "s3cr3t"); + await symlink( + join(tempDir, "password"), + join(tempDir, "test-unsafe", "danger"), + ); + const s = await readFile(join(tempDir, "test-unsafe", "danger"), "utf8"); + expect(s).toBe("s3cr3t"); + }); + + it("can **UNSAFELY** read the symlink content via the api", async () => { + expect(await fs.readFile("danger", "utf8")).toBe("s3cr3t"); + }); +}); + +describe("safe mode sandbox", () => { + let fs; + it("creates and reads file", async () => { + await mkdir(join(tempDir, "test-safe")); + fs = new SandboxedFilesystem(join(tempDir, "test-safe"), { + unsafeMode: false, + }); + expect(fs.unsafeMode).toBe(false); + expect(fs.readonly).toBe(false); + await fs.writeFile("a", "hi"); + const r = await fs.readFile("a", "utf8"); + expect(r).toEqual("hi"); + }); + + it("directly create a dangerous file that is a symlink outside of the sandbox -- this should work", async () => { + await writeFile(join(tempDir, "password"), "s3cr3t"); + await symlink( + join(tempDir, "password"), + join(tempDir, "test-safe", "danger"), + ); + const s = await readFile(join(tempDir, "test-safe", "danger"), "utf8"); + expect(s).toBe("s3cr3t"); + }); + + it("cannot read the symlink content via the api", async () => { + await expect(async () => { + await fs.readFile("danger", "utf8"); + }).rejects.toThrow("outside of sandbox"); + }); +}); + +describe("read only sandbox", () => { + let fs; + it("creates and reads file", async () => { + await mkdir(join(tempDir, "test-ro")); + fs = new SandboxedFilesystem(join(tempDir, "test-ro"), { + readonly: true, + }); + expect(fs.readonly).toBe(true); + await expect(async () => { + await fs.writeFile("a", "hi"); + }).rejects.toThrow("permission denied -- read only filesystem"); + try { + await fs.writeFile("a", "hi"); + } catch (err) { + expect(err.code).toEqual("EACCES"); + } + }); +}); + afterAll(async () => { await rm(tempDir, { force: true, recursive: true }); }); diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index 0f6746ebec..c820c6dc1b 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -7,6 +7,28 @@ import { } from "@cocalc/conat/files/watch"; export const DEFAULT_FILE_SERVICE = "fs"; +export interface FindOptions { + // timeout is very limited (e.g., 3s?) if fs is running on file + // server (not in own project) + timeout?: number; + // recursive is false by default (unlike actual find command) + recursive?: boolean; + // see typing below -- we can't just pass arbitrary args since + // that would not be secure. + expression?: FindExpression; +} + +export type FindExpression = + | { type: "name"; pattern: string } + | { type: "iname"; pattern: string } + | { type: "type"; value: "f" | "d" | "l" } + | { type: "size"; operator: "+" | "-"; value: string } + | { type: "mtime"; operator: "+" | "-"; days: number } + | { type: "newer"; file: string } + | { type: "and"; left: FindExpression; right: FindExpression } + | { type: "or"; left: FindExpression; right: FindExpression } + | { type: "not"; expr: FindExpression }; + export interface Filesystem { appendFile: (path: string, data: string | Buffer, encoding?) => Promise; chmod: (path: string, mode: string | number) => Promise; @@ -38,11 +60,13 @@ export interface Filesystem { // We add very little to the Filesystem api, but we have to add // a sandboxed "find" command, since it is a 1-call way to get - // arbitrary directory listing info. + // arbitrary directory listing info, which is just not possible + // with the fs API, but required in any serious application. // find -P {path} -maxdepth 1 -mindepth 1 -printf {printf} find: ( path: string, printf: string, + options?: FindOptions, ) => Promise<{ stdout: Buffer; truncated: boolean }>; } @@ -138,8 +162,8 @@ export async function fsServer({ service, fs, client }: Options) { async exists(path: string) { return await (await fs(this.subject)).exists(path); }, - async find(path: string, printf: string) { - return await (await fs(this.subject)).find(path, printf); + async find(path: string, printf: string, options?: FindOptions) { + return await (await fs(this.subject)).find(path, printf, options); }, async link(existingPath: string, newPath: string) { await (await fs(this.subject)).link(existingPath, newPath); From 06ddce723290a880b1f3be08a4ddd65bc47d8cb6 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 01:33:10 +0000 Subject: [PATCH 066/798] update file-server to not use fs.ls --- src/packages/file-server/btrfs/snapshots.ts | 6 +-- .../file-server/btrfs/subvolume-snapshots.ts | 13 +++--- .../file-server/btrfs/test/filesystem.test.ts | 4 +- .../btrfs/test/subvolume-stress.test.ts | 16 ++++---- .../file-server/btrfs/test/subvolume.test.ts | 40 +++++++++---------- 5 files changed, 38 insertions(+), 41 deletions(-) diff --git a/src/packages/file-server/btrfs/snapshots.ts b/src/packages/file-server/btrfs/snapshots.ts index daf8e09212..99d0856245 100644 --- a/src/packages/file-server/btrfs/snapshots.ts +++ b/src/packages/file-server/btrfs/snapshots.ts @@ -50,9 +50,9 @@ export async function updateRollingSnapshots({ } // get exactly the iso timestamp snapshot names: - const snapshotNames = (await snapshots.ls()) - .map((x) => x.name) - .filter((name) => DATE_REGEXP.test(name)); + const snapshotNames = (await snapshots.readdir()).filter((name) => + DATE_REGEXP.test(name), + ); snapshotNames.sort(); if (snapshotNames.length > 0) { const age = Date.now() - new Date(snapshotNames.slice(-1)[0]).valueOf(); diff --git a/src/packages/file-server/btrfs/subvolume-snapshots.ts b/src/packages/file-server/btrfs/subvolume-snapshots.ts index 9dcd4f30ad..ddcc3ca2e1 100644 --- a/src/packages/file-server/btrfs/subvolume-snapshots.ts +++ b/src/packages/file-server/btrfs/subvolume-snapshots.ts @@ -2,7 +2,6 @@ import { type Subvolume } from "./subvolume"; import { btrfs } from "./util"; import getLogger from "@cocalc/backend/logger"; import { join } from "path"; -import { type DirectoryListingEntry } from "@cocalc/util/types"; import { SnapshotCounts, updateRollingSnapshots } from "./snapshots"; export const SNAPSHOTS = ".snapshots"; @@ -48,9 +47,9 @@ export class SubvolumeSnapshots { }); }; - ls = async (): Promise => { + readdir = async (): Promise => { await this.makeSnapshotsDir(); - return await this.subvolume.fs.ls(SNAPSHOTS, { hidden: false }); + return await this.subvolume.fs.readdir(SNAPSHOTS); }; lock = async (name: string) => { @@ -85,18 +84,18 @@ export class SubvolumeSnapshots { // has newly written changes since last snapshot hasUnsavedChanges = async (): Promise => { - const s = await this.ls(); + const s = await this.readdir(); if (s.length == 0) { // more than just the SNAPSHOTS directory? - const v = await this.subvolume.fs.ls("", { hidden: true }); - if (v.length == 0 || (v.length == 1 && v[0].name == SNAPSHOTS)) { + const v = await this.subvolume.fs.readdir(""); + if (v.length == 0 || (v.length == 1 && v[0] == SNAPSHOTS)) { return false; } return true; } const pathGen = await getGeneration(this.subvolume.path); const snapGen = await getGeneration( - join(this.snapshotsDir, s[s.length - 1].name), + join(this.snapshotsDir, s[s.length - 1]), ); return snapGen < pathGen; }; diff --git a/src/packages/file-server/btrfs/test/filesystem.test.ts b/src/packages/file-server/btrfs/test/filesystem.test.ts index 673980af89..e0693b16bd 100644 --- a/src/packages/file-server/btrfs/test/filesystem.test.ts +++ b/src/packages/file-server/btrfs/test/filesystem.test.ts @@ -30,7 +30,7 @@ describe("operations with subvolumes", () => { const vol = await fs.subvolumes.get("cocalc"); expect(vol.name).toBe("cocalc"); // it has no snapshots - expect(await vol.snapshots.ls()).toEqual([]); + expect(await vol.snapshots.readdir()).toEqual([]); }); it("our subvolume is in the list", async () => { @@ -97,7 +97,7 @@ describe("clone of a subvolume with snapshots should have no snapshots", () => { it("clone has no snapshots", async () => { const clone = await fs.subvolumes.get("my-clone"); expect(await clone.fs.readFile("abc.txt", "utf8")).toEqual("hi"); - expect(await clone.snapshots.ls()).toEqual([]); + expect(await clone.snapshots.readdir()).toEqual([]); await clone.snapshots.create("my-clone-snap"); }); }); diff --git a/src/packages/file-server/btrfs/test/subvolume-stress.test.ts b/src/packages/file-server/btrfs/test/subvolume-stress.test.ts index 29bc048e69..69279d9b04 100644 --- a/src/packages/file-server/btrfs/test/subvolume-stress.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume-stress.test.ts @@ -29,16 +29,16 @@ describe(`stress test creating ${numSnapshots} snapshots`, () => { `created ${Math.round((numSnapshots / (Date.now() - start)) * 1000)} snapshots per second in serial`, ); snaps.sort(); - expect((await vol.snapshots.ls()).map(({ name }) => name).sort()).toEqual( - snaps.sort(), - ); + expect( + (await vol.snapshots.readdir()).filter((x) => !x.startsWith(".")).sort(), + ).toEqual(snaps.sort()); }); it(`delete our ${numSnapshots} snapshots`, async () => { for (let i = 0; i < numSnapshots; i++) { await vol.snapshots.delete(`snap${i}`); } - expect(await vol.snapshots.ls()).toEqual([]); + expect(await vol.snapshots.readdir()).toEqual([]); }); }); @@ -58,9 +58,8 @@ describe(`create ${numFiles} files`, () => { log( `created ${Math.round((numFiles / (Date.now() - start)) * 1000)} files per second in serial`, ); - const v = await vol.fs.ls(""); - const w = v.map(({ name }) => name); - expect(w.sort()).toEqual(names.sort()); + const v = await vol.fs.readdir(""); + expect(v.sort()).toEqual(names.sort()); }); it(`creates ${numFiles} files in parallel`, async () => { @@ -77,9 +76,8 @@ describe(`create ${numFiles} files`, () => { `created ${Math.round((numFiles / (Date.now() - start)) * 1000)} files per second in parallel`, ); const t0 = Date.now(); - const v = await vol.fs.ls("p"); + const w = await vol.fs.readdir("p"); log("get listing of files took", Date.now() - t0, "ms"); - const w = v.map(({ name }) => name); expect(w.sort()).toEqual(names.sort()); }); }); diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index 456282d8b3..1736f17c4f 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -19,7 +19,7 @@ describe("setting and getting quota of a subvolume", () => { }); it("get directory listing", async () => { - const v = await vol.fs.ls(""); + const v = await vol.fs.readdir(""); expect(v).toEqual([]); }); @@ -36,9 +36,8 @@ describe("setting and getting quota of a subvolume", () => { const { used } = await vol.quota.usage(); expect(used).toBeGreaterThan(0); - const v = await vol.fs.ls(""); - // size is potentially random, reflecting compression - expect(v).toEqual([{ name: "buf", mtime: v[0].mtime, size: v[0].size }]); + const v = await vol.fs.readdir(""); + expect(v).toEqual(["buf"]); }); it("fail to write a 50MB file (due to quota)", async () => { @@ -54,20 +53,20 @@ describe("the filesystem operations", () => { it("creates a volume and get empty listing", async () => { vol = await fs.subvolumes.get("fs"); - expect(await vol.fs.ls("")).toEqual([]); + expect(await vol.fs.readdir("")).toEqual([]); }); it("error listing non-existent path", async () => { vol = await fs.subvolumes.get("fs"); expect(async () => { - await vol.fs.ls("no-such-path"); + await vol.fs.readdir("no-such-path"); }).rejects.toThrow("ENOENT"); }); it("creates a text file to it", async () => { await vol.fs.writeFile("a.txt", "hello"); - const ls = await vol.fs.ls(""); - expect(ls).toEqual([{ name: "a.txt", mtime: ls[0].mtime, size: 5 }]); + const ls = await vol.fs.readdir(""); + expect(ls).toEqual(["a.txt"]); }); it("read the file we just created as utf8", async () => { @@ -87,17 +86,18 @@ describe("the filesystem operations", () => { let origStat; it("snapshot filesystem and see file is in snapshot", async () => { await vol.snapshots.create("snap"); - const s = await vol.fs.ls(vol.snapshots.path("snap")); - expect(s).toEqual([{ name: "a.txt", mtime: s[0].mtime, size: 5 }]); + const s = await vol.fs.readdir(vol.snapshots.path("snap")); + expect(s).toContain("a.txt"); - const stat = await vol.fs.stat("a.txt"); - origStat = stat; - expect(stat.mtimeMs / 1000).toBeCloseTo(s[0].mtime ?? 0); + const stat0 = await vol.fs.stat(vol.snapshots.path("snap")); + const stat1 = await vol.fs.stat("a.txt"); + origStat = stat1; + expect(stat1.mtimeMs).toBeCloseTo(stat0.mtimeMs, -2); }); it("unlink (delete) our file", async () => { await vol.fs.unlink("a.txt"); - expect(await vol.fs.ls("")).toEqual([]); + expect(await vol.fs.readdir("")).toEqual([".snapshots"]); }); it("snapshot still exists", async () => { @@ -185,9 +185,9 @@ describe("test snapshots", () => { }); it("snapshot the volume", async () => { - expect(await vol.snapshots.ls()).toEqual([]); + expect(await vol.snapshots.readdir()).toEqual([]); await vol.snapshots.create("snap1"); - expect((await vol.snapshots.ls()).map((x) => x.name)).toEqual(["snap1"]); + expect(await vol.snapshots.readdir()).toEqual(["snap1"]); expect(await vol.snapshots.hasUnsavedChanges()).toBe(false); }); @@ -223,7 +223,7 @@ describe("test snapshots", () => { await vol.snapshots.unlock("snap1"); await vol.snapshots.delete("snap1"); expect(await vol.snapshots.exists("snap1")).toBe(false); - expect(await vol.snapshots.ls()).toEqual([]); + expect(await vol.snapshots.readdir()).toEqual([]); }); }); @@ -266,7 +266,7 @@ describe("test bup backups", () => { it("add a directory and back up", async () => { await mkdir(join(vol.path, "mydir")); await vol.fs.writeFile(join("mydir", "file.txt"), "hello3"); - expect((await vol.fs.ls("mydir"))[0].name).toBe("file.txt"); + expect((await vol.fs.readdir("mydir"))[0]).toBe("file.txt"); await vol.bup.save(); const x = await vol.bup.ls("latest"); expect(x).toEqual([ @@ -287,8 +287,8 @@ describe("test bup backups", () => { }); it("most recent snapshot has a backup before the restore", async () => { - const s = await vol.snapshots.ls(); - const recent = s.slice(-1)[0].name; + const s = await vol.snapshots.readdir(); + const recent = s.slice(-1)[0]; const p = vol.snapshots.path(recent, "mydir", "file.txt"); expect(await vol.fs.readFile(p, "utf8")).toEqual("changed"); }); From c32f46984873cdb1f0bf84dcaac16dc040cd99a7 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 01:40:39 +0000 Subject: [PATCH 067/798] fix the packages/sync tests --- .../sync/editor/string/test/README.md | 5 ++++ .../sync/editor/string/test/client-test.ts | 8 ++++++ .../string/test/ephemeral-syncstring.ts | 3 ++- .../sync/editor/string/test/sync.0.test.ts | 4 +-- .../sync/editor/string/test/sync.1.test.ts | 25 ++++++------------- 5 files changed, 25 insertions(+), 20 deletions(-) create mode 100644 src/packages/sync/editor/string/test/README.md diff --git a/src/packages/sync/editor/string/test/README.md b/src/packages/sync/editor/string/test/README.md new file mode 100644 index 0000000000..0a064bf05c --- /dev/null +++ b/src/packages/sync/editor/string/test/README.md @@ -0,0 +1,5 @@ +There is additional _integration_ testing of the sync code in: + +``` +packages/backend/conat/test/sync-doc +``` \ No newline at end of file diff --git a/src/packages/sync/editor/string/test/client-test.ts b/src/packages/sync/editor/string/test/client-test.ts index b9d43e012a..0c28027621 100644 --- a/src/packages/sync/editor/string/test/client-test.ts +++ b/src/packages/sync/editor/string/test/client-test.ts @@ -187,3 +187,11 @@ export class Client extends EventEmitter implements Client0 { console.log(`shell: opts=${JSON.stringify(opts)}`); } } + +class Filesystem { + readFile = () => ""; + writeFile = () => {}; + utimes = () => {}; +} + +export const fs = new Filesystem() as any; diff --git a/src/packages/sync/editor/string/test/ephemeral-syncstring.ts b/src/packages/sync/editor/string/test/ephemeral-syncstring.ts index 81cb8001f0..086fa9a154 100644 --- a/src/packages/sync/editor/string/test/ephemeral-syncstring.ts +++ b/src/packages/sync/editor/string/test/ephemeral-syncstring.ts @@ -3,7 +3,7 @@ This is useful not just for testing, but also for implementing undo/redo for editing a text document when there is no actual file or project involved. */ -import { Client } from "./client-test"; +import { Client, fs } from "./client-test"; import { SyncString } from "../sync"; import { a_txt } from "./data"; import { once } from "@cocalc/util/async-utils"; @@ -16,6 +16,7 @@ export default async function ephemeralSyncstring() { path, client, ephemeral: true, + fs, }); // replace save to disk, since otherwise unless string is empty, // this will hang forever... and it is called on close. diff --git a/src/packages/sync/editor/string/test/sync.0.test.ts b/src/packages/sync/editor/string/test/sync.0.test.ts index 9d5289dbd1..82f080dac9 100644 --- a/src/packages/sync/editor/string/test/sync.0.test.ts +++ b/src/packages/sync/editor/string/test/sync.0.test.ts @@ -11,7 +11,7 @@ pnpm test sync.0.test.ts */ -import { Client } from "./client-test"; +import { Client, fs } from "./client-test"; import { SyncString } from "../sync"; import { a_txt } from "./data"; import { once } from "@cocalc/util/async-utils"; @@ -23,7 +23,7 @@ describe("create a blank minimal string SyncDoc and call public methods on it", let syncstring: SyncString; it("creates the syncstring and wait for it to be ready", async () => { - syncstring = new SyncString({ project_id, path, client }); + syncstring = new SyncString({ project_id, path, client, fs }); expect(syncstring.get_state()).toBe("init"); await once(syncstring, "ready"); expect(syncstring.get_state()).toBe("ready"); diff --git a/src/packages/sync/editor/string/test/sync.1.test.ts b/src/packages/sync/editor/string/test/sync.1.test.ts index a58fa9d3a4..842c85c610 100644 --- a/src/packages/sync/editor/string/test/sync.1.test.ts +++ b/src/packages/sync/editor/string/test/sync.1.test.ts @@ -12,7 +12,7 @@ pnpm test sync.1.test.ts */ -import { Client } from "./client-test"; +import { Client, fs } from "./client-test"; import { SyncString } from "../sync"; import { once } from "@cocalc/util/async-utils"; import { a_txt } from "./data"; @@ -28,7 +28,13 @@ describe("create syncstring and test doing some edits", () => { ]; it("creates the syncstring and wait until ready", async () => { - syncstring = new SyncString({ project_id, path, client, cursors: true }); + syncstring = new SyncString({ + project_id, + path, + client, + cursors: true, + fs, + }); expect(syncstring.get_state()).toBe("init"); await once(syncstring, "ready"); }); @@ -97,21 +103,6 @@ describe("create syncstring and test doing some edits", () => { expect(syncstring.is_read_only()).toBe(false); }); - it("save to disk", async () => { - expect(syncstring.has_unsaved_changes()).toBe(true); - const promise = syncstring.save_to_disk(); - // Mock: we set save to done in the syncstring - // table, otherwise the promise will never resolve. - (syncstring as any).set_save({ - state: "done", - error: "", - hash: syncstring.hash_of_live_version(), - }); - (syncstring as any).syncstring_table.emit("change-no-throttle"); - await promise; - expect(syncstring.has_unsaved_changes()).toBe(false); - }); - it("close and clean up", async () => { await syncstring.close(); expect(syncstring.get_state()).toBe("closed"); From 17077cf8ff515c9c414750ac2ab4c67dea6028a0 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 05:07:02 +0000 Subject: [PATCH 068/798] implement core of new listings --- .../backend/conat/files/test/listing.test.ts | 92 +++++++++++++ src/packages/conat/files/fs.ts | 7 + src/packages/conat/files/listing.ts | 126 ++++++++++++++++++ 3 files changed, 225 insertions(+) create mode 100644 src/packages/backend/conat/files/test/listing.test.ts create mode 100644 src/packages/conat/files/listing.ts diff --git a/src/packages/backend/conat/files/test/listing.test.ts b/src/packages/backend/conat/files/test/listing.test.ts new file mode 100644 index 0000000000..d19b459d94 --- /dev/null +++ b/src/packages/backend/conat/files/test/listing.test.ts @@ -0,0 +1,92 @@ +import { SandboxedFilesystem } from "@cocalc/backend/files/sandbox"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "path"; +import { randomId } from "@cocalc/conat/names"; +import listing from "@cocalc/conat/files/listing"; + +let tmp; +beforeAll(async () => { + tmp = await mkdtemp(join(tmpdir(), `cocalc-${randomId()}0`)); +}); + +afterAll(async () => { + try { + await rm(tmp, { force: true, recursive: true }); + } catch {} +}); + +describe("creating a listing monitor starting with an empty directory", () => { + let fs, dir; + it("creates sandboxed filesystem", async () => { + fs = new SandboxedFilesystem(tmp); + dir = await listing({ path: "", fs }); + }); + + it("initial listing is empty", () => { + expect(Object.keys(dir.files)).toEqual([]); + }); + + let iter; + it("create a file and get an update", async () => { + iter = dir.iter(); + await fs.writeFile("a.txt", "hello"); + let { value } = await iter.next(); + expect(value).toEqual({ + mtime: value.mtime, + name: "a.txt", + size: value.size, + }); + // it's possible that the file isn't written completely above. + if (value.size != 5) { + ({ value } = await iter.next()); + } + const stat = await fs.stat("a.txt"); + expect(stat.mtimeMs).toEqual(value.mtime); + expect(dir.files["a.txt"]).toEqual({ mtime: value.mtime, size: 5 }); + }); + + it("modify the file and get two updates -- one when it starts and another when done", async () => { + await fs.appendFile("a.txt", " there"); + const { value } = await iter.next(); + expect(value).toEqual({ mtime: value.mtime, name: "a.txt", size: 5 }); + const { value: value2 } = await iter.next(); + expect(value2).toEqual({ mtime: value2.mtime, name: "a.txt", size: 11 }); + const stat = await fs.stat("a.txt"); + expect(stat.mtimeMs).toEqual(value2.mtime); + expect(dir.files["a.txt"]).toEqual({ mtime: value2.mtime, size: 11 }); + }); + + it("create another monitor starting with the now nonempty directory", async () => { + const dir2 = await listing({ path: "", fs }); + expect(Object.keys(dir.files)).toEqual(["a.txt"]); + dir2.close(); + }); + + const count = 500; + it(`creates ${count} files and see they are found`, async () => { + const n = Object.keys(dir.files).length; + + for (let i = 0; i < count; i++) { + await fs.writeFile(`${i}`, ""); + } + const values: string[] = []; + while (true) { + const { value } = await iter.next(); + if (value == "a.txt") { + continue; + } + values.push(value); + if (value.name == `${count - 1}`) { + break; + } + } + expect(new Set(values).size).toEqual(count); + + expect(Object.keys(dir.files).length).toEqual(n + count); + }); + + it("cleans up", () => { + dir.close(); + }); +}); diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index c820c6dc1b..2c30db142f 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -5,6 +5,8 @@ import { watchClient, type WatchIterator, } from "@cocalc/conat/files/watch"; +import listing, { type Listing } from "./listing"; + export const DEFAULT_FILE_SERVICE = "fs"; export interface FindOptions { @@ -68,6 +70,8 @@ export interface Filesystem { printf: string, options?: FindOptions, ) => Promise<{ stdout: Buffer; truncated: boolean }>; + + listing?: (path: string) => Promise; } interface IStats { @@ -286,6 +290,9 @@ export function fsClient({ await ensureWatchServerExists(path, options); return await watchClient({ client, subject, path, options }); }; + call.listing = async (path: string) => { + return await listing({ fs: call, path }); + }; return call; } diff --git a/src/packages/conat/files/listing.ts b/src/packages/conat/files/listing.ts new file mode 100644 index 0000000000..a82c7afd20 --- /dev/null +++ b/src/packages/conat/files/listing.ts @@ -0,0 +1,126 @@ +/* +Directory Listing + +Tests in packages/backend/conat/files/test/listing.test.ts +*/ + +import { EventEmitter } from "events"; +import { join } from "path"; +import { type Filesystem } from "./fs"; +import { EventIterator } from "@cocalc/util/event-iterator"; + +interface FileData { + mtime: number; + size: number; +} + +type Files = { [name: string]: FileData }; + +interface Options { + path: string; + fs: Filesystem; +} + +export default async function listing(opts: Options): Promise { + const listing = new Listing(opts); + await listing.init(); + return listing; +} + +export class Listing extends EventEmitter { + public files?: Files = {}; + public truncated?: boolean; + private watch?; + private iters: EventIterator[] = []; + constructor(public readonly opts: Options) { + super(); + } + + iter = () => { + const iter = new EventIterator(this, "change", { + map: (args) => { + return { name: args[0], ...args[1] }; + }, + }); + this.iters.push(iter); + return iter; + }; + + close = () => { + this.emit("closed"); + this.iters.map((iter) => iter.end()); + this.iters.length = 0; + this.watch?.close(); + delete this.files; + delete this.watch; + }; + + init = async () => { + const { fs, path } = this.opts; + this.watch = await fs.watch(path); + const { files, truncated } = await getListing(fs, path); + this.files = files; + this.truncated = truncated; + this.emit("ready"); + this.handleUpdates(); + }; + + private handleUpdates = async () => { + for await (const { filename } of this.watch) { + if (this.files == null) { + return; + } + this.update(filename); + } + }; + + private update = async (filename: string) => { + if (this.files == null) { + // closed or not initialized yet + return; + } + try { + const stats = await this.opts.fs.stat(join(this.opts.path, filename)); + if (this.files == null) { + return; + } + this.files[filename] = { mtime: stats.mtimeMs, size: stats.size }; + } catch (err) { + if (err.code == "ENOENT") { + // file deleted + delete this.files[filename]; + } else { + console.warn("WARNING:", err); + // TODO: some other error -- e.g., network down or permissions, so we don't know anything. + // Should we retry (?). + return; + } + } + this.emit("change", filename, this.files[filename]); + }; +} + +async function getListing( + fs: Filesystem, + path: string, +): Promise<{ files: Files; truncated: boolean }> { + const { stdout, truncated } = await fs.find(path, "%f\\0%T@\\0%s\n"); + const buf = Buffer.from(stdout); + const files: Files = {}; + // todo -- what about non-utf8...? + + const s = buf.toString().trim(); + if (!s) { + return { files, truncated }; + } + for (const line of s.split("\n")) { + try { + const v = line.split("\0"); + const name = v[0]; + const mtime = parseFloat(v[1]) * 1000; + const size = parseInt(v[2]); + files[name] = { mtime, size }; + } catch {} + } + return { files, truncated }; +} From 7fbde56e14cd42efcc4d95861d6833846e98e5e7 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 13:46:35 +0000 Subject: [PATCH 069/798] timetravel -- implement opening timetravel doc in a much more straightforward explicit way: basically, you need to open the main document - it was implicit and complicated before; now just open the main file as a background tab. Simple and much easier to get right. --- .../frame-editors/code-editor/actions.ts | 2 +- .../time-travel-editor/actions.ts | 73 +++++---- src/packages/sync/client/sync-client.ts | 147 +----------------- 3 files changed, 52 insertions(+), 170 deletions(-) diff --git a/src/packages/frontend/frame-editors/code-editor/actions.ts b/src/packages/frontend/frame-editors/code-editor/actions.ts index 28f025e153..744008bf83 100644 --- a/src/packages/frontend/frame-editors/code-editor/actions.ts +++ b/src/packages/frontend/frame-editors/code-editor/actions.ts @@ -445,7 +445,7 @@ export class Actions< // Flag that there is activity (causes icon to turn orange). private activity = (): void => { - this._get_project_actions().flag_file_activity(this.path); + this._get_project_actions()?.flag_file_activity(this.path); }; // This is currently NOT used in this base class. It's used in other diff --git a/src/packages/frontend/frame-editors/time-travel-editor/actions.ts b/src/packages/frontend/frame-editors/time-travel-editor/actions.ts index edbe640391..1448177aad 100644 --- a/src/packages/frontend/frame-editors/time-travel-editor/actions.ts +++ b/src/packages/frontend/frame-editors/time-travel-editor/actions.ts @@ -21,7 +21,6 @@ import { List } from "immutable"; import { once } from "@cocalc/util/async-utils"; import { filename_extension, path_split } from "@cocalc/util/misc"; import { SyncDoc } from "@cocalc/sync/editor/generic/sync-doc"; -import { webapp_client } from "../../webapp-client"; import { exec } from "@cocalc/frontend/frame-editors/generic/client"; import { ViewDocument } from "./view-document"; import { @@ -32,8 +31,8 @@ import { FrameTree } from "../frame-tree/types"; import { export_to_json } from "./export-to-json"; import type { Document } from "@cocalc/sync/editor/generic/types"; import LRUCache from "lru-cache"; -import { syncdbPath } from "@cocalc/util/jupyter/names"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { until } from "@cocalc/util/async-utils"; const EXTENSION = ".time-travel"; @@ -81,7 +80,6 @@ export class TimeTravelActions extends CodeEditorActions { protected doctype: string = "none"; // actual document is managed elsewhere private docpath: string; private docext: string; - private syncpath: string; syncdoc?: SyncDoc; private first_load: boolean = true; ambient_actions?: CodeEditorActions; @@ -96,11 +94,7 @@ export class TimeTravelActions extends CodeEditorActions { this.docpath = head + "/" + this.docpath; } // log("init", { path: this.path }); - this.syncpath = this.docpath; this.docext = filename_extension(this.docpath); - if (this.docext == "ipynb") { - this.syncpath = syncdbPath(this.docpath); - } this.setState({ versions: List([]), loading: true, @@ -118,27 +112,50 @@ export class TimeTravelActions extends CodeEditorActions { init_frame_tree = () => {}; - close = (): void => { - if (this.syncdoc != null) { - this.syncdoc.close(); - delete this.syncdoc; - } - super.close(); - }; - set_error = (error) => { this.setState({ error }); }; private init_syncdoc = async (): Promise => { - const persistent = this.docext == "ipynb" || this.docext == "sagews"; // ugly for now (?) - this.syncdoc = await webapp_client.sync_client.open_existing_sync_document({ - project_id: this.project_id, - path: this.syncpath, - persistent, + let mainFileActions: any = null; + await until(async () => { + if (this.isClosed()) { + return true; + } + mainFileActions = this.redux.getEditorActions( + this.project_id, + this.docpath, + ); + console.log("mainFileActions", mainFileActions != null); + if (mainFileActions == null) { + console.log("opening file"); + // open the file that we're showing timetravel for, so that the + // actions are available + try { + await this.open_file({ foreground: false, explicit: false }); + } catch (err) { + console.warn(err); + } + // will try again above in the next loop + return false; + } else { + const doc = mainFileActions._syncstring; + if (doc == null || doc.get_state() == "closed") { + // file is closing + return false; + } + // got it! + return true; + } }); - if (this.syncdoc == null) return; - this.syncdoc.on("change", debounce(this.syncdoc_changed, 1000)); + if (this.isClosed() || mainFileActions == null) { + return; + } + this.syncdoc = mainFileActions._syncstring; + + if (this.syncdoc == null || this.syncdoc.get_state() == "closed") { + return; + } if (this.syncdoc.get_state() != "ready") { try { await once(this.syncdoc, "ready"); @@ -146,14 +163,16 @@ export class TimeTravelActions extends CodeEditorActions { return; } } - if (this.syncdoc == null) return; + this.syncdoc.on("change", debounce(this.syncdoc_changed, 750)); // cause initial load -- we could be plugging into an already loaded syncdoc, // so there wouldn't be any change event, so we have to trigger this. this.syncdoc_changed(); this.syncdoc.on("close", () => { - // in our code we don't check if the state is closed, but instead - // that this.syncdoc is not null. + console.log("in timetravel, syncdoc was closed"); + // in the actions in this file, we don't check if the state is closed, but instead + // that this.syncdoc is not null: delete this.syncdoc; + this.init_syncdoc(); }); this.setState({ @@ -289,10 +308,10 @@ export class TimeTravelActions extends CodeEditorActions { } }; - open_file = async (): Promise => { + open_file = async (opts?): Promise => { // log("open_file"); const actions = this.redux.getProjectActions(this.project_id); - await actions.open_file({ path: this.docpath, foreground: true }); + await actions.open_file({ path: this.docpath, foreground: true, ...opts }); }; // Revert the live version of the document to a specific version */ diff --git a/src/packages/sync/client/sync-client.ts b/src/packages/sync/client/sync-client.ts index d3287e9295..2edd1eb008 100644 --- a/src/packages/sync/client/sync-client.ts +++ b/src/packages/sync/client/sync-client.ts @@ -8,9 +8,8 @@ Functionality related to Sync. */ import { once } from "@cocalc/util/async-utils"; -import { defaults, required } from "@cocalc/util/misc"; import { SyncDoc, SyncOpts0 } from "@cocalc/sync/editor/generic/sync-doc"; -import { SyncDB, SyncDBOpts0 } from "@cocalc/sync/editor/db"; +import { SyncDBOpts0 } from "@cocalc/sync/editor/db"; import { SyncString } from "@cocalc/sync/editor/string/sync"; import { synctable, @@ -20,7 +19,6 @@ import { synctable_no_changefeed, } from "@cocalc/sync/table"; import type { AppClient } from "./types"; -import { getSyncDocType } from "@cocalc/conat/sync/syncdoc-info"; interface SyncOpts extends Omit { noCache?: boolean; @@ -71,146 +69,11 @@ export class SyncClient { ); } - // These are not working properly, e.g., if you close and open - // a LARGE jupyter notebook quickly (so save to disk takes a while), - // then it gets broken until browser refresh. The problem is that - // the doc is still closing right when a new one starts being created. - // So for now we just revert to the non-cached-here approach. - // There is other caching elsewhere. - - // public sync_string(opts: SyncOpts): SyncString { - // return syncstringCache({ ...opts, client: this.client }); - // } - - // public sync_db(opts: SyncDBOpts): SyncDB { - // return syncdbCache({ ...opts, client: this.client }); - // } - - public sync_string(opts: SyncOpts): SyncString { - const opts0: SyncOpts0 = defaults(opts, { - id: undefined, - project_id: required, - path: required, - file_use_interval: "default", - cursors: false, - patch_interval: 1000, - save_interval: 2000, - persistent: false, - data_server: undefined, - client: this.client, - ephemeral: false, - fs: undefined, - }); - return new SyncString(opts0); - } - - public sync_db(opts: SyncDBOpts): SyncDoc { - const opts0: SyncDBOpts0 = defaults(opts, { - id: undefined, - project_id: required, - path: required, - file_use_interval: "default", - cursors: false, - patch_interval: 1000, - save_interval: 2000, - change_throttle: undefined, - persistent: false, - data_server: undefined, - - primary_keys: required, - string_cols: [], - - client: this.client, - - ephemeral: false, - - fs: undefined, - }); - return new SyncDB(opts0); + public sync_string(_opts: SyncOpts): SyncString { + throw Error("deprecated"); } - public async open_existing_sync_document({ - project_id, - path, - data_server, - persistent, - }: { - project_id: string; - path: string; - data_server?: string; - persistent?: boolean; - }): Promise { - const doctype = await getSyncDocType({ - project_id, - path, - client: this.client, - }); - const { type } = doctype; - const f = `sync_${type}`; - return (this as any)[f]({ - project_id, - path, - data_server, - persistent, - ...doctype.opts, - }); + public sync_db(_opts: SyncDBOpts): SyncDoc { + throw Error("deprecated"); } } - -/* -const syncdbCache = refCacheSync({ - name: "syncdb", - - createKey: ({ project_id, path }: SyncDBOpts) => { - return JSON.stringify({ project_id, path }); - }, - - createObject: (opts: SyncDBOpts) => { - const opts0: SyncDBOpts0 = defaults(opts, { - id: undefined, - project_id: required, - path: required, - file_use_interval: "default", - cursors: false, - patch_interval: 1000, - save_interval: 2000, - change_throttle: undefined, - persistent: false, - data_server: undefined, - - primary_keys: required, - string_cols: [], - - client: required, - - ephemeral: false, - }); - return new SyncDB(opts0); - }, -}); - -const syncstringCache = refCacheSync({ - name: "syncstring", - createKey: ({ project_id, path }: SyncOpts) => { - const key = JSON.stringify({ project_id, path }); - return key; - }, - - createObject: (opts: SyncOpts) => { - const opts0: SyncOpts0 = defaults(opts, { - id: undefined, - project_id: required, - path: required, - file_use_interval: "default", - cursors: false, - patch_interval: 1000, - save_interval: 2000, - persistent: false, - data_server: undefined, - client: required, - ephemeral: false, - }); - return new SyncString(opts0); - }, -}); -*/ From b011fdfe9053db4572617574d458e066d6fde3bd Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 14:51:37 +0000 Subject: [PATCH 070/798] file opening in frontend -- fix leak I had just caused by opening background tabs --- src/packages/frontend/file-editors.ts | 4 +- .../time-travel-editor/actions.ts | 3 -- src/packages/frontend/project/open-file.ts | 6 +-- src/packages/frontend/project_actions.ts | 49 ++++++++++--------- 4 files changed, 31 insertions(+), 31 deletions(-) diff --git a/src/packages/frontend/file-editors.ts b/src/packages/frontend/file-editors.ts index ed692f555f..aed28b2fbc 100644 --- a/src/packages/frontend/file-editors.ts +++ b/src/packages/frontend/file-editors.ts @@ -259,9 +259,9 @@ export async function remove( save(path, redux, project_id, is_public); } - if (!is_public) { + if (!is_public && project_id) { // Also free the corresponding side chat, if it was created. - require("./chat/register").remove( + (await import("./chat/register")).remove( meta_file(path, "chat"), redux, project_id, diff --git a/src/packages/frontend/frame-editors/time-travel-editor/actions.ts b/src/packages/frontend/frame-editors/time-travel-editor/actions.ts index 1448177aad..2ef79fc2c6 100644 --- a/src/packages/frontend/frame-editors/time-travel-editor/actions.ts +++ b/src/packages/frontend/frame-editors/time-travel-editor/actions.ts @@ -126,9 +126,7 @@ export class TimeTravelActions extends CodeEditorActions { this.project_id, this.docpath, ); - console.log("mainFileActions", mainFileActions != null); if (mainFileActions == null) { - console.log("opening file"); // open the file that we're showing timetravel for, so that the // actions are available try { @@ -168,7 +166,6 @@ export class TimeTravelActions extends CodeEditorActions { // so there wouldn't be any change event, so we have to trigger this. this.syncdoc_changed(); this.syncdoc.on("close", () => { - console.log("in timetravel, syncdoc was closed"); // in the actions in this file, we don't check if the state is closed, but instead // that this.syncdoc is not null: delete this.syncdoc; diff --git a/src/packages/frontend/project/open-file.ts b/src/packages/frontend/project/open-file.ts index c905f6914b..d5d20be4b2 100644 --- a/src/packages/frontend/project/open-file.ts +++ b/src/packages/frontend/project/open-file.ts @@ -347,16 +347,14 @@ export async function open_file( return; } - if (PRELOAD_BACKGROUND_TABS) { - await actions.initFileRedux(opts.path); - } - if (opts.foreground) { actions.foreground_project(opts.change_history); const tab = path_to_tab(opts.path); actions.set_active_tab(tab, { change_history: opts.change_history, }); + } else if (PRELOAD_BACKGROUND_TABS) { + await actions.initFileRedux(opts.path); } if (alreadyOpened && opts.fragmentId) { diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index c6ca0fed22..803fbde4a0 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -12,7 +12,6 @@ import { List, Map, fromJS, Set as immutableSet } from "immutable"; import { isEqual, throttle } from "lodash"; import { join } from "path"; import { defineMessage } from "react-intl"; - import { computeServerManager, type ComputeServerManager, @@ -1061,28 +1060,34 @@ export class ProjectActions extends Actions { }; /* Initialize the redux store and react component for editing - a particular file. + a particular file, if necessary. */ - initFileRedux = async ( - path: string, - is_public: boolean = false, - ext?: string, // use this extension even instead of path's extension. - ): Promise => { - // LAZY IMPORT, so that editors are only available - // when you are going to use them. Helps with code splitting. - await import("./editors/register-all"); - - // Initialize the file's store and actions - const name = await project_file.initializeAsync( - path, - this.redux, - this.project_id, - is_public, - undefined, - ext, - ); - return name; - }; + initFileRedux = reuseInFlight( + async ( + path: string, + is_public: boolean = false, + ext?: string, // use this extension even instead of path's extension. + ): Promise => { + const cur = redux.getEditorActions(this.project_id, path); + if (cur != null) { + return cur.name; + } + // LAZY IMPORT, so that editors are only available + // when you are going to use them. Helps with code splitting. + await import("./editors/register-all"); + + // Initialize the file's store and actions + const name = await project_file.initializeAsync( + path, + this.redux, + this.project_id, + is_public, + undefined, + ext, + ); + return name; + }, + ); private init_file_react_redux = async ( path: string, From 3bd88b8fb85841caaadd9be0ca40198f94029208 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 14:57:16 +0000 Subject: [PATCH 071/798] github ci: fix test failing due to missing yapf --- .github/workflows/make-and-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/make-and-test.yml b/.github/workflows/make-and-test.yml index 1498f67653..325c8e5031 100644 --- a/.github/workflows/make-and-test.yml +++ b/.github/workflows/make-and-test.yml @@ -39,7 +39,7 @@ jobs: detached: true - uses: actions/checkout@v4 - name: Install python3 requests - run: sudo apt-get install python3-requests + run: sudo apt-get install python3-requests python3-yapf - name: Check doc links run: cd src/scripts && python3 check_doc_urls.py || sleep 5 || python3 check_doc_urls.py @@ -99,7 +99,7 @@ jobs: python3 -m pip install --upgrade pip virtualenv python3 -m virtualenv venv source venv/bin/activate - pip install ipykernel + pip install ipykernel yapf python -m ipykernel install --prefix=./jupyter-local --name python3-local --display-name "Python 3 (Local)" From cabf024a86737d5162e2d227ec74d40fcb5139bc Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 16:46:48 +0000 Subject: [PATCH 072/798] create a new frontend integration test package and one tiny little test; starting work in listings that can be used from the frontend --- .../conat/files/test/local-path.test.ts | 5 +- src/packages/backend/package.json | 17 +--- src/packages/conat/files/fs.ts | 8 +- src/packages/conat/files/listing.ts | 2 +- .../frontend/project/listing/use-listing.ts | 86 +++++++++++++++++++ src/packages/pnpm-lock.yaml | 46 ++++++++-- src/packages/test/jest.config.js | 13 +++ src/packages/test/package.json | 37 ++++++++ src/packages/test/test/setup.js | 5 ++ src/packages/test/tsconfig.json | 22 +++++ src/packages/test/use-listing.test.ts | 33 +++++++ src/packages/util/async-utils.ts | 6 +- src/workspaces.py | 1 + 13 files changed, 257 insertions(+), 24 deletions(-) create mode 100644 src/packages/frontend/project/listing/use-listing.ts create mode 100644 src/packages/test/jest.config.js create mode 100644 src/packages/test/package.json create mode 100644 src/packages/test/test/setup.js create mode 100644 src/packages/test/tsconfig.json create mode 100644 src/packages/test/use-listing.test.ts diff --git a/src/packages/backend/conat/files/test/local-path.test.ts b/src/packages/backend/conat/files/test/local-path.test.ts index a05a7b297a..43825d45bc 100644 --- a/src/packages/backend/conat/files/test/local-path.test.ts +++ b/src/packages/backend/conat/files/test/local-path.test.ts @@ -4,7 +4,10 @@ import { fsClient } from "@cocalc/conat/files/fs"; import { randomId } from "@cocalc/conat/names"; import { before, after } from "@cocalc/backend/conat/test/setup"; import { uuid } from "@cocalc/util/misc"; -import { createPathFileserver, cleanupFileservers } from "./util"; +import { + createPathFileserver, + cleanupFileservers, +} from "@cocalc/backend/conat/files/test/util"; beforeAll(before); diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index 4a71ea0a22..48c602b8a1 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -13,10 +13,7 @@ "./auth/*": "./dist/auth/*.js", "./auth/tokens/*": "./dist/auth/tokens/*.js" }, - "keywords": [ - "utilities", - "cocalc" - ], + "keywords": ["utilities", "cocalc"], "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf dist node_modules", @@ -24,7 +21,6 @@ "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", "test": "pnpm exec jest --forceExit", "test-conat": " pnpm exec jest --forceExit conat", - "testp": "pnpm exec jest --forceExit", "depcheck": "pnpx depcheck --ignores events", "prepublishOnly": "pnpm test", "conat-watch": "node ./bin/conat-watch.cjs", @@ -34,20 +30,13 @@ "conat-persist": "DEBUG=cocalc:* node ./bin/conat-persist.cjs", "conat-test-server": "node ./bin/conat-test-server.cjs" }, - "files": [ - "dist/**", - "bin/**", - "README.md", - "package.json" - ], + "files": ["dist/**", "bin/**", "README.md", "package.json"], "author": "SageMath, Inc.", "license": "SEE LICENSE.md", "dependencies": { "@cocalc/backend": "workspace:*", "@cocalc/conat": "workspace:*", "@cocalc/util": "workspace:*", - "@types/debug": "^4.1.12", - "@types/jest": "^29.5.14", "awaiting": "^3.0.0", "better-sqlite3": "^11.10.0", "chokidar": "^3.6.0", @@ -68,6 +57,8 @@ }, "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/backend", "devDependencies": { + "@types/debug": "^4.1.12", + "@types/jest": "^29.5.14", "@types/node": "^18.16.14" } } diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index 2c30db142f..c522b29b24 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -252,15 +252,19 @@ export async function fsServer({ service, fs, client }: Options) { }; } +export type FilesystemClient = Filesystem & { + listing: (path: string) => Promise; +}; + export function fsClient({ client, subject, }: { client?: Client; subject: string; -}): Filesystem { +}): FilesystemClient { client ??= conat(); - let call = client.call(subject); + let call = client.call(subject); let constants: any = null; const stat0 = call.stat.bind(call); diff --git a/src/packages/conat/files/listing.ts b/src/packages/conat/files/listing.ts index a82c7afd20..c2d8313fdb 100644 --- a/src/packages/conat/files/listing.ts +++ b/src/packages/conat/files/listing.ts @@ -14,7 +14,7 @@ interface FileData { size: number; } -type Files = { [name: string]: FileData }; +export type Files = { [name: string]: FileData }; interface Options { path: string; diff --git a/src/packages/frontend/project/listing/use-listing.ts b/src/packages/frontend/project/listing/use-listing.ts new file mode 100644 index 0000000000..e29318d872 --- /dev/null +++ b/src/packages/frontend/project/listing/use-listing.ts @@ -0,0 +1,86 @@ +/* +A directory listing hook. +*/ + +import { useMemo, useState } from "react"; +import { DirectoryListingEntry } from "@cocalc/util/types"; +import useAsyncEffect from "use-async-effect"; +import { throttle } from "lodash"; +import { field_cmp } from "@cocalc/util/misc"; +import { type Files } from "@cocalc/conat/files/listing"; +import { type FilesystemClient } from "@cocalc/conat/files/fs"; + +const DEFAULT_THROTTLE_FILE_UPDATE = 500; + +type SortField = "name" | "mtime" | "size"; +type SortDirection = "inc" | "dec"; + +export default function useListing({ + fs, + path, + sortField = "name", + sortDirection = "inc", +}: { + fs: FilesystemClient; + path: string; + sortField?: SortField; + sortDirection?: SortDirection; +}): { listing: null | DirectoryListingEntry[]; error } { + const { files, error } = useFiles({ fs, path }); + + const listing = useMemo(() => { + if (files == null) { + return null; + } + const v: DirectoryListingEntry[] = []; + for (const name in files) { + v.push({ name, ...files[name] }); + } + v.sort(field_cmp("name")); + if (sortDirection == "dec") { + v.reverse(); + } + return v; + }, [sortField, sortDirection, files]); + + return { listing, error }; +} + +export function useFiles({ + fs, + path, + throttleUpdate = DEFAULT_THROTTLE_FILE_UPDATE, +}: { + fs: FilesystemClient; + path: string; + throttleUpdate?: number; +}): { files: Files | null; error } { + const [files, setFiles] = useState(null); + const [error, setError] = useState(null); + + useAsyncEffect(async () => { + let listing; + try { + listing = await fs.listing(path); + } catch (err) { + setError(err); + return; + } + + const update = () => { + setFiles({ ...listing.files }); + }; + update(); + + listing.on( + "change", + throttle(update, throttleUpdate, { leading: true, trailing: true }), + ); + + return () => { + listing.close(); + }; + }, [fs, path]); + + return { files, error }; +} diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index db43c158d4..901cc56769 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -87,12 +87,6 @@ importers: '@cocalc/util': specifier: workspace:* version: link:../util - '@types/debug': - specifier: ^4.1.12 - version: 4.1.12 - '@types/jest': - specifier: ^29.5.14 - version: 29.5.14 awaiting: specifier: ^3.0.0 version: 3.0.0 @@ -133,6 +127,12 @@ importers: specifier: ^0.0.10 version: 0.0.10 devDependencies: + '@types/debug': + specifier: ^4.1.12 + version: 4.1.12 + '@types/jest': + specifier: ^29.5.14 + version: 29.5.14 '@types/node': specifier: ^18.16.14 version: 18.19.118 @@ -1765,6 +1765,40 @@ importers: specifier: ^18.16.14 version: 18.19.118 + test: + dependencies: + '@cocalc/backend': + specifier: workspace:* + version: link:../backend + '@cocalc/conat': + specifier: workspace:* + version: link:../conat + '@cocalc/frontend': + specifier: workspace:* + version: link:../frontend + '@cocalc/util': + specifier: workspace:* + version: link:../util + devDependencies: + '@testing-library/jest-dom': + specifier: ^6.6.3 + version: 6.6.3 + '@testing-library/react': + specifier: ^16.3.0 + version: 16.3.0(@testing-library/dom@10.4.0)(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) + '@types/debug': + specifier: ^4.1.12 + version: 4.1.12 + '@types/jest': + specifier: ^29.5.14 + version: 29.5.14 + '@types/node': + specifier: ^18.16.14 + version: 18.19.118 + jest-environment-jsdom: + specifier: ^30.0.2 + version: 30.0.4 + util: dependencies: '@ant-design/colors': diff --git a/src/packages/test/jest.config.js b/src/packages/test/jest.config.js new file mode 100644 index 0000000000..3e9e290535 --- /dev/null +++ b/src/packages/test/jest.config.js @@ -0,0 +1,13 @@ +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + preset: "ts-jest", + testEnvironment: "jsdom", + testEnvironmentOptions: { + // needed or jest imports the ts directly rather than the compiled + // dist exported from our package.json. Without this imports won't work. + // See https://jestjs.io/docs/configuration#testenvironment-string + customExportConditions: ["node", "node-addons"], + }, + testMatch: ["**/?(*.)+(spec|test).ts?(x)"], + setupFilesAfterEnv: ["./test/setup.js"], +}; diff --git a/src/packages/test/package.json b/src/packages/test/package.json new file mode 100644 index 0000000000..bed4234e0c --- /dev/null +++ b/src/packages/test/package.json @@ -0,0 +1,37 @@ +{ + "name": "@cocalc/test", + "version": "1.0.0", + "description": "CoCalc Integration Testing", + "exports": { + "./*": "./dist/*.js" + }, + "keywords": ["test", "cocalc"], + "scripts": { + "preinstall": "npx only-allow pnpm", + "clean": "rm -rf dist node_modules", + "build": "pnpm exec tsc --build", + "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", + "test": "pnpm exec jest --forceExit", + "depcheck": "pnpx depcheck --ignores events" + }, + "author": "SageMath, Inc.", + "license": "SEE LICENSE.md", + "dependencies": { + "@cocalc/backend": "workspace:*", + "@cocalc/conat": "workspace:*", + "@cocalc/frontend": "workspace:*", + "@cocalc/util": "workspace:*" + }, + "repository": { + "type": "git", + "url": "https://github.com/sagemathinc/cocalc" + }, + "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/test", + "devDependencies": { + "@types/node": "^18.16.14", + "@types/debug": "^4.1.12", + "@types/jest": "^29.5.14", + "@testing-library/jest-dom": "^6.6.3", + "jest-environment-jsdom": "^30.0.2" + } +} diff --git a/src/packages/test/test/setup.js b/src/packages/test/test/setup.js new file mode 100644 index 0000000000..fb85d307c6 --- /dev/null +++ b/src/packages/test/test/setup.js @@ -0,0 +1,5 @@ +require("@testing-library/jest-dom"); +process.env.COCALC_TEST_MODE = true; + +global.TextEncoder = require("util").TextEncoder; +global.TextDecoder = require("util").TextDecoder; diff --git a/src/packages/test/tsconfig.json b/src/packages/test/tsconfig.json new file mode 100644 index 0000000000..6e3f1bef81 --- /dev/null +++ b/src/packages/test/tsconfig.json @@ -0,0 +1,22 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "composite": true, + "incremental": true, + "rootDir": "./", + "outDir": "dist", + "sourceMap": true, + "lib": ["es5", "es6", "es2017", "dom"], + "declaration": true + }, + "exclude": ["dist", "node_modules"], + "types": ["jest", "@testing-library/jest-dom"], + "references": [ + { + "path": "../util", + "path": "../conat", + "path": "../backend", + "path": "../frontend" + } + ] +} diff --git a/src/packages/test/use-listing.test.ts b/src/packages/test/use-listing.test.ts new file mode 100644 index 0000000000..465d4d088a --- /dev/null +++ b/src/packages/test/use-listing.test.ts @@ -0,0 +1,33 @@ +import { renderHook } from "@testing-library/react"; +import { fsClient } from "@cocalc/conat/files/fs"; +import { before, after } from "@cocalc/backend/conat/test/setup"; +import { uuid } from "@cocalc/util/misc"; +import { + createPathFileserver, + cleanupFileservers, +} from "@cocalc/backend/conat/files/test/util"; +import { useFiles } from "@cocalc/frontend/project/listing/use-listing"; + +beforeAll(before); + +describe("use all the standard api functions of fs", () => { + const project_id = uuid(); + let fs, server; + it("creates fileserver service and fs client", async () => { + server = await createPathFileserver(); + fs = fsClient({ subject: `${server.service}.project-${project_id}` }); + }); + + it("test useFiles", async () => { + const f = () => { + return useFiles({ fs, path: "", throttleUpdate: 0 }); + }; + const { result } = renderHook(f); + expect(result.current).toEqual({ files: null, error: null }); + }); +}); + +afterAll(async () => { + await after(); + await cleanupFileservers(); +}); diff --git a/src/packages/util/async-utils.ts b/src/packages/util/async-utils.ts index ac40a124e1..d5142926e4 100644 --- a/src/packages/util/async-utils.ts +++ b/src/packages/util/async-utils.ts @@ -184,7 +184,11 @@ function captureStackWithoutPrinting() { If the obj throws 'closed' before the event is emitted, then this throws an error, since clearly event can never be emitted. */ -const DEBUG_ONCE = false; // log a better stack trace in some cases + +// Set DEBUG_ONCE to true and see a MUCH better stack trace about what +// caused once to throw in some cases! Do not leave this on though, +// since it uses extra time and memory grabbing a stack trace on every call. +const DEBUG_ONCE = false; export async function once( obj: EventEmitter, event: string, diff --git a/src/workspaces.py b/src/workspaces.py index 738782d291..53011ac7f6 100755 --- a/src/workspaces.py +++ b/src/workspaces.py @@ -126,6 +126,7 @@ def all_packages() -> List[str]: 'packages/file-server', 'packages/next', 'packages/hub', # hub won't build if next isn't already built + 'packages/test' ] for x in os.listdir('packages'): path = os.path.join("packages", x) From 8a226398f7301a8d4688e4425cd9756ebe54b637 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 17:40:50 +0000 Subject: [PATCH 073/798] test: actually unit testing the useFiles hook --- src/packages/conat/files/listing.ts | 5 + .../frontend/project/listing/use-listing.ts | 19 ++-- src/packages/test/use-listing.test.ts | 98 +++++++++++++++++-- 3 files changed, 107 insertions(+), 15 deletions(-) diff --git a/src/packages/conat/files/listing.ts b/src/packages/conat/files/listing.ts index c2d8313fdb..63d9bab6c9 100644 --- a/src/packages/conat/files/listing.ts +++ b/src/packages/conat/files/listing.ts @@ -86,13 +86,18 @@ export class Listing extends EventEmitter { } this.files[filename] = { mtime: stats.mtimeMs, size: stats.size }; } catch (err) { + if (this.files == null) { + return; + } if (err.code == "ENOENT") { // file deleted delete this.files[filename]; } else { + //if (!process.env.COCALC_TEST_MODE) { console.warn("WARNING:", err); // TODO: some other error -- e.g., network down or permissions, so we don't know anything. // Should we retry (?). + //} return; } } diff --git a/src/packages/frontend/project/listing/use-listing.ts b/src/packages/frontend/project/listing/use-listing.ts index e29318d872..d36c676e11 100644 --- a/src/packages/frontend/project/listing/use-listing.ts +++ b/src/packages/frontend/project/listing/use-listing.ts @@ -25,8 +25,12 @@ export default function useListing({ path: string; sortField?: SortField; sortDirection?: SortDirection; -}): { listing: null | DirectoryListingEntry[]; error } { - const { files, error } = useFiles({ fs, path }); +}): { + listing: null | DirectoryListingEntry[]; + error: null | Error; + refresh: () => void; +} { + const { files, error, refresh } = useFiles({ fs, path }); const listing = useMemo(() => { if (files == null) { @@ -43,7 +47,7 @@ export default function useListing({ return v; }, [sortField, sortDirection, files]); - return { listing, error }; + return { listing, error, refresh }; } export function useFiles({ @@ -54,16 +58,19 @@ export function useFiles({ fs: FilesystemClient; path: string; throttleUpdate?: number; -}): { files: Files | null; error } { +}): { files: Files | null; error: null | Error; refresh: () => void } { const [files, setFiles] = useState(null); const [error, setError] = useState(null); + const [counter, setCounter] = useState(0); useAsyncEffect(async () => { let listing; try { listing = await fs.listing(path); + setError(null); } catch (err) { setError(err); + setFiles(null); return; } @@ -80,7 +87,7 @@ export function useFiles({ return () => { listing.close(); }; - }, [fs, path]); + }, [fs, path, counter]); - return { files, error }; + return { files, error, refresh: () => setCounter(counter + 1) }; } diff --git a/src/packages/test/use-listing.test.ts b/src/packages/test/use-listing.test.ts index 465d4d088a..831a0fd692 100644 --- a/src/packages/test/use-listing.test.ts +++ b/src/packages/test/use-listing.test.ts @@ -1,6 +1,6 @@ -import { renderHook } from "@testing-library/react"; +import { act, renderHook, waitFor } from "@testing-library/react"; import { fsClient } from "@cocalc/conat/files/fs"; -import { before, after } from "@cocalc/backend/conat/test/setup"; +import { before, after, wait } from "@cocalc/backend/conat/test/setup"; import { uuid } from "@cocalc/util/misc"; import { createPathFileserver, @@ -10,7 +10,7 @@ import { useFiles } from "@cocalc/frontend/project/listing/use-listing"; beforeAll(before); -describe("use all the standard api functions of fs", () => { +describe("the useFiles hook", () => { const project_id = uuid(); let fs, server; it("creates fileserver service and fs client", async () => { @@ -18,12 +18,92 @@ describe("use all the standard api functions of fs", () => { fs = fsClient({ subject: `${server.service}.project-${project_id}` }); }); - it("test useFiles", async () => { - const f = () => { - return useFiles({ fs, path: "", throttleUpdate: 0 }); - }; - const { result } = renderHook(f); - expect(result.current).toEqual({ files: null, error: null }); + it("test useFiles and file creation", async () => { + let path = "", + fs2 = fs; + const { result, rerender } = renderHook(() => + useFiles({ fs: fs2, path, throttleUpdate: 0 }), + ); + + expect(result.current).toEqual({ + files: null, + error: null, + refresh: expect.any(Function), + }); + + // eventually it will be initialized to not be null + await waitFor(() => { + expect(result.current.files).not.toBeNull(); + }); + expect(result.current).toEqual({ + files: {}, + error: null, + refresh: expect.any(Function), + }); + + // now write a file + await act(async () => { + await fs.writeFile("hello.txt", "world"); + }); + + await waitFor(() => { + expect(result.current.files["hello.txt"]).toBeDefined(); + }); + + expect(result.current).toEqual({ + files: { + "hello.txt": { + size: 5, + mtime: expect.any(Number), + }, + }, + error: null, + refresh: expect.any(Function), + }); + + // change the path to one that does not exist and rerender, + // resulting in an ENOENT error + path = "scratch"; + rerender(); + await waitFor(() => { + expect(result.current.files?.["hello.txt"]).not.toBeDefined(); + }); + expect(result.current.error.code).toBe("ENOENT"); + + await act(async () => { + // create the path, a file in there, refresh and it works + await fs.mkdir(path); + await fs.writeFile("scratch/b.txt", "hi"); + result.current.refresh(); + }); + + await waitFor(() => { + expect(result.current).toEqual({ + files: { + "b.txt": { + size: 2, + mtime: expect.any(Number), + }, + }, + error: null, + refresh: expect.any(Function), + }); + }); + + // change fs and see the hook update + const project_id2 = uuid(); + fs2 = fsClient({ + subject: `${server.service}.project-${project_id2}`, + }); + path = ""; + rerender(); + await waitFor(() => { + expect(result.current).toEqual({ + files: {}, + error: null, + refresh: expect.any(Function), + }); + }); }); }); From db04858ddae2796c7a96863097b258b51b82b566 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 17:58:15 +0000 Subject: [PATCH 074/798] add basic test of use-listing --- src/packages/conat/core/client.ts | 8 +- .../frontend/project/listing/use-files.ts | 58 +++++++++ .../frontend/project/listing/use-listing.ts | 59 ++-------- .../listing/use-files.test.ts} | 10 +- .../test/project/listing/use-listing.test.ts | 110 ++++++++++++++++++ 5 files changed, 186 insertions(+), 59 deletions(-) create mode 100644 src/packages/frontend/project/listing/use-files.ts rename src/packages/test/{use-listing.test.ts => project/listing/use-files.test.ts} (90%) create mode 100644 src/packages/test/project/listing/use-listing.test.ts diff --git a/src/packages/conat/core/client.ts b/src/packages/conat/core/client.ts index e11b2ec0a7..adb360d216 100644 --- a/src/packages/conat/core/client.ts +++ b/src/packages/conat/core/client.ts @@ -1963,7 +1963,7 @@ export function messageData( export type Subscription = EventIterator; export class ConatError extends Error { - code: string | number; + code?: string | number; constructor(mesg: string, { code }) { super(mesg); this.code = code; @@ -1989,15 +1989,15 @@ function toConatError(socketIoError) { } } -export function headerToError(headers) { +export function headerToError(headers): ConatError { const err = Error(headers.error); if (headers.error_attrs) { for (const field in headers.error_attrs) { err[field] = headers.error_attrs[field]; } } - if (err['code'] === undefined && headers.code) { - err['code'] = headers.code; + if (err["code"] === undefined && headers.code) { + err["code"] = headers.code; } return err; } diff --git a/src/packages/frontend/project/listing/use-files.ts b/src/packages/frontend/project/listing/use-files.ts new file mode 100644 index 0000000000..f8c1b96aab --- /dev/null +++ b/src/packages/frontend/project/listing/use-files.ts @@ -0,0 +1,58 @@ +/* +Hook that provides all files in a directory via a Conat FilesystemClient. +This automatically updates when files change. + +TESTS: See packages/test/project/listing/ + +*/ + +import useAsyncEffect from "use-async-effect"; +import { useState } from "react"; +import { throttle } from "lodash"; +import { type Files } from "@cocalc/conat/files/listing"; +import { type FilesystemClient } from "@cocalc/conat/files/fs"; +import { type ConatError } from "@cocalc/conat/core/client"; + +const DEFAULT_THROTTLE_FILE_UPDATE = 500; + +export default function useFiles({ + fs, + path, + throttleUpdate = DEFAULT_THROTTLE_FILE_UPDATE, +}: { + fs: FilesystemClient; + path: string; + throttleUpdate?: number; +}): { files: Files | null; error: null | ConatError; refresh: () => void } { + const [files, setFiles] = useState(null); + const [error, setError] = useState(null); + const [counter, setCounter] = useState(0); + + useAsyncEffect(async () => { + let listing; + try { + listing = await fs.listing(path); + setError(null); + } catch (err) { + setError(err); + setFiles(null); + return; + } + + const update = () => { + setFiles({ ...listing.files }); + }; + update(); + + listing.on( + "change", + throttle(update, throttleUpdate, { leading: true, trailing: true }), + ); + + return () => { + listing.close(); + }; + }, [fs, path, counter]); + + return { files, error, refresh: () => setCounter(counter + 1) }; +} diff --git a/src/packages/frontend/project/listing/use-listing.ts b/src/packages/frontend/project/listing/use-listing.ts index d36c676e11..43cfd4207d 100644 --- a/src/packages/frontend/project/listing/use-listing.ts +++ b/src/packages/frontend/project/listing/use-listing.ts @@ -1,16 +1,15 @@ /* A directory listing hook. + +TESTS: See packages/test/project/listing/ */ -import { useMemo, useState } from "react"; +import { useMemo } from "react"; import { DirectoryListingEntry } from "@cocalc/util/types"; -import useAsyncEffect from "use-async-effect"; -import { throttle } from "lodash"; import { field_cmp } from "@cocalc/util/misc"; -import { type Files } from "@cocalc/conat/files/listing"; +import useFiles from "./use-files"; import { type FilesystemClient } from "@cocalc/conat/files/fs"; - -const DEFAULT_THROTTLE_FILE_UPDATE = 500; +import { type ConatError } from "@cocalc/conat/core/client"; type SortField = "name" | "mtime" | "size"; type SortDirection = "inc" | "dec"; @@ -20,17 +19,19 @@ export default function useListing({ path, sortField = "name", sortDirection = "inc", + throttleUpdate, }: { fs: FilesystemClient; path: string; sortField?: SortField; sortDirection?: SortDirection; + throttleUpdate?: number; }): { listing: null | DirectoryListingEntry[]; - error: null | Error; + error: null | ConatError; refresh: () => void; } { - const { files, error, refresh } = useFiles({ fs, path }); + const { files, error, refresh } = useFiles({ fs, path, throttleUpdate }); const listing = useMemo(() => { if (files == null) { @@ -49,45 +50,3 @@ export default function useListing({ return { listing, error, refresh }; } - -export function useFiles({ - fs, - path, - throttleUpdate = DEFAULT_THROTTLE_FILE_UPDATE, -}: { - fs: FilesystemClient; - path: string; - throttleUpdate?: number; -}): { files: Files | null; error: null | Error; refresh: () => void } { - const [files, setFiles] = useState(null); - const [error, setError] = useState(null); - const [counter, setCounter] = useState(0); - - useAsyncEffect(async () => { - let listing; - try { - listing = await fs.listing(path); - setError(null); - } catch (err) { - setError(err); - setFiles(null); - return; - } - - const update = () => { - setFiles({ ...listing.files }); - }; - update(); - - listing.on( - "change", - throttle(update, throttleUpdate, { leading: true, trailing: true }), - ); - - return () => { - listing.close(); - }; - }, [fs, path, counter]); - - return { files, error, refresh: () => setCounter(counter + 1) }; -} diff --git a/src/packages/test/use-listing.test.ts b/src/packages/test/project/listing/use-files.test.ts similarity index 90% rename from src/packages/test/use-listing.test.ts rename to src/packages/test/project/listing/use-files.test.ts index 831a0fd692..4dc27fe789 100644 --- a/src/packages/test/use-listing.test.ts +++ b/src/packages/test/project/listing/use-files.test.ts @@ -1,12 +1,12 @@ import { act, renderHook, waitFor } from "@testing-library/react"; import { fsClient } from "@cocalc/conat/files/fs"; -import { before, after, wait } from "@cocalc/backend/conat/test/setup"; +import { before, after } from "@cocalc/backend/conat/test/setup"; import { uuid } from "@cocalc/util/misc"; import { createPathFileserver, cleanupFileservers, } from "@cocalc/backend/conat/files/test/util"; -import { useFiles } from "@cocalc/frontend/project/listing/use-listing"; +import useFiles from "@cocalc/frontend/project/listing/use-files"; beforeAll(before); @@ -41,13 +41,13 @@ describe("the useFiles hook", () => { refresh: expect.any(Function), }); - // now write a file + // now create a file await act(async () => { await fs.writeFile("hello.txt", "world"); }); await waitFor(() => { - expect(result.current.files["hello.txt"]).toBeDefined(); + expect(result.current.files?.["hello.txt"]).toBeDefined(); }); expect(result.current).toEqual({ @@ -68,7 +68,7 @@ describe("the useFiles hook", () => { await waitFor(() => { expect(result.current.files?.["hello.txt"]).not.toBeDefined(); }); - expect(result.current.error.code).toBe("ENOENT"); + expect(result.current.error?.code).toBe("ENOENT"); await act(async () => { // create the path, a file in there, refresh and it works diff --git a/src/packages/test/project/listing/use-listing.test.ts b/src/packages/test/project/listing/use-listing.test.ts new file mode 100644 index 0000000000..bb0b6d2117 --- /dev/null +++ b/src/packages/test/project/listing/use-listing.test.ts @@ -0,0 +1,110 @@ +import { act, renderHook, waitFor } from "@testing-library/react"; +import { fsClient } from "@cocalc/conat/files/fs"; +import { before, after } from "@cocalc/backend/conat/test/setup"; +import { uuid } from "@cocalc/util/misc"; +import { + createPathFileserver, + cleanupFileservers, +} from "@cocalc/backend/conat/files/test/util"; +import useListing from "@cocalc/frontend/project/listing/use-listing"; + +beforeAll(before); + +describe("the useListing hook", () => { + const project_id = uuid(); + let fs, server; + it("creates fileserver service and fs client", async () => { + server = await createPathFileserver(); + fs = fsClient({ subject: `${server.service}.project-${project_id}` }); + }); + + it("test useListing and file creation", async () => { + let path = "", + fs2 = fs; + const { result, rerender } = renderHook(() => + useListing({ fs: fs2, path, throttleUpdate: 0 }), + ); + + expect(result.current).toEqual({ + listing: null, + error: null, + refresh: expect.any(Function), + }); + + // eventually it will be initialized to not be null + await waitFor(() => { + expect(result.current.listing).not.toBeNull(); + }); + + expect(result.current).toEqual({ + listing: [], + error: null, + refresh: expect.any(Function), + }); + + // now create a file + await act(async () => { + await fs.writeFile("hello.txt", "world"); + }); + + await waitFor(() => { + expect(result.current.listing?.length).toEqual(1); + }); + + expect(result.current).toEqual({ + listing: [{ name: "hello.txt", size: 5, mtime: expect.any(Number) }], + error: null, + refresh: expect.any(Function), + }); + + // change the path to one that does not exist and rerender, + // resulting in an ENOENT error + path = "scratch"; + rerender(); + await waitFor(() => { + expect(result.current.listing).toBeNull(); + expect(result.current.error?.code).toBe("ENOENT"); + }); + + await act(async () => { + // create the path, a file in there, refresh and it works + await fs.mkdir(path); + await fs.writeFile("scratch/b.txt", "hi"); + result.current.refresh(); + }); + + await waitFor(() => { + expect(result.current).toEqual({ + listing: [ + { + name: "b.txt", + size: 2, + mtime: expect.any(Number), + }, + ], + error: null, + refresh: expect.any(Function), + }); + }); + + // change fs and see the hook update + const project_id2 = uuid(); + fs2 = fsClient({ + subject: `${server.service}.project-${project_id2}`, + }); + path = ""; + rerender(); + await waitFor(() => { + expect(result.current).toEqual({ + listing: [], + error: null, + refresh: expect.any(Function), + }); + }); + }); +}); + +afterAll(async () => { + await after(); + await cleanupFileservers(); +}); From 73b70800834d43b491aa644385242060c8359e2a Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 18:15:33 +0000 Subject: [PATCH 075/798] unit tests of use-listing that involve sorting --- .../frontend/project/listing/use-listing.ts | 14 ++- .../test/project/listing/use-listing.test.ts | 95 +++++++++++++++++++ 2 files changed, 105 insertions(+), 4 deletions(-) diff --git a/src/packages/frontend/project/listing/use-listing.ts b/src/packages/frontend/project/listing/use-listing.ts index 43cfd4207d..5418f50cba 100644 --- a/src/packages/frontend/project/listing/use-listing.ts +++ b/src/packages/frontend/project/listing/use-listing.ts @@ -12,13 +12,13 @@ import { type FilesystemClient } from "@cocalc/conat/files/fs"; import { type ConatError } from "@cocalc/conat/core/client"; type SortField = "name" | "mtime" | "size"; -type SortDirection = "inc" | "dec"; +type SortDirection = "asc" | "desc"; export default function useListing({ fs, path, sortField = "name", - sortDirection = "inc", + sortDirection = "asc", throttleUpdate, }: { fs: FilesystemClient; @@ -41,9 +41,15 @@ export default function useListing({ for (const name in files) { v.push({ name, ...files[name] }); } - v.sort(field_cmp("name")); - if (sortDirection == "dec") { + if (sortField != "name" && sortField != "mtime" && sortField != "size") { + console.warn(`invalid sort field: '${sortField}'`); + } + v.sort(field_cmp(sortField)); + if (sortDirection == "desc") { v.reverse(); + } else if (sortDirection == "asc") { + } else { + console.warn(`invalid sort direction: '${sortDirection}'`); } return v; }, [sortField, sortDirection, files]); diff --git a/src/packages/test/project/listing/use-listing.test.ts b/src/packages/test/project/listing/use-listing.test.ts index bb0b6d2117..c6d86dbeeb 100644 --- a/src/packages/test/project/listing/use-listing.test.ts +++ b/src/packages/test/project/listing/use-listing.test.ts @@ -104,6 +104,101 @@ describe("the useListing hook", () => { }); }); +describe("test sorting many files with useListing", () => { + const project_id = uuid(); + let fs, server; + it("creates fileserver service and fs client", async () => { + server = await createPathFileserver(); + fs = fsClient({ subject: `${server.service}.project-${project_id}` }); + }); + + it("create some files", async () => { + await fs.writeFile("a.txt", "abc"); + await fs.writeFile("b.txt", "b"); + await fs.writeFile("huge.txt", "b".repeat(1000)); + + // make b.txt old + await fs.utimes( + "b.txt", + (Date.now() - 60_000) / 1000, + (Date.now() - 60_000) / 1000, + ); + }); + + it("test useListing with many files and sorting", async () => { + let path = "", + sortField = "name", + sortDirection = "asc"; + const { result, rerender } = renderHook(() => + useListing({ fs, path, throttleUpdate: 0, sortField, sortDirection }), + ); + + await waitFor(() => { + expect(result.current.listing?.length).toEqual(3); + }); + expect(result.current.listing.map(({ name }) => name)).toEqual([ + "a.txt", + "b.txt", + "huge.txt", + ]); + + sortDirection = "desc"; + sortField = "name"; + rerender(); + await waitFor(() => { + expect(result.current.listing.map(({ name }) => name)).toEqual([ + "huge.txt", + "b.txt", + "a.txt", + ]); + }); + + sortDirection = "asc"; + sortField = "mtime"; + rerender(); + await waitFor(() => { + expect(result.current.listing.map(({ name }) => name)).toEqual([ + "b.txt", + "a.txt", + "huge.txt", + ]); + }); + + sortDirection = "desc"; + sortField = "mtime"; + rerender(); + await waitFor(() => { + expect(result.current.listing.map(({ name }) => name)).toEqual([ + "huge.txt", + "a.txt", + "b.txt", + ]); + }); + + sortDirection = "asc"; + sortField = "size"; + rerender(); + await waitFor(() => { + expect(result.current.listing.map(({ name }) => name)).toEqual([ + "b.txt", + "a.txt", + "huge.txt", + ]); + }); + + sortDirection = "desc"; + sortField = "size"; + rerender(); + await waitFor(() => { + expect(result.current.listing.map(({ name }) => name)).toEqual([ + "huge.txt", + "a.txt", + "b.txt", + ]); + }); + }); +}); + afterAll(async () => { await after(); await cleanupFileservers(); From 7b3c4dd81ae424a54d3069a8866b8804a308027c Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 22:10:14 +0000 Subject: [PATCH 076/798] frontend listings: first pass at using the new fs interface and hooks (not done and the old stuff is still being computed) --- src/packages/backend/files/sandbox/index.ts | 5 + src/packages/conat/files/fs.ts | 30 +++- src/packages/conat/files/listing.ts | 65 ++++++- .../frontend/project/explorer/explorer.tsx | 165 +++++------------- .../explorer/file-listing/file-listing.tsx | 113 ++++++------ .../frontend/project/listing/use-files.ts | 74 ++++---- .../frontend/project/listing/use-fs.ts | 20 +++ .../frontend/project/listing/use-listing.ts | 14 +- .../test/project/listing/use-files.test.ts | 2 + .../test/project/listing/use-listing.test.ts | 34 ++-- 10 files changed, 289 insertions(+), 233 deletions(-) create mode 100644 src/packages/frontend/project/listing/use-fs.ts diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index b65ba1ff3f..aa09b12c25 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -50,6 +50,7 @@ import { lstat, readdir, readFile, + readlink, realpath, rename, rm, @@ -241,6 +242,10 @@ export class SandboxedFilesystem { return await readdir(await this.safeAbsPath(path)); }; + readlink = async (path: string): Promise => { + return await readlink(await this.safeAbsPath(path)); + }; + realpath = async (path: string): Promise => { const x = await realpath(await this.safeAbsPath(path)); return x.slice(this.path.length + 1); diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index c522b29b24..bd60f4763c 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -5,7 +5,7 @@ import { watchClient, type WatchIterator, } from "@cocalc/conat/files/watch"; -import listing, { type Listing } from "./listing"; +import listing, { type Listing, type FileTypeLabel } from "./listing"; export const DEFAULT_FILE_SERVICE = "fs"; @@ -43,6 +43,7 @@ export interface Filesystem { mkdir: (path: string, options?) => Promise; readFile: (path: string, encoding?: any) => Promise; readdir: (path: string) => Promise; + readlink: (path: string) => Promise; realpath: (path: string) => Promise; rename: (oldPath: string, newPath: string) => Promise; rm: (path: string, options?) => Promise; @@ -135,6 +136,26 @@ class Stats { isSocket = () => (this.mode & this.constants.S_IFMT) === this.constants.S_IFSOCK; + + get type(): FileTypeLabel { + switch (this.mode & this.constants.S_IFMT) { + case this.constants.S_IFLNK: + return "l"; + case this.constants.S_IFREG: + return "f"; + case this.constants.S_IFDIR: + return "d"; + case this.constants.S_IFBLK: + return "b"; + case this.constants.S_IFCHR: + return "c"; + case this.constants.S_IFSOCK: + return "s"; + case this.constants.S_IFIFO: + return "p"; + } + return "f"; + } } interface Options { @@ -184,6 +205,9 @@ export async function fsServer({ service, fs, client }: Options) { async readdir(path: string) { return await (await fs(this.subject)).readdir(path); }, + async readlink(path: string) { + return await (await fs(this.subject)).readlink(path); + }, async realpath(path: string) { return await (await fs(this.subject)).realpath(path); }, @@ -252,8 +276,10 @@ export async function fsServer({ service, fs, client }: Options) { }; } -export type FilesystemClient = Filesystem & { +export type FilesystemClient = Omit, "lstat"> & { listing: (path: string) => Promise; + stat: (path: string) => Promise; + lstat: (path: string) => Promise; }; export function fsClient({ diff --git a/src/packages/conat/files/listing.ts b/src/packages/conat/files/listing.ts index 63d9bab6c9..d45bec5401 100644 --- a/src/packages/conat/files/listing.ts +++ b/src/packages/conat/files/listing.ts @@ -2,23 +2,44 @@ Directory Listing Tests in packages/backend/conat/files/test/listing.test.ts + + */ import { EventEmitter } from "events"; import { join } from "path"; -import { type Filesystem } from "./fs"; +import { type FilesystemClient } from "./fs"; import { EventIterator } from "@cocalc/util/event-iterator"; +export type FileTypeLabel = "f" | "d" | "l" | "b" | "c" | "s" | "p"; + +export const typeDescription = { + f: "regular file", + d: "directory", + l: "symlink", + b: "block device", + c: "character device", + s: "socket", + p: "fifo", +}; + interface FileData { mtime: number; size: number; + // isdir = mainly for backward compat: + isdir?: boolean; + // issymlink = mainly for backward compat: + issymlink?: boolean; + link_target?: string; + // see typeDescription above. + type?: FileTypeLabel; } export type Files = { [name: string]: FileData }; interface Options { path: string; - fs: Filesystem; + fs: FilesystemClient; } export default async function listing(opts: Options): Promise { @@ -48,6 +69,7 @@ export class Listing extends EventEmitter { close = () => { this.emit("closed"); + this.removeAllListeners(); this.iters.map((iter) => iter.end()); this.iters.length = 0; this.watch?.close(); @@ -80,11 +102,26 @@ export class Listing extends EventEmitter { return; } try { - const stats = await this.opts.fs.stat(join(this.opts.path, filename)); + const stats = await this.opts.fs.lstat(join(this.opts.path, filename)); if (this.files == null) { return; } - this.files[filename] = { mtime: stats.mtimeMs, size: stats.size }; + const data: FileData = { + mtime: stats.mtimeMs / 1000, + size: stats.size, + type: stats.type, + }; + if (stats.isSymbolicLink()) { + // resolve target. + data.link_target = await this.opts.fs.readlink( + join(this.opts.path, filename), + ); + data.issymlink = true; + } + if (stats.isDirectory()) { + data.isdir = true; + } + this.files[filename] = data; } catch (err) { if (this.files == null) { return; @@ -106,10 +143,13 @@ export class Listing extends EventEmitter { } async function getListing( - fs: Filesystem, + fs: FilesystemClient, path: string, ): Promise<{ files: Files; truncated: boolean }> { - const { stdout, truncated } = await fs.find(path, "%f\\0%T@\\0%s\n"); + const { stdout, truncated } = await fs.find( + path, + "%f\\0%T@\\0%s\\0%y\\0%l\n", + ); const buf = Buffer.from(stdout); const files: Files = {}; // todo -- what about non-utf8...? @@ -122,9 +162,18 @@ async function getListing( try { const v = line.split("\0"); const name = v[0]; - const mtime = parseFloat(v[1]) * 1000; + const mtime = parseFloat(v[1]); const size = parseInt(v[2]); - files[name] = { mtime, size }; + files[name] = { mtime, size, type: v[3] as FileTypeLabel }; + if (v[3] == "l") { + files[name].issymlink = true; + } + if (v[3] == "d") { + files[name].isdir = true; + } + if (v[4]) { + files[name].link_target = v[4]; + } } catch {} } return { files, truncated }; diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index 6ea7f84b1b..e4818639cd 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -3,11 +3,9 @@ * License: MS-RSL – see LICENSE.md for details */ -import { Alert } from "antd"; import * as immutable from "immutable"; import * as _ from "lodash"; import React from "react"; -import { FormattedMessage } from "react-intl"; import { UsersViewing } from "@cocalc/frontend/account/avatar/users-viewing"; import { Button, ButtonGroup, Col, Row } from "@cocalc/frontend/antd-bootstrap"; import { @@ -24,7 +22,6 @@ import { ErrorDisplay, Icon, Loading, - Paragraph, SettingBox, } from "@cocalc/frontend/components"; import { ComputeServerDocStatus } from "@cocalc/frontend/compute/doc-status"; @@ -45,7 +42,6 @@ import { useProjectContext } from "../context"; import { AccessErrors } from "./access-errors"; import { ActionBar } from "./action-bar"; import { ActionBox } from "./action-box"; -import { FetchDirectoryErrors } from "./fetch-directory-errors"; import { FileListing } from "./file-listing"; import { default_ext } from "./file-listing/utils"; import { MiscSideButtons } from "./misc-side-buttons"; @@ -432,117 +428,45 @@ const Explorer0 = rclass( return ; } - render_file_listing( - listing: ListingItem[] | undefined, - file_map, - fetch_directory_error: any, - project_is_running: boolean, - ) { - if (fetch_directory_error) { - return ( -
- -
- -
- ); - } else if (listing != undefined) { - return ( - this.props.actions.fetch_directory_listing(), + }} + config={{ clickable: ".upload-button" }} + style={{ + flex: "1 0 auto", + display: "flex", + flexDirection: "column", + }} + className="smc-vfill" + > + this.props.actions.fetch_directory_listing(), - }} - config={{ clickable: ".upload-button" }} - style={{ - flex: "1 0 auto", - display: "flex", - flexDirection: "column", - }} - className="smc-vfill" - > - - - ); - } else { - if (project_is_running) { - // ensure directory listing starts getting computed. - redux.getProjectStore(this.props.project_id).get_listings(); - return ( -
- -
- ); - } else { - return ( - } - style={{ textAlign: "center" }} - showIcon - description={ - - start this project.`} - values={{ - a: (c) => ( - { - redux - .getActions("projects") - .start_project(this.props.project_id); - }} - > - {c} - - ), - }} - /> - - } - /> - ); - } - } + shift_is_down={this.state.shift_is_down} + sort_by={this.props.actions.set_sorted_file_column} + other_settings={this.props.other_settings} + library={this.props.library} + redux={redux} + last_scroll_top={this.props.file_listing_scroll_top} + configuration_main={this.props.configuration?.get("main")} + show_hidden={this.props.show_hidden} + show_masked={this.props.show_masked} + /> + + ); } file_listing_page_size() { @@ -714,7 +638,6 @@ const Explorer0 = rclass( const displayed_listing = this.props.displayed_listing; const { listing, file_map } = displayed_listing; - const directory_error = displayed_listing.error; const file_listing_page_size = this.file_listing_page_size(); if (listing != undefined) { @@ -786,17 +709,7 @@ const Explorer0 = rclass( padding: "0 5px 5px 5px", }} > - {this.render_file_listing( - visible_listing, - file_map, - directory_error, - project_is_running, - )} - {listing != undefined - ? this.render_paging_buttons( - Math.ceil(listing.length / file_listing_page_size), - ) - : undefined} + {this.render_file_listing()} ; listing: any[]; - file_map: object; file_search: string; checked_files: immutable.Set; current_path: string; @@ -55,6 +58,11 @@ interface Props { last_scroll_top?: number; configuration_main?: MainConfiguration; isRunning?: boolean; // true if this project is running + + show_hidden?: boolean; + show_masked?: boolean; + + stale?: boolean; } export function watchFiles({ actions, current_path }): void { @@ -67,13 +75,48 @@ export function watchFiles({ actions, current_path }): void { } } -export const FileListing: React.FC = ({ +function sortDesc(active_file_sort?): { + sortField: SortField; + sortDirection: "asc" | "desc"; +} { + const { column_name, is_descending } = active_file_sort?.toJS() ?? { + column_name: "name", + is_descending: false, + }; + if (column_name == "time") { + return { + sortField: "mtime", + sortDirection: is_descending ? "asc" : "desc", + }; + } + return { + sortField: column_name, + sortDirection: is_descending ? "desc" : "asc", + }; +} + +export function FileListing(props) { + const fs = useFs({ project_id: props.project_id }); + const { listing, error } = useListing({ + fs, + path: props.current_path, + ...sortDesc(props.active_file_sort), + }); + if (error) { + return ; + } + if (listing == null) { + return ; + } + return ; +} + +function FileListing0({ actions, redux, name, active_file_sort, listing, - file_map, checked_files, current_path, create_folder, @@ -85,8 +128,16 @@ export const FileListing: React.FC = ({ configuration_main, file_search = "", isRunning, -}: Props) => { - const [starting, setStarting] = useState(false); + show_hidden, + stale, + // show_masked, +}: Props) { + if (!show_hidden) { + listing = listing.filter((x) => !x.name.startsWith(".")); + } + if (file_search) { + listing = listing.filter((x) => x.name.includes(file_search)); + } const prev_current_path = usePrevious(current_path); @@ -135,7 +186,6 @@ export const FileListing: React.FC = ({ ): Rendered { const checked = checked_files.has(misc.path_to_file(current_path, name)); const color = misc.rowBackground({ index, checked }); - const { is_public } = file_map[name]; return ( = ({ name={name} display_name={display_name} time={time} - size={size} + size={isdir ? undefined : size} issymlink={issymlink} color={color} selected={ @@ -151,7 +201,7 @@ export const FileListing: React.FC = ({ } mask={mask} public_data={public_data} - is_public={is_public} + is_public={false} checked={checked} key={index} current_path={current_path} @@ -188,7 +238,7 @@ export const FileListing: React.FC = ({ return ( { const a = listing[index]; @@ -236,44 +286,9 @@ export const FileListing: React.FC = ({ ); } - if (!isRunning && listing.length == 0) { - return ( - - { - if (starting) return; - try { - setStarting(true); - await actions.fetch_directory_listing_directly( - current_path, - true, - ); - } finally { - setStarting(false); - } - }} - > - Start this project to see your files. - {starting && } - - - } - /> - ); - } - return ( <> - {!isRunning && listing.length > 0 && ( + {stale && (
@@ -318,4 +333,4 @@ export const FileListing: React.FC = ({ ); -}; +} diff --git a/src/packages/frontend/project/listing/use-files.ts b/src/packages/frontend/project/listing/use-files.ts index f8c1b96aab..925a284894 100644 --- a/src/packages/frontend/project/listing/use-files.ts +++ b/src/packages/frontend/project/listing/use-files.ts @@ -7,11 +7,12 @@ TESTS: See packages/test/project/listing/ */ import useAsyncEffect from "use-async-effect"; -import { useState } from "react"; +import { useRef, useState } from "react"; import { throttle } from "lodash"; import { type Files } from "@cocalc/conat/files/listing"; import { type FilesystemClient } from "@cocalc/conat/files/fs"; import { type ConatError } from "@cocalc/conat/core/client"; +import useCounter from "@cocalc/frontend/app-framework/counter-hook"; const DEFAULT_THROTTLE_FILE_UPDATE = 500; @@ -20,39 +21,50 @@ export default function useFiles({ path, throttleUpdate = DEFAULT_THROTTLE_FILE_UPDATE, }: { - fs: FilesystemClient; + // fs = undefined is supported and just waits until you provide a fs that is defined + fs?: FilesystemClient | null; path: string; throttleUpdate?: number; }): { files: Files | null; error: null | ConatError; refresh: () => void } { const [files, setFiles] = useState(null); const [error, setError] = useState(null); - const [counter, setCounter] = useState(0); - - useAsyncEffect(async () => { - let listing; - try { - listing = await fs.listing(path); - setError(null); - } catch (err) { - setError(err); - setFiles(null); - return; - } - - const update = () => { - setFiles({ ...listing.files }); - }; - update(); - - listing.on( - "change", - throttle(update, throttleUpdate, { leading: true, trailing: true }), - ); - - return () => { - listing.close(); - }; - }, [fs, path, counter]); - - return { files, error, refresh: () => setCounter(counter + 1) }; + const { val: counter, inc: refresh } = useCounter(); + const listingRef = useRef(null); + + useAsyncEffect( + async () => { + if (fs == null) { + setError(null); + setFiles(null); + return; + } + let listing; + try { + listing = await fs.listing(path); + listingRef.current = listing; + setError(null); + } catch (err) { + setError(err); + setFiles(null); + return; + } + + const update = () => { + setFiles({ ...listing.files }); + }; + update(); + + listing.on( + "change", + throttle(update, throttleUpdate, { leading: true, trailing: true }), + ); + }, + () => { + listingRef.current?.close(); + delete listingRef.current; + }, + [fs, path, counter], + ); + + return { files, error, refresh }; } diff --git a/src/packages/frontend/project/listing/use-fs.ts b/src/packages/frontend/project/listing/use-fs.ts new file mode 100644 index 0000000000..ff76d79801 --- /dev/null +++ b/src/packages/frontend/project/listing/use-fs.ts @@ -0,0 +1,20 @@ +/* +Hook for getting a FilesystemClient. +*/ +import { webapp_client } from "@cocalc/frontend/webapp-client"; +import { type FilesystemClient } from "@cocalc/conat/files/fs"; +import { useState } from "react"; + +// this will probably get more complicated temporarily when we +// are transitioning between filesystems (hence why we return null in +// the typing for now) +export default function useFs({ + project_id, +}: { + project_id: string; +}): FilesystemClient | null { + const [fs] = useState(() => + webapp_client.conat_client.conat().fs({ project_id }), + ); + return fs; +} diff --git a/src/packages/frontend/project/listing/use-listing.ts b/src/packages/frontend/project/listing/use-listing.ts index 5418f50cba..75ffc37f40 100644 --- a/src/packages/frontend/project/listing/use-listing.ts +++ b/src/packages/frontend/project/listing/use-listing.ts @@ -11,8 +11,8 @@ import useFiles from "./use-files"; import { type FilesystemClient } from "@cocalc/conat/files/fs"; import { type ConatError } from "@cocalc/conat/core/client"; -type SortField = "name" | "mtime" | "size"; -type SortDirection = "asc" | "desc"; +export type SortField = "name" | "mtime" | "size" | "type"; +export type SortDirection = "asc" | "desc"; export default function useListing({ fs, @@ -21,7 +21,8 @@ export default function useListing({ sortDirection = "asc", throttleUpdate, }: { - fs: FilesystemClient; + // fs = undefined is supported and just waits until you provide a fs that is defined + fs?: FilesystemClient | null; path: string; sortField?: SortField; sortDirection?: SortDirection; @@ -41,7 +42,12 @@ export default function useListing({ for (const name in files) { v.push({ name, ...files[name] }); } - if (sortField != "name" && sortField != "mtime" && sortField != "size") { + if ( + sortField != "name" && + sortField != "mtime" && + sortField != "size" && + sortField != "type" + ) { console.warn(`invalid sort field: '${sortField}'`); } v.sort(field_cmp(sortField)); diff --git a/src/packages/test/project/listing/use-files.test.ts b/src/packages/test/project/listing/use-files.test.ts index 4dc27fe789..f36ad76040 100644 --- a/src/packages/test/project/listing/use-files.test.ts +++ b/src/packages/test/project/listing/use-files.test.ts @@ -55,6 +55,7 @@ describe("the useFiles hook", () => { "hello.txt": { size: 5, mtime: expect.any(Number), + type: "f", }, }, error: null, @@ -83,6 +84,7 @@ describe("the useFiles hook", () => { "b.txt": { size: 2, mtime: expect.any(Number), + type: "f", }, }, error: null, diff --git a/src/packages/test/project/listing/use-listing.test.ts b/src/packages/test/project/listing/use-listing.test.ts index c6d86dbeeb..37807219dc 100644 --- a/src/packages/test/project/listing/use-listing.test.ts +++ b/src/packages/test/project/listing/use-listing.test.ts @@ -6,7 +6,11 @@ import { createPathFileserver, cleanupFileservers, } from "@cocalc/backend/conat/files/test/util"; -import useListing from "@cocalc/frontend/project/listing/use-listing"; +import useListing, { + type SortField, + type SortDirection, +} from "@cocalc/frontend/project/listing/use-listing"; +import { type FilesystemClient } from "@cocalc/conat/files/fs"; beforeAll(before); @@ -20,18 +24,19 @@ describe("the useListing hook", () => { it("test useListing and file creation", async () => { let path = "", - fs2 = fs; + fs2: FilesystemClient | undefined = undefined; const { result, rerender } = renderHook(() => useListing({ fs: fs2, path, throttleUpdate: 0 }), ); - expect(result.current).toEqual({ listing: null, error: null, refresh: expect.any(Function), }); + fs2 = fs; + rerender(); - // eventually it will be initialized to not be null + // now that fs2 is set, eventually it will be initialized to not be null await waitFor(() => { expect(result.current.listing).not.toBeNull(); }); @@ -52,7 +57,9 @@ describe("the useListing hook", () => { }); expect(result.current).toEqual({ - listing: [{ name: "hello.txt", size: 5, mtime: expect.any(Number) }], + listing: [ + { name: "hello.txt", size: 5, mtime: expect.any(Number), type: "f" }, + ], error: null, refresh: expect.any(Function), }); @@ -79,6 +86,7 @@ describe("the useListing hook", () => { { name: "b.txt", size: 2, + type: "f", mtime: expect.any(Number), }, ], @@ -127,8 +135,8 @@ describe("test sorting many files with useListing", () => { it("test useListing with many files and sorting", async () => { let path = "", - sortField = "name", - sortDirection = "asc"; + sortField: SortField = "name", + sortDirection: SortDirection = "asc"; const { result, rerender } = renderHook(() => useListing({ fs, path, throttleUpdate: 0, sortField, sortDirection }), ); @@ -136,7 +144,7 @@ describe("test sorting many files with useListing", () => { await waitFor(() => { expect(result.current.listing?.length).toEqual(3); }); - expect(result.current.listing.map(({ name }) => name)).toEqual([ + expect(result.current.listing?.map(({ name }) => name)).toEqual([ "a.txt", "b.txt", "huge.txt", @@ -146,7 +154,7 @@ describe("test sorting many files with useListing", () => { sortField = "name"; rerender(); await waitFor(() => { - expect(result.current.listing.map(({ name }) => name)).toEqual([ + expect(result.current.listing?.map(({ name }) => name)).toEqual([ "huge.txt", "b.txt", "a.txt", @@ -157,7 +165,7 @@ describe("test sorting many files with useListing", () => { sortField = "mtime"; rerender(); await waitFor(() => { - expect(result.current.listing.map(({ name }) => name)).toEqual([ + expect(result.current.listing?.map(({ name }) => name)).toEqual([ "b.txt", "a.txt", "huge.txt", @@ -168,7 +176,7 @@ describe("test sorting many files with useListing", () => { sortField = "mtime"; rerender(); await waitFor(() => { - expect(result.current.listing.map(({ name }) => name)).toEqual([ + expect(result.current.listing?.map(({ name }) => name)).toEqual([ "huge.txt", "a.txt", "b.txt", @@ -179,7 +187,7 @@ describe("test sorting many files with useListing", () => { sortField = "size"; rerender(); await waitFor(() => { - expect(result.current.listing.map(({ name }) => name)).toEqual([ + expect(result.current.listing?.map(({ name }) => name)).toEqual([ "b.txt", "a.txt", "huge.txt", @@ -190,7 +198,7 @@ describe("test sorting many files with useListing", () => { sortField = "size"; rerender(); await waitFor(() => { - expect(result.current.listing.map(({ name }) => name)).toEqual([ + expect(result.current.listing?.map(({ name }) => name)).toEqual([ "huge.txt", "a.txt", "b.txt", From 5bdfca9451f90ed33169902e1001b3e20156c098 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 22:23:12 +0000 Subject: [PATCH 077/798] remove some not-necessary default styling; increase react virtuoso default viewport --- .../frontend/components/virtuoso-scroll-hook.ts | 5 +++-- .../explorer/file-listing/file-listing.tsx | 16 ++++------------ 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/src/packages/frontend/components/virtuoso-scroll-hook.ts b/src/packages/frontend/components/virtuoso-scroll-hook.ts index fab8ed41c9..8972a24333 100644 --- a/src/packages/frontend/components/virtuoso-scroll-hook.ts +++ b/src/packages/frontend/components/virtuoso-scroll-hook.ts @@ -46,7 +46,7 @@ export default function useVirtuosoScrollHook({ }, []); if (disabled) return {}; const lastScrollRef = useRef( - initialState ?? { index: 0, offset: 0 } + initialState ?? { index: 0, offset: 0 }, ); const recordScrollState = useMemo(() => { return (state: ScrollState) => { @@ -64,8 +64,9 @@ export default function useVirtuosoScrollHook({ }, [onScroll, cacheId]); return { + increaseViewportBy: 2000 /* a lot better default than 0 */, initialTopMostItemIndex: - (cacheId ? cache.get(cacheId) ?? initialState : initialState) ?? 0, + (cacheId ? (cache.get(cacheId) ?? initialState) : initialState) ?? 0, scrollerRef: handleScrollerRef, onScroll: () => { const scrollTop = scrollerRef.current?.scrollTop; diff --git a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx index 01c73c66be..8c10a2987c 100644 --- a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx +++ b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx @@ -13,7 +13,6 @@ import { useEffect, useRef, useState } from "react"; import { useInterval } from "react-interval-hook"; import { FormattedMessage } from "react-intl"; import { Virtuoso, VirtuosoHandle } from "react-virtuoso"; -import { Col, Row } from "@cocalc/frontend/antd-bootstrap"; import { AppRedux, Rendered, @@ -312,25 +311,18 @@ function FileListing0({ />
)} - - {listing.length > 0 && ( - - )} - {listing.length > 0 && {render_rows()}} + + {render_rows()} {render_no_files()} - + ); } From 247a6a06c070f3d58f81223acf213d6b3f358bfe Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 22:58:08 +0000 Subject: [PATCH 078/798] show file creation buttons even if you don't search (hsy suggested this in a meeting) --- .../explorer/file-listing/file-listing.tsx | 11 +--- .../explorer/file-listing/no-files.tsx | 55 ++++++++++++------- src/packages/frontend/project_actions.ts | 6 -- 3 files changed, 37 insertions(+), 35 deletions(-) diff --git a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx index 8c10a2987c..e85b0c754a 100644 --- a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx +++ b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx @@ -46,12 +46,10 @@ interface Props { file_search: string; checked_files: immutable.Set; current_path: string; - create_folder: (switch_over?: boolean) => void; // TODO: should be action! - create_file: (ext?: string, switch_over?: boolean) => void; // TODO: should be action! selected_file_index?: number; project_id: string; shift_is_down: boolean; - sort_by: (heading: string) => void; // TODO: should be data + sort_by: (heading: string) => void; library?: object; other_settings?: immutable.Map; last_scroll_top?: number; @@ -118,8 +116,6 @@ function FileListing0({ listing, checked_files, current_path, - create_folder, - create_file, selected_file_index, project_id, shift_is_down, @@ -277,8 +273,6 @@ function FileListing0({ current_path={current_path} actions={actions} file_search={file_search} - create_folder={create_folder} - create_file={create_file} project_id={project_id} configuration_main={configuration_main} /> @@ -320,8 +314,7 @@ function FileListing0({ }} > - {render_rows()} - {render_no_files()} + {listing.length > 0 ? render_rows() : render_no_files()} ); diff --git a/src/packages/frontend/project/explorer/file-listing/no-files.tsx b/src/packages/frontend/project/explorer/file-listing/no-files.tsx index ac6f58f6d2..a4c42fbf2b 100644 --- a/src/packages/frontend/project/explorer/file-listing/no-files.tsx +++ b/src/packages/frontend/project/explorer/file-listing/no-files.tsx @@ -6,7 +6,6 @@ import { Button } from "antd"; import { useMemo } from "react"; import { FormattedMessage, useIntl } from "react-intl"; - import { useTypedRedux } from "@cocalc/frontend/app-framework"; import { Paragraph, Text } from "@cocalc/frontend/components"; import { Icon } from "@cocalc/frontend/components/icon"; @@ -20,12 +19,12 @@ import { capitalize } from "@cocalc/util/misc"; import { COLORS } from "@cocalc/util/theme"; import { HelpAlert } from "./help-alert"; import { full_path_text } from "./utils"; +import { default_filename } from "@cocalc/frontend/account"; +import { join } from "path"; interface Props { name: string; actions: ProjectActions; - create_folder: () => void; - create_file: () => void; file_search: string; current_path?: string; project_id: string; @@ -34,9 +33,8 @@ interface Props { export default function NoFiles({ actions, - create_folder, - create_file, file_search = "", + current_path, project_id, configuration_main, }: Props) { @@ -109,12 +107,16 @@ export default function NoFiles({ padding: "30px", }} onClick={(): void => { - if (file_search.length === 0) { + if (!file_search?.trim()) { actions.set_active_tab("new"); } else if (file_search[file_search.length - 1] === "/") { - create_folder(); + actions.create_folder({ + name: join(current_path ?? "", file_search), + }); } else { - create_file(); + actions.create_file({ + name: join(current_path ?? "", actualNewFilename), + }); } }} > @@ -137,18 +139,31 @@ export default function NoFiles({ file_search={file_search} actual_new_filename={actualNewFilename} /> - {file_search.length > 0 && ( -
-

Or Select a File Type

- -
- )} +
+

Select a File Type

+ { + ext = ext ? ext : "ipynb"; + const filename = file_search.trim() + ? file_search + "." + ext + : default_filename(ext, project_id); + actions.create_file({ + name: join(current_path ?? "", filename), + }); + }} + create_folder={() => { + const filename = default_filename(undefined, project_id); + actions.create_folder({ + name: file_search.trim() + ? file_search + : join(current_path ?? "", filename), + }); + }} + projectActions={actions} + availableFeatures={availableFeatures} + filename={actualNewFilename} + /> +
); } diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index 803fbde4a0..7a27ec3cea 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -620,9 +620,6 @@ export class ProjectActions extends Actions { if (opts.change_history) { this.set_url_to_path(store.get("current_path") ?? "", ""); } - if (opts.update_file_listing) { - this.fetch_directory_listing(); - } break; case "new": @@ -1911,7 +1908,6 @@ export class ProjectActions extends Actions { // returns a function that takes the err and output and // does the right activity logging stuff. return (err?, output?) => { - this.fetch_directory_listing(); if (err) { this.set_activity({ id, error: err }); } else if ( @@ -1962,7 +1958,6 @@ export class ProjectActions extends Actions { throw err; } finally { this.set_activity({ id, stop: "" }); - this.fetch_directory_listing(); } }; @@ -2752,7 +2747,6 @@ export class ProjectActions extends Actions { }); return; } - this.fetch_directory_listing({ path: p, compute_server_id }); if (switch_over) { this.open_directory(p); } From 4c41caa9955993bed30dd5ca97529c7ab8ff4c95 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 23:11:06 +0000 Subject: [PATCH 079/798] fix package.json issue --- src/packages/test/package.json | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/packages/test/package.json b/src/packages/test/package.json index bed4234e0c..94c8d6ef9e 100644 --- a/src/packages/test/package.json +++ b/src/packages/test/package.json @@ -5,7 +5,10 @@ "exports": { "./*": "./dist/*.js" }, - "keywords": ["test", "cocalc"], + "keywords": [ + "test", + "cocalc" + ], "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf dist node_modules", @@ -28,10 +31,11 @@ }, "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/test", "devDependencies": { - "@types/node": "^18.16.14", + "@testing-library/jest-dom": "^6.6.3", + "@testing-library/react": "^16.3.0", "@types/debug": "^4.1.12", "@types/jest": "^29.5.14", - "@testing-library/jest-dom": "^6.6.3", + "@types/node": "^18.16.14", "jest-environment-jsdom": "^30.0.2" } } From 95c705475c0e3b4c55a73b185294178564560f47 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 23:13:19 +0000 Subject: [PATCH 080/798] fix depcheck issue with new test package --- src/packages/test/package.json | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/packages/test/package.json b/src/packages/test/package.json index 94c8d6ef9e..820229b57e 100644 --- a/src/packages/test/package.json +++ b/src/packages/test/package.json @@ -5,17 +5,14 @@ "exports": { "./*": "./dist/*.js" }, - "keywords": [ - "test", - "cocalc" - ], + "keywords": ["test", "cocalc"], "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf dist node_modules", "build": "pnpm exec tsc --build", "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", "test": "pnpm exec jest --forceExit", - "depcheck": "pnpx depcheck --ignores events" + "depcheck": "pnpx depcheck --ignores @types/debug,@types/jest,@types/node,jest-environment-jsdom" }, "author": "SageMath, Inc.", "license": "SEE LICENSE.md", From d332bb6d05cb5fbc0450902b5f23c50c39c22ce6 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 24 Jul 2025 01:54:04 +0000 Subject: [PATCH 081/798] refactor code for filtering the listing --- .../frontend/project/explorer/action-bar.tsx | 6 ++--- .../explorer/file-listing/file-listing.tsx | 18 +++++++-------- .../project/listing/filter-listing.ts | 23 +++++++++++++++++++ 3 files changed, 34 insertions(+), 13 deletions(-) create mode 100644 src/packages/frontend/project/listing/filter-listing.ts diff --git a/src/packages/frontend/project/explorer/action-bar.tsx b/src/packages/frontend/project/explorer/action-bar.tsx index 7cfc6f58d5..eec0f43eda 100644 --- a/src/packages/frontend/project/explorer/action-bar.tsx +++ b/src/packages/frontend/project/explorer/action-bar.tsx @@ -8,7 +8,6 @@ import * as immutable from "immutable"; import { throttle } from "lodash"; import React, { useEffect, useRef, useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; - import { Button, ButtonToolbar } from "@cocalc/frontend/antd-bootstrap"; import { Gap, Icon } from "@cocalc/frontend/components"; import { useStudentProjectFunctionality } from "@cocalc/frontend/course"; @@ -19,7 +18,6 @@ import { labels } from "@cocalc/frontend/i18n"; import { file_actions, ProjectActions } from "@cocalc/frontend/project_store"; import * as misc from "@cocalc/util/misc"; import { COLORS } from "@cocalc/util/theme"; - import { useProjectContext } from "../context"; const ROW_INFO_STYLE = { @@ -43,7 +41,7 @@ interface Props { project_is_running?: boolean; } -export const ActionBar: React.FC = (props: Props) => { +export function ActionBar(props: Props) { const intl = useIntl(); const [showLabels, setShowLabels] = useState(true); const { mainWidthPx } = useProjectContext(); @@ -350,7 +348,7 @@ export const ActionBar: React.FC = (props: Props) => { ); -}; +} export const ACTION_BUTTONS_DIR = [ "download", diff --git a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx index e85b0c754a..d5ed3dbb9f 100644 --- a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx +++ b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx @@ -34,6 +34,7 @@ import useFs from "@cocalc/frontend/project/listing/use-fs"; import useListing, { type SortField, } from "@cocalc/frontend/project/listing/use-listing"; +import filterListing from "@cocalc/frontend/project/listing/filter-listing"; interface Props { // TODO: everything but actions/redux should be immutable JS data, and use shouldComponentUpdate @@ -94,7 +95,7 @@ function sortDesc(active_file_sort?): { export function FileListing(props) { const fs = useFs({ project_id: props.project_id }); - const { listing, error } = useListing({ + let { listing, error } = useListing({ fs, path: props.current_path, ...sortDesc(props.active_file_sort), @@ -102,6 +103,13 @@ export function FileListing(props) { if (error) { return ; } + + listing = filterListing({ + listing, + search: props.file_search, + showHidden: props.show_hidden, + }); + if (listing == null) { return ; } @@ -123,17 +131,9 @@ function FileListing0({ configuration_main, file_search = "", isRunning, - show_hidden, stale, // show_masked, }: Props) { - if (!show_hidden) { - listing = listing.filter((x) => !x.name.startsWith(".")); - } - if (file_search) { - listing = listing.filter((x) => x.name.includes(file_search)); - } - const prev_current_path = usePrevious(current_path); function watch() { diff --git a/src/packages/frontend/project/listing/filter-listing.ts b/src/packages/frontend/project/listing/filter-listing.ts new file mode 100644 index 0000000000..53e296da7f --- /dev/null +++ b/src/packages/frontend/project/listing/filter-listing.ts @@ -0,0 +1,23 @@ +import { DirectoryListingEntry } from "@cocalc/util/types"; + +export default function filterListing({ + listing, + search, + showHidden, +}: { + listing?: DirectoryListingEntry[] | null; + search?: string; + showHidden?: boolean; +}): DirectoryListingEntry[] | null { + if (listing == null) { + return null; + } + if (!showHidden) { + listing = listing.filter((x) => !x.name.startsWith(".")); + } + search = search?.trim()?.toLowerCase(); + if (!search || search.startsWith("/")) { + return listing; + } + return listing.filter((x) => x.name.toLowerCase().includes(search)); +} From a7987bd590dc62947fc4e3a2dd30982b6fb18cc0 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 24 Jul 2025 04:57:29 +0000 Subject: [PATCH 082/798] ripping out the old listings related code and rewriting/replacing it -- this is in the middle --- src/packages/frontend/file-upload.tsx | 8 +- .../frontend/project/directory-listing.ts | 168 ----------- .../frontend/project/explorer/explorer.tsx | 92 +----- .../explorer/file-listing/file-listing.tsx | 52 +--- .../project/fetch-directory-listing.ts | 127 -------- .../frontend/project/listing/use-files.ts | 35 ++- .../frontend/project/listing/use-listing.ts | 88 ++++-- src/packages/frontend/project/utils.ts | 2 +- src/packages/frontend/project_actions.ts | 141 +++------ src/packages/frontend/project_store.ts | 278 ------------------ 10 files changed, 162 insertions(+), 829 deletions(-) delete mode 100644 src/packages/frontend/project/directory-listing.ts delete mode 100644 src/packages/frontend/project/fetch-directory-listing.ts diff --git a/src/packages/frontend/file-upload.tsx b/src/packages/frontend/file-upload.tsx index 9e1d1a81f8..171280e5c0 100644 --- a/src/packages/frontend/file-upload.tsx +++ b/src/packages/frontend/file-upload.tsx @@ -192,7 +192,7 @@ interface FileUploadWrapperProps { project_id: string; // The project to upload files to dest_path: string; // The path for files to be sent config?: object; // All supported dropzone.js config options - event_handlers: { + event_handlers?: { complete?: Function | Function[]; sending?: Function | Function[]; removedfile?: Function | Function[]; @@ -245,7 +245,7 @@ export function FileUploadWrapper({ previewTemplate: ReactDOMServer.renderToStaticMarkup( preview_template?.() ?? , ), - addRemoveLinks: event_handlers.removedfile != null, + addRemoveLinks: event_handlers?.removedfile != null, ...UPLOAD_OPTIONS, }, true, @@ -309,7 +309,7 @@ export function FileUploadWrapper({ // from the dropzone. This is true by default if there is // no "removedfile" handler, and false otherwise. function close_preview( - remove_all: boolean = event_handlers.removedfile == null, + remove_all: boolean = event_handlers?.removedfile == null, ) { if (typeof on_close === "function") { on_close(); @@ -381,7 +381,7 @@ export function FileUploadWrapper({ } function set_up_events(): void { - if (dropzone.current == null) { + if (dropzone.current == null || event_handlers == null) { return; } diff --git a/src/packages/frontend/project/directory-listing.ts b/src/packages/frontend/project/directory-listing.ts deleted file mode 100644 index a636d49617..0000000000 --- a/src/packages/frontend/project/directory-listing.ts +++ /dev/null @@ -1,168 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { server_time } from "@cocalc/util/misc"; -import { once, retry_until_success } from "@cocalc/util/async-utils"; -import { webapp_client } from "../webapp-client"; -import { redux } from "../app-framework"; -import { dirname } from "path"; - -//const log = (...args) => console.log("directory-listing", ...args); -const log = (..._args) => {}; - -interface ListingOpts { - project_id: string; - path: string; - hidden: boolean; - max_time_s: number; - group: string; - trigger_start_project?: boolean; - compute_server_id: number; -} - -// This makes an api call directly to the project to get a directory listing. - -export async function get_directory_listing(opts: ListingOpts) { - log("get_directory_listing", opts); - - let method, state, time0, timeout; - - if (["owner", "collaborator", "admin"].indexOf(opts.group) != -1) { - method = webapp_client.project_client.directory_listing; - // Also, make sure project starts running, in case it isn't. - state = (redux.getStore("projects") as any).getIn([ - "project_map", - opts.project_id, - "state", - "state", - ]); - if (state != null && state !== "running") { - timeout = 0.5; - time0 = server_time(); - if (opts.trigger_start_project === false) { - return { files: [], noRunning: true }; - } - redux.getActions("projects").start_project(opts.project_id); - } else { - timeout = 1; - } - } else { - throw Error("you do not have access to this project"); - } - - let listing_err: Error | undefined; - method = method.bind(webapp_client.project_client); - async function f(): Promise { - try { - return await method({ - project_id: opts.project_id, - path: opts.path, - hidden: opts.hidden, - compute_server_id: opts.compute_server_id, - timeout, - }); - } catch (err) { - if (err.message != null) { - if (err.message.indexOf("ENOENT") != -1) { - listing_err = Error("no_dir"); - return; - } else if (err.message.indexOf("ENOTDIR") != -1) { - listing_err = Error("not_a_dir"); - return; - } - } - if (timeout < 5) { - timeout *= 1.3; - } - throw err; - } - } - - let listing; - try { - listing = await retry_until_success({ - f, - max_time: opts.max_time_s * 1000, - start_delay: 100, - max_delay: 1000, - }); - } catch (err) { - listing_err = err; - } finally { - // no error, but `listing` has no value, too - // https://github.com/sagemathinc/cocalc/issues/3223 - if (!listing_err && listing == null) { - listing_err = Error("no_dir"); - } - if (time0 && state !== "running" && !listing_err) { - // successfully opened, started, and got directory listing - redux.getProjectActions(opts.project_id).log({ - event: "start_project", - time: server_time().valueOf() - time0.valueOf(), - }); - } - - if (listing_err) { - throw listing_err; - } else { - return listing; - } - } -} - -import { Listings } from "@cocalc/frontend/conat/listings"; - -export async function get_directory_listing2(opts: ListingOpts): Promise { - log("get_directory_listing2", opts); - const start = Date.now(); - const store = redux.getProjectStore(opts.project_id); - const compute_server_id = - opts.compute_server_id ?? store.get("compute_server_id"); - const listings: Listings = await store.get_listings(compute_server_id); - listings.watch(opts.path); - if (opts.path) { - listings.watch(dirname(opts.path)); - } - while (Date.now() - start < opts.max_time_s * 1000) { - if (listings.getMissing(opts.path)) { - if ( - store.getIn(["directory_listings", compute_server_id, opts.path]) != - null - ) { - // just update an already loaded listing: - try { - const files = await listings.getListingDirectly( - opts.path, - opts.trigger_start_project, - ); - return { files }; - } catch (err) { - console.log( - `WARNING: temporary problem getting directory listing -- ${err}`, - ); - } - } else { - // ensure all listing entries get loaded soon. - redux - .getProjectActions(opts.project_id) - ?.fetch_directory_listing_directly( - opts.path, - opts.trigger_start_project, - compute_server_id, - ); - } - } - // return what we have now, if anything. - const files = await listings.get(opts.path, opts.trigger_start_project); - if (files != null) { - return { files }; - } - await once( - listings, - "change", - opts.max_time_s * 1000 - (Date.now() - start), - ); - } -} diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index e4818639cd..ccaddf8ead 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -7,7 +7,7 @@ import * as immutable from "immutable"; import * as _ from "lodash"; import React from "react"; import { UsersViewing } from "@cocalc/frontend/account/avatar/users-viewing"; -import { Button, ButtonGroup, Col, Row } from "@cocalc/frontend/antd-bootstrap"; +import { Col, Row } from "@cocalc/frontend/antd-bootstrap"; import { project_redux_name, rclass, @@ -20,7 +20,6 @@ import { A, ActivityDisplay, ErrorDisplay, - Icon, Loading, SettingBox, } from "@cocalc/frontend/components"; @@ -49,12 +48,6 @@ import { NewButton } from "./new-button"; import { PathNavigator } from "./path-navigator"; import { SearchBar } from "./search-bar"; import ExplorerTour from "./tour/tour"; -import { ListingItem } from "./types"; - -function pager_range(page_size, page_number) { - const start_index = page_size * page_number; - return { start_index, end_index: start_index + page_size }; -} export type Configuration = ShallowTypedMap<{ main: MainConfiguration }>; @@ -105,11 +98,6 @@ interface ReduxProps { selected_file_index?: number; file_creation_error?: string; ext_selection?: string; - displayed_listing: { - listing: ListingItem[]; - error: any; - file_map: Map; - }; new_name?: string; library?: object; show_library?: boolean; @@ -184,7 +172,6 @@ const Explorer0 = rclass( selected_file_index: rtypes.number, file_creation_error: rtypes.string, ext_selection: rtypes.string, - displayed_listing: rtypes.object, new_name: rtypes.string, library: rtypes.object, show_library: rtypes.bool, @@ -291,49 +278,16 @@ const Explorer0 = rclass( this.props.actions.setState({ file_search: "", page_number: 0 }); }; - render_paging_buttons(num_pages: number): React.JSX.Element | undefined { - if (num_pages > 1) { - return ( - - - - - - - - - - ); - } - } - - render_files_action_box(file_map?) { - if (file_map == undefined) { - return; - } + render_files_action_box() { return ( @@ -366,15 +320,15 @@ const Explorer0 = rclass( ); } - render_files_actions(listing, project_is_running) { + render_files_actions(project_is_running) { return ( this.props.actions.fetch_directory_listing(), - }} config={{ clickable: ".upload-button" }} style={{ flex: "1 0 auto", @@ -476,9 +427,7 @@ const Explorer0 = rclass( ); } - render_control_row( - visible_listing: ListingItem[] | undefined, - ): React.JSX.Element { + render_control_row(): React.JSX.Element { return (
{this.render_error()} {this.render_activity()} - {this.render_control_row(visible_listing)} + {this.render_control_row()} {this.props.ext_selection != null && ( )} @@ -680,9 +614,7 @@ const Explorer0 = rclass( minWidth: "20em", }} > - {listing != undefined - ? this.render_files_actions(listing, project_is_running) - : undefined} + {this.render_files_actions(project_is_running)}
{this.render_project_files_buttons()} @@ -695,7 +627,7 @@ const Explorer0 = rclass( {this.props.checked_files.size > 0 && this.props.file_action != undefined ? ( - {this.render_files_action_box(file_map)} + {this.render_files_action_box()} ) : undefined} @@ -732,7 +664,6 @@ const SearchTerminalBar = React.forwardRef( current_path, file_search, actions, - visible_listing, selected_file_index, file_creation_error, create_file, @@ -742,7 +673,6 @@ const SearchTerminalBar = React.forwardRef( current_path: string; file_search: string; actions: ProjectActions; - visible_listing: ListingItem[] | undefined; selected_file_index?: number; file_creation_error?: string; create_file: (ext?: string, switch_over?: boolean) => void; @@ -750,6 +680,8 @@ const SearchTerminalBar = React.forwardRef( }, ref: React.LegacyRef | undefined, ) => { + // [ ] TODO + const visible_listing = []; return (
; @@ -130,41 +118,9 @@ function FileListing0({ sort_by, configuration_main, file_search = "", - isRunning, stale, // show_masked, }: Props) { - const prev_current_path = usePrevious(current_path); - - function watch() { - watchFiles({ actions, current_path }); - } - - // once after mounting, when changing paths, and in regular intervals call watch() - useEffect(() => { - watch(); - }, []); - - useEffect(() => { - if (current_path != prev_current_path) watch(); - }, [current_path, prev_current_path]); - - useInterval(watch, WATCH_THROTTLE_MS); - - const [missing, setMissing] = useState(0); - - useEffect(() => { - if (isRunning) return; - if (listing.length == 0) return; - (async () => { - const missing = await redux - .getProjectStore(project_id) - .get_listings() - .getMissingUsingDatabase(current_path); - setMissing(missing ?? 0); - })(); - }, [current_path, isRunning]); - const computeServerId = useTypedRedux({ project_id }, "compute_server_id"); function render_row( @@ -287,11 +243,9 @@ function FileListing0({ > missing {missing} files} other {}}. + defaultMessage={`Showing stale directory listing. To update the directory listing start this project.`} values={{ - is_missing: missing > 0, - missing, a: (c) => ( { diff --git a/src/packages/frontend/project/fetch-directory-listing.ts b/src/packages/frontend/project/fetch-directory-listing.ts deleted file mode 100644 index 026de8cbf3..0000000000 --- a/src/packages/frontend/project/fetch-directory-listing.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { is_running_or_starting } from "./project-start-warning"; -import type { ProjectActions } from "@cocalc/frontend/project_actions"; -import { trunc_middle, uuid } from "@cocalc/util/misc"; -import { get_directory_listing } from "./directory-listing"; -import { fromJS, Map } from "immutable"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; - -//const log = (...args) => console.log("fetchDirectoryListing", ...args); -const log = (..._args) => {}; - -interface FetchDirectoryListingOpts { - path?: string; - // WARNING: THINK VERY HARD BEFORE YOU USE force=true, due to efficiency! - force?: boolean; - // can be explicit here; otherwise will fall back to store.get('compute_server_id') - compute_server_id?: number; -} - -function getPath( - actions, - opts?: FetchDirectoryListingOpts, -): string | undefined { - return opts?.path ?? actions.get_store()?.get("current_path"); -} - -function getComputeServerId(actions, opts): number { - return ( - opts?.compute_server_id ?? - actions.get_store()?.get("compute_server_id") ?? - 0 - ); -} - -const fetchDirectoryListing = reuseInFlight( - async ( - actions: ProjectActions, - opts: FetchDirectoryListingOpts = {}, - ): Promise => { - let status; - let store = actions.get_store(); - if (store == null) { - return; - } - const { force } = opts; - const path = getPath(actions, opts); - const compute_server_id = getComputeServerId(actions, opts); - - if (force && path != null) { - // update our interest. - store.get_listings().watch(path, true); - } - log({ force, path, compute_server_id }); - - if (path == null) { - // nothing to do if path isn't defined -- there is no current path -- - // see https://github.com/sagemathinc/cocalc/issues/818 - return; - } - - const id = uuid(); - if (path) { - status = `Loading file list - ${trunc_middle(path, 30)}`; - } else { - status = "Loading file list"; - } - - let error = ""; - try { - // only show actions indicator, if the project is running or starting - // if it is stopped, we get a stale listing from the database, which is fine. - if (is_running_or_starting(actions.project_id)) { - log("show activity"); - actions.set_activity({ id, status }); - } - - log("make sure user is fully signed in"); - await actions.redux.getStore("account").async_wait({ - until: (s) => s.get("is_logged_in") && s.get("account_id"), - }); - - log("getting listing"); - const listing = await get_directory_listing({ - project_id: actions.project_id, - path, - hidden: true, - max_time_s: 15, - trigger_start_project: false, - group: "collaborator", // nothing else is implemented - compute_server_id, - }); - log("got ", listing.files); - const value = fromJS(listing.files); - log("saving result"); - store = actions.get_store(); - if (store == null) { - return; - } - const directory_listings = store.get("directory_listings"); - let listing2 = directory_listings.get(compute_server_id) ?? Map(); - if (listing.noRunning && (listing2.get(path)?.size ?? 0) > 0) { - // do not change it - return; - } - listing2 = listing2.set(path, value); - actions.setState({ - directory_listings: directory_listings.set(compute_server_id, listing2), - }); - } catch (err) { - log("error", err); - error = `${err}`; - } finally { - actions.set_activity({ id, stop: "", error }); - } - }, - { - createKey: (args) => { - const actions = args[0]; - // reuse in flight on the project id, compute server id and path - return `${actions.project_id}-${getComputeServerId( - actions, - args[1], - )}-${getPath(actions, args[1])}`; - }, - }, -); - -export default fetchDirectoryListing; diff --git a/src/packages/frontend/project/listing/use-files.ts b/src/packages/frontend/project/listing/use-files.ts index 925a284894..8c6bc7f130 100644 --- a/src/packages/frontend/project/listing/use-files.ts +++ b/src/packages/frontend/project/listing/use-files.ts @@ -13,20 +13,45 @@ import { type Files } from "@cocalc/conat/files/listing"; import { type FilesystemClient } from "@cocalc/conat/files/fs"; import { type ConatError } from "@cocalc/conat/core/client"; import useCounter from "@cocalc/frontend/app-framework/counter-hook"; +import LRU from "lru-cache"; +import type { JSONValue } from "@cocalc/util/types"; +export { Files }; const DEFAULT_THROTTLE_FILE_UPDATE = 500; +const CACHE_SIZE = 100; + +const cache = new LRU({ max: CACHE_SIZE }); + +export function getFiles({ + cacheId, + path, +}: { + cacheId?: JSONValue; + path: string; +}): Files | null { + if (cacheId == null) { + return null; + } + return cache.get(key(cacheId, path)) ?? null; +} + export default function useFiles({ fs, path, throttleUpdate = DEFAULT_THROTTLE_FILE_UPDATE, + cacheId, }: { // fs = undefined is supported and just waits until you provide a fs that is defined fs?: FilesystemClient | null; path: string; throttleUpdate?: number; + // cacheId -- if given, save most recently loaded Files for a path in an in-memory LRU cache. + // An example cacheId could be {project_id, compute_server_id}. + // This is used to speed up the first load, and can also be fetched synchronously. + cacheId?: JSONValue; }): { files: Files | null; error: null | ConatError; refresh: () => void } { - const [files, setFiles] = useState(null); + const [files, setFiles] = useState(getFiles({ cacheId, path })); const [error, setError] = useState(null); const { val: counter, inc: refresh } = useCounter(); const listingRef = useRef(null); @@ -48,7 +73,9 @@ export default function useFiles({ setFiles(null); return; } - + if (cacheId != null) { + cache.set(key(cacheId, path), listing.files); + } const update = () => { setFiles({ ...listing.files }); }; @@ -68,3 +95,7 @@ export default function useFiles({ return { files, error, refresh }; } + +function key(cacheId: JSONValue, path: string) { + return JSON.stringify({ cacheId, path }); +} diff --git a/src/packages/frontend/project/listing/use-listing.ts b/src/packages/frontend/project/listing/use-listing.ts index 75ffc37f40..da5a1870ac 100644 --- a/src/packages/frontend/project/listing/use-listing.ts +++ b/src/packages/frontend/project/listing/use-listing.ts @@ -10,16 +10,35 @@ import { field_cmp } from "@cocalc/util/misc"; import useFiles from "./use-files"; import { type FilesystemClient } from "@cocalc/conat/files/fs"; import { type ConatError } from "@cocalc/conat/core/client"; +import type { JSONValue } from "@cocalc/util/types"; +import { getFiles, type Files } from "./use-files"; export type SortField = "name" | "mtime" | "size" | "type"; export type SortDirection = "asc" | "desc"; +export function getListing({ + path, + cacheId, + sortField, + sortDirection, +}: { + path; + string; + cacheId?: JSONValue; + sortField?: SortField; + sortDirection?: SortDirection; +}): null | DirectoryListingEntry[] { + const files = getFiles({ cacheId, path }); + return filesToListing({ files, sortField, sortDirection }); +} + export default function useListing({ fs, path, sortField = "name", sortDirection = "asc", throttleUpdate, + cacheId, }: { // fs = undefined is supported and just waits until you provide a fs that is defined fs?: FilesystemClient | null; @@ -27,38 +46,59 @@ export default function useListing({ sortField?: SortField; sortDirection?: SortDirection; throttleUpdate?: number; + cacheId?: JSONValue; }): { listing: null | DirectoryListingEntry[]; error: null | ConatError; refresh: () => void; } { - const { files, error, refresh } = useFiles({ fs, path, throttleUpdate }); + const { files, error, refresh } = useFiles({ + fs, + path, + throttleUpdate, + cacheId, + }); const listing = useMemo(() => { - if (files == null) { - return null; - } - const v: DirectoryListingEntry[] = []; - for (const name in files) { - v.push({ name, ...files[name] }); - } - if ( - sortField != "name" && - sortField != "mtime" && - sortField != "size" && - sortField != "type" - ) { - console.warn(`invalid sort field: '${sortField}'`); - } - v.sort(field_cmp(sortField)); - if (sortDirection == "desc") { - v.reverse(); - } else if (sortDirection == "asc") { - } else { - console.warn(`invalid sort direction: '${sortDirection}'`); - } - return v; + return filesToListing({ files, sortField, sortDirection }); }, [sortField, sortDirection, files]); return { listing, error, refresh }; } + +function filesToListing({ + files, + sortField = "name", + sortDirection = "asc", +}: { + files?: Files | null; + sortField?: SortField; + sortDirection?: SortDirection; +}): null | DirectoryListingEntry[] { + if (files == null) { + return null; + } + if (files == null) { + return null; + } + const v: DirectoryListingEntry[] = []; + for (const name in files) { + v.push({ name, ...files[name] }); + } + if ( + sortField != "name" && + sortField != "mtime" && + sortField != "size" && + sortField != "type" + ) { + console.warn(`invalid sort field: '${sortField}'`); + } + v.sort(field_cmp(sortField)); + if (sortDirection == "desc") { + v.reverse(); + } else if (sortDirection == "asc") { + } else { + console.warn(`invalid sort direction: '${sortDirection}'`); + } + return v; +} diff --git a/src/packages/frontend/project/utils.ts b/src/packages/frontend/project/utils.ts index e1bfb9cbaf..d0f2b5a1c1 100644 --- a/src/packages/frontend/project/utils.ts +++ b/src/packages/frontend/project/utils.ts @@ -78,7 +78,7 @@ export class NewFilenames { // generate a new filename, by optionally avoiding the keys in the dictionary public gen( type?: NewFilenameTypes, - avoid?: { [name: string]: boolean }, + avoid?: { [name: string]: any } | null, ): string { type = this.sanitize_type(type); // reset the enumeration if type changes diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index 7a27ec3cea..d76df13f18 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -49,7 +49,6 @@ import { import Fragment, { FragmentId } from "@cocalc/frontend/misc/fragment-id"; import * as project_file from "@cocalc/frontend/project-file"; import { delete_files } from "@cocalc/frontend/project/delete-files"; -import fetchDirectoryListing from "@cocalc/frontend/project/fetch-directory-listing"; import { ProjectEvent, SoftwareEnvironmentEvent, @@ -108,6 +107,11 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { MARKERS } from "@cocalc/util/sagews"; import { client_db } from "@cocalc/util/schema"; import { get_editor } from "./editors/react-wrapper"; +import { type FilesystemClient } from "@cocalc/conat/files/fs"; +import { + getFiles, + type Files, +} from "@cocalc/frontend/project/listing/use-files"; const { defaults, required } = misc; @@ -1498,8 +1502,6 @@ export class ProjectActions extends Actions { page_number: 0, most_recent_file_click: undefined, }); - - store.get_listings().watch(path, true); }; setComputeServerId = (compute_server_id: number) => { @@ -1559,33 +1561,14 @@ export class ProjectActions extends Actions { // Update the directory listing cache for the given path. // Uses current path if path not provided. - fetch_directory_listing = async (opts?): Promise => { - await fetchDirectoryListing(this, opts); + fetch_directory_listing = async (_opts?): Promise => { + console.log("TODO: eliminate code that uses fetch_directory_listing"); }; - public async fetch_directory_listing_directly( - path: string, - trigger_start_project?: boolean, - compute_server_id?: number, - ): Promise { - const store = this.get_store(); - if (store == null) return; - compute_server_id = this.getComputeServerId(compute_server_id); - const listings = store.get_listings(compute_server_id); - try { - const files = await listings.getListingDirectly( - path, - trigger_start_project, - ); - const directory_listings = store.get("directory_listings"); - let listing = directory_listings.get(compute_server_id) ?? Map(); - listing = listing.set(path, files); - this.setState({ - directory_listings: directory_listings.set(compute_server_id, listing), - }); - } catch (err) { - console.warn(`Unable to fetch directory listing -- "${err}"`); - } + public async fetch_directory_listing_directly(): Promise { + console.log( + "TODO: eliminate code that uses fetch_directory_listing_directly", + ); } // Sets the active file_sort to next_column_name @@ -1755,40 +1738,6 @@ export class ProjectActions extends Actions { }); } - // this isn't really an action, but very helpful! - public get_filenames_in_current_dir(): - | { [name: string]: boolean } - | undefined { - const store = this.get_store(); - if (store == undefined) { - return; - } - - const files_in_dir = {}; - // This will set files_in_dir to our current view of the files in the current - // directory (at least the visible ones) or do nothing in case we don't know - // anything about files (highly unlikely). Unfortunately (for this), our - // directory listings are stored as (immutable) lists, so we have to make - // a map out of them. - const compute_server_id = store.get("compute_server_id"); - const listing = store.getIn([ - "directory_listings", - compute_server_id, - store.get("current_path"), - ]); - - if (typeof listing === "string") { - // must be an error - return undefined; // simple fallback - } - if (listing != null) { - listing.map(function (x) { - files_in_dir[x.get("name")] = true; - }); - } - return files_in_dir; - } - suggestDuplicateFilenameInCurrentDirectory = ( name: string, ): string | undefined => { @@ -2496,21 +2445,40 @@ export class ProjectActions extends Actions { } } + private _filesystem: FilesystemClient; + fs = (): FilesystemClient => { + this._filesystem ??= webapp_client.conat_client + .conat() + .fs({ project_id: this.project_id }); + return this._filesystem; + }; + + // if available in cache, this returns the filenames in the current directory, + // which is often useful, or null if they are not known. This is sync, so it + // can't query the backend. (Here Files is a map from path names to data about them.) + get_filenames_in_current_dir = (): Files | null => { + const store = this.get_store(); + if (store == undefined) { + return null; + } + const path = store.get("current_path"); + if (path == null) { + return null; + } + // todo: compute_server_id here and in place that does useListing! + return getFiles({ cacheId: { project_id: this.project_id }, path }); + }; + // return true if exists and is a directory - private async isdir(path: string): Promise { + isdir = async (path: string): Promise => { if (path == "") return true; // easy special case try { - await webapp_client.project_client.exec({ - project_id: this.project_id, - command: "test", - args: ["-d", path], - err_on_exit: true, - }); - return true; + const stats = await this.fs().stat(path); + return stats.isDirectory(); } catch (_) { return false; } - } + }; public async move_files(opts: { src: string[]; @@ -3235,17 +3203,16 @@ export class ProjectActions extends Actions { // log // settings // search - async load_target( + load_target = async ( target, foreground = true, ignore_kiosk = false, change_history = true, fragmentId?: FragmentId, - ): Promise { + ): Promise => { const segments = target.split("/"); const full_path = segments.slice(1).join("/"); const parent_path = segments.slice(1, segments.length - 1).join("/"); - const last = segments.slice(-1).join(); const main_segment = segments[0] as FixedTab | "home"; switch (main_segment) { case "active": @@ -3264,27 +3231,9 @@ export class ProjectActions extends Actions { if (store == null) { return; // project closed already } - // We check whether the path is a directory or not, first by checking if - // we have a recent directory listing in our cache, and if not, by calling - // isdir, which is a single exec. - let isdir; - let { item, err } = store.get_item_in_path(last, parent_path); - if (item == null || err) { - try { - isdir = await webapp_client.project_client.isdir({ - project_id: this.project_id, - path: normalize(full_path), - }); - } catch (err) { - // TODO: e.g., project is not running? - // I've seen this, e.g., when trying to open a file when not running, and it just - // gets retried and works. - console.log(`Error opening '${target}' -- ${err}`); - return; - } - } else { - isdir = item.get("isdir"); - } + + // We check whether the path is a directory or not: + const isdir = await this.isdir(full_path); if (isdir) { this.open_directory(full_path, change_history); } else { @@ -3343,7 +3292,7 @@ export class ProjectActions extends Actions { misc.unreachable(main_segment); console.warn(`project/load_target: don't know segment ${main_segment}`); } - } + }; set_compute_image = async (compute_image: string): Promise => { const projects_store = this.redux.getStore("projects"); diff --git a/src/packages/frontend/project_store.ts b/src/packages/frontend/project_store.ts index 3b06e820c6..9ce25746ec 100644 --- a/src/packages/frontend/project_store.ts +++ b/src/packages/frontend/project_store.ts @@ -25,7 +25,6 @@ import { Table, TypedMap, } from "@cocalc/frontend/app-framework"; -import { Listings, listings } from "@cocalc/frontend/conat/listings"; import { fileURL } from "@cocalc/frontend/lib/cocalc-urls"; import { get_local_storage } from "@cocalc/frontend/misc"; import { QueryParams } from "@cocalc/frontend/misc/query-params"; @@ -42,8 +41,6 @@ import { ProjectConfiguration, } from "@cocalc/frontend/project_configuration"; import * as misc from "@cocalc/util/misc"; -import { compute_file_masks } from "./project/explorer/compute-file-masks"; -import { DirectoryListing } from "./project/explorer/types"; import { FixedTab } from "./project/page/file-tab"; import { FlyoutActiveMode, @@ -76,8 +73,6 @@ export interface ProjectStoreState { just_closed_files: immutable.List; public_paths?: immutable.Map>; - // directory_listings is a map from compute_server_id to {path:[listing for that path on the compute server]} - directory_listings: immutable.Map; show_upload: boolean; create_file_alert: boolean; displayed_listing?: any; // computed(object), @@ -181,7 +176,6 @@ export interface ProjectStoreState { export class ProjectStore extends Store { public project_id: string; private previous_runstate: string | undefined; - private listings: { [compute_server_id: number]: Listings } = {}; public readonly computeServerIdLocalStorageKey: string; // Function to call to initialize one of the tables in this store. @@ -226,10 +220,6 @@ export class ProjectStore extends Store { if (projects_store !== undefined) { projects_store.removeListener("change", this._projects_store_change); } - for (const id in this.listings) { - this.listings[id].close(); - delete this.listings[id]; - } // close any open file tabs, properly cleaning up editor state: const open = this.get("open_files")?.toJS(); if (open != null) { @@ -296,7 +286,6 @@ export class ProjectStore extends Store { open_files: immutable.Map>({}), open_files_order: immutable.List([]), just_closed_files: immutable.List([]), - directory_listings: immutable.Map(), // immutable, show_upload: false, create_file_alert: false, displayed_listing: undefined, // computed(object), @@ -371,156 +360,6 @@ export class ProjectStore extends Store { }, }, - // cached pre-processed file listing, which should always be up to date when - // called, and properly depends on dependencies. - displayed_listing: { - dependencies: [ - "active_file_sort", - "current_path", - "directory_listings", - "stripped_public_paths", - "file_search", - "other_settings", - "show_hidden", - "show_masked", - "compute_server_id", - ] as const, - fn: () => { - const search_escape_char = "/"; - const listingStored = this.getIn([ - "directory_listings", - this.get("compute_server_id"), - this.get("current_path"), - ]); - if (typeof listingStored === "string") { - if ( - listingStored.indexOf("ECONNREFUSED") !== -1 || - listingStored.indexOf("ENOTFOUND") !== -1 - ) { - return { error: "no_instance" }; // the host VM is down - } else if (listingStored.indexOf("o such path") !== -1) { - return { error: "no_dir" }; - } else if (listingStored.indexOf("ot a directory") !== -1) { - return { error: "not_a_dir" }; - } else if (listingStored.indexOf("not running") !== -1) { - // yes, no underscore. - return { error: "not_running" }; - } else { - return { error: listingStored }; - } - } - if (listingStored == null) { - return {}; - } - try { - if (listingStored?.errno) { - return { error: misc.to_json(listingStored) }; - } - } catch (err) { - return { - error: "Error getting directory listing - please try again.", - }; - } - - if (listingStored?.toJS == null) { - return { - error: "Unable to get directory listing - please try again.", - }; - } - - // We can proceed and get the listing as a JS object. - let listing: DirectoryListing = listingStored.toJS(); - - if (this.get("other_settings").get("mask_files")) { - compute_file_masks(listing); - } - - if (this.get("current_path") === ".snapshots") { - compute_snapshot_display_names(listing); - } - - const search = this.get("file_search"); - if (search && search[0] !== search_escape_char) { - listing = _matched_files(search.toLowerCase(), listing); - } - - const sorter = (() => { - switch (this.get("active_file_sort").get("column_name")) { - case "name": - return _sort_on_string_field("name"); - case "time": - return _sort_on_numerical_field("mtime", -1); - case "size": - return _sort_on_numerical_field("size"); - case "type": - return (a, b) => { - if (a.isdir && !b.isdir) { - return -1; - } else if (b.isdir && !a.isdir) { - return 1; - } else { - return misc.cmp_array( - a.name.split(".").reverse(), - b.name.split(".").reverse(), - ); - } - }; - } - })(); - - listing.sort(sorter); - - if (this.get("active_file_sort").get("is_descending")) { - listing.reverse(); - } - - if (!this.get("show_hidden")) { - listing = (() => { - const result: DirectoryListing = []; - for (const l of listing) { - if (!l.name.startsWith(".")) { - result.push(l); - } - } - return result; - })(); - } - - if (!this.get("show_masked", true)) { - // if we do not gray out files (and hence haven't computed the file mask yet) - // we do it now! - if (!this.get("other_settings").get("mask_files")) { - compute_file_masks(listing); - } - - const filtered: DirectoryListing = []; - for (const f of listing) { - if (!f.mask) filtered.push(f); - } - listing = filtered; - } - - const file_map = {}; - for (const v of listing) { - file_map[v.name] = v; - } - - const data = { - listing, - public: {}, - path: this.get("current_path"), - file_map, - }; - - mutate_data_to_compute_public_files( - data, - this.get("stripped_public_paths"), - this.get("current_path"), - ); - - return data; - }, - }, stripped_public_paths: { dependencies: ["public_paths"] as const, @@ -565,20 +404,6 @@ export class ProjectStore extends Store { return this.getIn(["open_files", path]) != null; }; - get_item_in_path = (name, path) => { - const listing = this.get("directory_listings").get(path); - if (typeof listing === "string") { - // must be an error - return { err: listing }; - } - return { - item: - listing != null - ? listing.find((val) => val.get("name") === name) - : undefined, - }; - }; - fileURL = (path, compute_server_id?: number) => { return fileURL({ project_id: this.project_id, @@ -605,89 +430,6 @@ export class ProjectStore extends Store { // note that component is NOT an immutable.js object: return this.getIn(["open_files", path, "component"])?.Editor != null; } - - public get_listings(compute_server_id: number | null = null): Listings { - const computeServerId = compute_server_id ?? this.get("compute_server_id"); - if (this.listings[computeServerId] == null) { - const listingsTable = listings(this.project_id, computeServerId); - this.listings[computeServerId] = listingsTable; - listingsTable.watch(this.get("current_path") ?? "", true); - listingsTable.on("change", async (paths) => { - let directory_listings_for_server = - this.getIn(["directory_listings", computeServerId]) ?? - immutable.Map(); - - const missing: string[] = []; - for (const path of paths) { - if (listingsTable.getMissing(path)) { - missing.push(path); - } - const files = await listingsTable.getForStore(path); - directory_listings_for_server = directory_listings_for_server.set( - path, - files, - ); - } - const f = () => { - const actions = redux.getProjectActions(this.project_id); - const directory_listings = this.get("directory_listings").set( - computeServerId, - directory_listings_for_server, - ); - actions.setState({ directory_listings }); - }; - f(); - - if (missing.length > 0) { - for (const path of missing) { - try { - const files = immutable.fromJS( - await listingsTable.getListingDirectly(path), - ); - directory_listings_for_server = directory_listings_for_server.set( - path, - files, - ); - } catch { - // happens if e.g., the project is not running - continue; - } - } - f(); - } - }); - } - if (this.listings[computeServerId] == null) { - throw Error("bug"); - } - return this.listings[computeServerId]; - } -} - -function _matched_files(search: string, listing: DirectoryListing) { - if (listing == null) { - return []; - } - const words = misc.search_split(search); - const v: DirectoryListing = []; - for (const x of listing) { - const name = (x.display_name ?? x.name ?? "").toLowerCase(); - if ( - misc.search_match(name, words) || - (x.isdir && misc.search_match(name + "/", words)) - ) { - v.push(x); - } - } - return v; -} - -function compute_snapshot_display_names(listing): void { - for (const item of listing) { - const tm = misc.parse_bup_timestamp(item.name); - item.display_name = `${tm}`; - item.mtime = tm.valueOf() / 1000; - } } // Mutates data to include info on public paths. @@ -723,26 +465,6 @@ export function mutate_data_to_compute_public_files( } } -function _sort_on_string_field(field) { - return function (a, b) { - return misc.cmp( - a[field] !== undefined ? a[field].toLowerCase() : "", - b[field] !== undefined ? b[field].toLowerCase() : "", - ); - }; -} - -function _sort_on_numerical_field(field, factor = 1) { - return (a, b) => { - const c = misc.cmp( - (a[field] != null ? a[field] : -1) * factor, - (b[field] != null ? b[field] : -1) * factor, - ); - if (c) return c; - // break ties using the name, so well defined. - return misc.cmp(a.name, b.name) * factor; - }; -} export function init(project_id: string, redux: AppRedux): ProjectStore { const name = project_redux_name(project_id); From bf6f1824dddcf2a6d280b3f199ff4c23837456c9 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 24 Jul 2025 05:11:20 +0000 Subject: [PATCH 083/798] trivial --- src/packages/frontend/components/virtuoso-scroll-hook.ts | 4 +++- .../frontend/project/explorer/file-listing/file-listing.tsx | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/packages/frontend/components/virtuoso-scroll-hook.ts b/src/packages/frontend/components/virtuoso-scroll-hook.ts index 8972a24333..43a2050623 100644 --- a/src/packages/frontend/components/virtuoso-scroll-hook.ts +++ b/src/packages/frontend/components/virtuoso-scroll-hook.ts @@ -15,6 +15,8 @@ the upstream Virtuoso project: https://github.com/petyosi/react-virtuoso/blob/m import LRU from "lru-cache"; import { useCallback, useMemo, useRef } from "react"; +const DEFAULT_VIEWPORT = 1000; + export interface ScrollState { index: number; offset: number; @@ -64,7 +66,7 @@ export default function useVirtuosoScrollHook({ }, [onScroll, cacheId]); return { - increaseViewportBy: 2000 /* a lot better default than 0 */, + increaseViewportBy: DEFAULT_VIEWPORT, initialTopMostItemIndex: (cacheId ? (cache.get(cacheId) ?? initialState) : initialState) ?? 0, scrollerRef: handleScrollerRef, diff --git a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx index f7986dc500..4ef83fc7b7 100644 --- a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx +++ b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx @@ -81,10 +81,11 @@ function sortDesc(active_file_sort?): { } export function FileListing(props) { + const path = props.current_path; const fs = useFs({ project_id: props.project_id }); let { listing, error } = useListing({ fs, - path: props.current_path, + path, ...sortDesc(props.active_file_sort), cacheId: { project_id: props.project_id }, }); From e3a31bdfd7641a227464d6807419bfebdd47db8a Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 26 Jul 2025 16:00:14 +0000 Subject: [PATCH 084/798] conat fs.readdir -- implement options --- .../conat/files/test/local-path.test.ts | 58 ++++++++++++++++ src/packages/backend/files/sandbox/index.ts | 21 +++++- src/packages/conat/core/client.ts | 4 +- src/packages/conat/files/fs.ts | 67 ++++++++++++++++++- src/packages/conat/sync-doc/syncdb.ts | 6 +- src/packages/conat/sync-doc/syncstring.ts | 4 +- .../frontend/course/assignments/actions.ts | 8 +-- src/packages/sync/editor/generic/sync-doc.ts | 4 +- 8 files changed, 155 insertions(+), 17 deletions(-) diff --git a/src/packages/backend/conat/files/test/local-path.test.ts b/src/packages/backend/conat/files/test/local-path.test.ts index 43825d45bc..527012f559 100644 --- a/src/packages/backend/conat/files/test/local-path.test.ts +++ b/src/packages/backend/conat/files/test/local-path.test.ts @@ -118,6 +118,64 @@ describe("use all the standard api functions of fs", () => { expect(v).toEqual(["0", "1", "2", "3", "4", fire]); }); + it("readdir with the withFileTypes option", async () => { + const path = "readdir-1"; + await fs.mkdir(path); + expect(await fs.readdir(path, { withFileTypes: true })).toEqual([]); + await fs.writeFile(join(path, "a.txt"), ""); + + { + const v = await fs.readdir(path, { withFileTypes: true }); + expect(v.map(({ name }) => name)).toEqual(["a.txt"]); + expect(v.map((x) => x.isFile())).toEqual([true]); + } + { + await fs.mkdir(join(path, "co")); + const v = await fs.readdir(path, { withFileTypes: true }); + expect(v.map(({ name }) => name)).toEqual(["a.txt", "co"]); + expect(v.map((x) => x.isFile())).toEqual([true, false]); + expect(v.map((x) => x.isDirectory())).toEqual([false, true]); + } + + { + await fs.symlink(join(path, "a.txt"), join(path, "link")); + const v = await fs.readdir(path, { withFileTypes: true }); + expect(v.map(({ name }) => name)).toEqual(["a.txt", "co", "link"]); + expect(v[2].isSymbolicLink()).toEqual(true); + } + }); + + it("readdir with the recursive option", async () => { + const path = "readdir-2"; + await fs.mkdir(path); + expect(await fs.readdir(path, { recursive: true })).toEqual([]); + await fs.mkdir(join(path, "subdir")); + await fs.writeFile(join(path, "subdir", "b.txt"), "x"); + const v = await fs.readdir(path, { recursive: true }); + expect(v).toEqual(["subdir", "subdir/b.txt"]); + + // and withFileTypes + const w = await fs.readdir(path, { recursive: true, withFileTypes: true }); + expect(w.map(({ name }) => name)).toEqual(["subdir", "b.txt"]); + expect(w[0]).toEqual( + expect.objectContaining({ + name: "subdir", + parentPath: path, + path, + }), + ); + expect(w[0].isDirectory()).toBe(true); + expect(w[1]).toEqual( + expect.objectContaining({ + name: "b.txt", + parentPath: join(path, "subdir"), + path: join(path, "subdir"), + }), + ); + expect(w[1].isFile()).toBe(true); + expect(await fs.readFile(join(w[1].path, w[1].name), "utf8")).toEqual("x"); + }); + it("use the find command instead of readdir", async () => { const { stdout } = await fs.find("dirtest", "%f\n"); const v = stdout.toString().trim().split("\n"); diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index aa09b12c25..52749163fc 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -141,7 +141,6 @@ export class SandboxedFilesystem { } // pathInSandbox is *definitely* a path in the sandbox: const pathInSandbox = join(this.path, resolve("/", path)); - if (this.unsafeMode) { // not secure -- just convenient. return pathInSandbox; @@ -238,8 +237,24 @@ export class SandboxedFilesystem { return await readFile(await this.safeAbsPath(path), encoding); }; - readdir = async (path: string): Promise => { - return await readdir(await this.safeAbsPath(path)); + readdir = async (path: string, options?) => { + const x = (await readdir(await this.safeAbsPath(path), options)) as any[]; + if (options?.withFileTypes) { + // each entry in x has a path and parentPath field, which refers to the + // absolute paths to the directory that contains x or the target of x (if + // it is a link). This is an absolute path on the fileserver, which we try + // not to expose from the sandbox, hence we modify them all if possible. + for (const a of x) { + if (a.path.startsWith(this.path)) { + a.path = a.path.slice(this.path.length + 1); + } + if (a.parentPath.startsWith(this.path)) { + a.parentPath = a.parentPath.slice(this.path.length + 1); + } + } + } + + return x; }; readlink = async (path: string): Promise => { diff --git a/src/packages/conat/core/client.ts b/src/packages/conat/core/client.ts index adb360d216..d4349349b3 100644 --- a/src/packages/conat/core/client.ts +++ b/src/packages/conat/core/client.ts @@ -1504,9 +1504,9 @@ export class Client extends EventEmitter { await astream({ ...opts, client: this }), synctable: async (opts: SyncTableOptions): Promise => await createSyncTable({ ...opts, client: this }), - string: (opts: Omit): SyncString => + string: (opts: Omit, "fs">): SyncString => syncstring({ ...opts, client: this }), - db: (opts: Omit): SyncDB => + db: (opts: Omit, "fs">): SyncDB => syncdb({ ...opts, client: this }), }; diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index bd60f4763c..dc197a6ca1 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -1,3 +1,10 @@ +/* +Tests are in + +packages/backend/conat/files/test/local-path.test.ts + +*/ + import { type Client } from "@cocalc/conat/core/client"; import { conat } from "@cocalc/conat/client"; import { @@ -42,7 +49,9 @@ export interface Filesystem { lstat: (path: string) => Promise; mkdir: (path: string, options?) => Promise; readFile: (path: string, encoding?: any) => Promise; - readdir: (path: string) => Promise; + readdir(path: string, options?): Promise; + readdir(path: string, options: { withFileTypes?: false }): Promise; + readdir(path: string, options: { withFileTypes: true }): Promise; readlink: (path: string) => Promise; realpath: (path: string) => Promise; rename: (oldPath: string, newPath: string) => Promise; @@ -75,6 +84,40 @@ export interface Filesystem { listing?: (path: string) => Promise; } +interface IDirent { + name: string; + parentPath: string; + path: string; + type?: number; +} + +const DIRENT_TYPES = { + 0: "UNKNOWN", + 1: "FILE", + 2: "DIR", + 3: "LINK", + 4: "FIFO", + 5: "SOCKET", + 6: "CHAR", + 7: "BLOCK", +}; + +class Dirent { + constructor( + public name: string, + public parentPath: string, + public path: string, + public type: number, + ) {} + isFile = () => DIRENT_TYPES[this.type] == "FILE"; + isDirectory = () => DIRENT_TYPES[this.type] == "DIR"; + isSymbolicLink = () => DIRENT_TYPES[this.type] == "LINK"; + isFIFO = () => DIRENT_TYPES[this.type] == "FIFO"; + isSocket = () => DIRENT_TYPES[this.type] == "SOCKET"; + isCharacterDevice = () => DIRENT_TYPES[this.type] == "CHAR"; + isBlockDevice = () => DIRENT_TYPES[this.type] == "BLOCK"; +} + interface IStats { dev: number; ino: number; @@ -202,8 +245,16 @@ export async function fsServer({ service, fs, client }: Options) { async readFile(path: string, encoding?) { return await (await fs(this.subject)).readFile(path, encoding); }, - async readdir(path: string) { - return await (await fs(this.subject)).readdir(path); + async readdir(path: string, options?) { + const files = await (await fs(this.subject)).readdir(path, options); + if (!options?.withFileTypes) { + return files; + } + // Dirent - change the [Symbol(type)] field to something serializable so client can use this: + return files.map((x) => { + // @ts-ignore + return { ...x, type: x[Object.getOwnPropertySymbols(x)[0]] }; + }); }, async readlink(path: string) { return await (await fs(this.subject)).readlink(path); @@ -292,6 +343,16 @@ export function fsClient({ client ??= conat(); let call = client.call(subject); + const readdir0 = call.readdir.bind(call); + call.readdir = async (path: string, options?) => { + const files = await readdir0(path, options); + if (options?.withFileTypes) { + return files.map((x) => new Dirent(x.name, x.parentPath, x.path, x.type)); + } else { + return files; + } + }; + let constants: any = null; const stat0 = call.stat.bind(call); call.stat = async (path: string) => { diff --git a/src/packages/conat/sync-doc/syncdb.ts b/src/packages/conat/sync-doc/syncdb.ts index b67add9dea..4211ad8477 100644 --- a/src/packages/conat/sync-doc/syncdb.ts +++ b/src/packages/conat/sync-doc/syncdb.ts @@ -2,7 +2,11 @@ import { SyncClient } from "./sync-client"; import { SyncDB, type SyncDBOpts0 } from "@cocalc/sync/editor/db"; import { type Client as ConatClient } from "@cocalc/conat/core/client"; -export interface SyncDBOptions extends Omit { +export type MakeOptional = Omit & + Partial>; + +export interface SyncDBOptions + extends MakeOptional, "fs"> { client: ConatClient; // name of the file service that hosts this file: service?: string; diff --git a/src/packages/conat/sync-doc/syncstring.ts b/src/packages/conat/sync-doc/syncstring.ts index 37cf1185af..322a2621ed 100644 --- a/src/packages/conat/sync-doc/syncstring.ts +++ b/src/packages/conat/sync-doc/syncstring.ts @@ -4,8 +4,10 @@ import { type SyncStringOpts, } from "@cocalc/sync/editor/string/sync"; import { type Client as ConatClient } from "@cocalc/conat/core/client"; +import { type MakeOptional } from "./syncdb"; -export interface SyncStringOptions extends Omit { +export interface SyncStringOptions + extends MakeOptional, "fs"> { client: ConatClient; // name of the file server that hosts this document: service?: string; diff --git a/src/packages/frontend/course/assignments/actions.ts b/src/packages/frontend/course/assignments/actions.ts index a50934da00..2be16c86fe 100644 --- a/src/packages/frontend/course/assignments/actions.ts +++ b/src/packages/frontend/course/assignments/actions.ts @@ -1641,10 +1641,8 @@ ${details} const project_id = store.get("course_project_id"); let files; try { - files = await redux - .getProjectStore(project_id) - .get_listings() - .getListingDirectly(path); + const { fs } = this.course_actions.syncdb; + files = await fs.readdir(path, { withFileTypes: true }); } catch (err) { // This happens, e.g., if the instructor moves the directory // that contains their version of the ipynb file. @@ -1658,7 +1656,7 @@ ${details} if (this.course_actions.is_closed()) return result; const to_read = files - .filter((entry) => !entry.isdir && endswith(entry.name, ".ipynb")) + .filter((entry) => entry.isFile() && endswith(entry.name, ".ipynb")) .map((entry) => entry.name); const f: (file: string) => Promise = async (file) => { diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 68e780ef0a..2295693231 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -119,7 +119,7 @@ export interface SyncOpts0 { data_server?: DataServer; // filesystem interface. - fs?: Filesystem; + fs: Filesystem; // if true, do not implicitly save on commit. This is very // useful for unit testing to easily simulate offline state. @@ -228,7 +228,7 @@ export class SyncDoc extends EventEmitter { private useConat: boolean; legacy: LegacyHistory; - private fs?: Filesystem; + public readonly fs: Filesystem; private noAutosave?: boolean; From f039e0d246dc2e8100d805d98674f4e2aae457bc Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 26 Jul 2025 16:17:51 +0000 Subject: [PATCH 085/798] add test of non-utf8 Buffer readdir --- .../conat/files/test/local-path.test.ts | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/packages/backend/conat/files/test/local-path.test.ts b/src/packages/backend/conat/files/test/local-path.test.ts index 527012f559..fa8d0f5acd 100644 --- a/src/packages/backend/conat/files/test/local-path.test.ts +++ b/src/packages/backend/conat/files/test/local-path.test.ts @@ -1,4 +1,4 @@ -import { link, readFile, symlink } from "node:fs/promises"; +import { link, readFile, stat, symlink, writeFile } from "node:fs/promises"; import { join } from "path"; import { fsClient } from "@cocalc/conat/files/fs"; import { randomId } from "@cocalc/conat/names"; @@ -8,6 +8,7 @@ import { createPathFileserver, cleanupFileservers, } from "@cocalc/backend/conat/files/test/util"; +import { TextDecoder } from "node:util"; beforeAll(before); @@ -176,6 +177,24 @@ describe("use all the standard api functions of fs", () => { expect(await fs.readFile(join(w[1].path, w[1].name), "utf8")).toEqual("x"); }); + it("readdir works with non-utf8 filenames in the path", async () => { + // this test uses internal implementation details (kind of crappy) + const path = "readdir-3"; + await fs.mkdir(path); + const fullPath = join(server.path, project_id, path); + + process.chdir(fullPath); + + const buf = Buffer.from([0xff, 0xfe, 0xfd]); + expect(() => { + const decoder = new TextDecoder("utf-8", { fatal: true }); + decoder.decode(buf); + }).toThrow("not valid"); + await writeFile(buf, "hi"); + const w = await fs.readdir(path, { encoding: "buffer" }); + expect(w[0]).toEqual(buf); + }); + it("use the find command instead of readdir", async () => { const { stdout } = await fs.find("dirtest", "%f\n"); const v = stdout.toString().trim().split("\n"); From e6b1f6bbdd8555f554e526ccdd996a711d6be666 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 26 Jul 2025 16:35:15 +0000 Subject: [PATCH 086/798] change mtime for listings, etc., to be in ms --- src/packages/backend/conat/files/test/listing.test.ts | 3 ++- src/packages/backend/conat/files/test/local-path.test.ts | 7 ++++--- src/packages/conat/files/listing.ts | 5 +++-- src/packages/conat/persist/storage.ts | 2 +- .../frontend/project/explorer/file-listing/file-row.tsx | 2 +- .../frontend/project/page/flyouts/files-controls.tsx | 4 ++-- src/packages/frontend/project/page/flyouts/files.tsx | 4 ++-- 7 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/packages/backend/conat/files/test/listing.test.ts b/src/packages/backend/conat/files/test/listing.test.ts index d19b459d94..1427f0b440 100644 --- a/src/packages/backend/conat/files/test/listing.test.ts +++ b/src/packages/backend/conat/files/test/listing.test.ts @@ -59,7 +59,8 @@ describe("creating a listing monitor starting with an empty directory", () => { it("create another monitor starting with the now nonempty directory", async () => { const dir2 = await listing({ path: "", fs }); - expect(Object.keys(dir.files)).toEqual(["a.txt"]); + expect(Object.keys(dir2.files!)).toEqual(["a.txt"]); + expect(dir.files["a.txt"].mtime).toBeCloseTo(dir2.files!["a.txt"].mtime); dir2.close(); }); diff --git a/src/packages/backend/conat/files/test/local-path.test.ts b/src/packages/backend/conat/files/test/local-path.test.ts index fa8d0f5acd..b7fdfc156d 100644 --- a/src/packages/backend/conat/files/test/local-path.test.ts +++ b/src/packages/backend/conat/files/test/local-path.test.ts @@ -1,4 +1,4 @@ -import { link, readFile, stat, symlink, writeFile } from "node:fs/promises"; +import { link, readFile, symlink, writeFile } from "node:fs/promises"; import { join } from "path"; import { fsClient } from "@cocalc/conat/files/fs"; import { randomId } from "@cocalc/conat/names"; @@ -183,14 +183,15 @@ describe("use all the standard api functions of fs", () => { await fs.mkdir(path); const fullPath = join(server.path, project_id, path); - process.chdir(fullPath); - const buf = Buffer.from([0xff, 0xfe, 0xfd]); expect(() => { const decoder = new TextDecoder("utf-8", { fatal: true }); decoder.decode(buf); }).toThrow("not valid"); + const c = process.cwd(); + process.chdir(fullPath); await writeFile(buf, "hi"); + process.chdir(c); const w = await fs.readdir(path, { encoding: "buffer" }); expect(w[0]).toEqual(buf); }); diff --git a/src/packages/conat/files/listing.ts b/src/packages/conat/files/listing.ts index d45bec5401..912363c0ab 100644 --- a/src/packages/conat/files/listing.ts +++ b/src/packages/conat/files/listing.ts @@ -24,6 +24,7 @@ export const typeDescription = { }; interface FileData { + // last modification time as time since epoch in **milliseconds** (as is usual for javascript) mtime: number; size: number; // isdir = mainly for backward compat: @@ -107,7 +108,7 @@ export class Listing extends EventEmitter { return; } const data: FileData = { - mtime: stats.mtimeMs / 1000, + mtime: stats.mtimeMs, size: stats.size, type: stats.type, }; @@ -162,7 +163,7 @@ async function getListing( try { const v = line.split("\0"); const name = v[0]; - const mtime = parseFloat(v[1]); + const mtime = parseFloat(v[1]) * 1000; const size = parseInt(v[2]); files[name] = { mtime, size, type: v[3] as FileTypeLabel }; if (v[3] == "l") { diff --git a/src/packages/conat/persist/storage.ts b/src/packages/conat/persist/storage.ts index 53971163ad..b12e849daa 100644 --- a/src/packages/conat/persist/storage.ts +++ b/src/packages/conat/persist/storage.ts @@ -367,7 +367,7 @@ export class PersistentStream extends EventEmitter { try { await this.db.backup(path); } catch (err) { - console.log(err); + // console.log(err); logger.debug("WARNING: error creating a backup", path, err); } }); diff --git a/src/packages/frontend/project/explorer/file-listing/file-row.tsx b/src/packages/frontend/project/explorer/file-listing/file-row.tsx index ca95d8258d..869d99b776 100644 --- a/src/packages/frontend/project/explorer/file-listing/file-row.tsx +++ b/src/packages/frontend/project/explorer/file-listing/file-row.tsx @@ -234,7 +234,7 @@ export const FileRow: React.FC = React.memo((props) => { try { return ( ); diff --git a/src/packages/frontend/project/page/flyouts/files-controls.tsx b/src/packages/frontend/project/page/flyouts/files-controls.tsx index 9669e1126b..18c2b5db2a 100644 --- a/src/packages/frontend/project/page/flyouts/files-controls.tsx +++ b/src/packages/frontend/project/page/flyouts/files-controls.tsx @@ -101,7 +101,7 @@ export function FilesSelectedControls({ function renderFileInfoBottom() { if (singleFile != null) { const { size, mtime, isdir } = singleFile; - const age = typeof mtime === "number" ? 1000 * mtime : null; + const age = typeof mtime === "number" ? mtime : null; return ( {age ? ( @@ -144,7 +144,7 @@ export function FilesSelectedControls({ const { size = 0, mtime, isdir } = file; totSize += isdir ? 0 : size; if (typeof mtime === "number") { - const dt = new Date(1000 * mtime); + const dt = new Date(mtime); if (startDT.getTime() === 0 || dt < startDT) startDT = dt; if (endDT.getTime() === 0 || dt > endDT) endDT = dt; } diff --git a/src/packages/frontend/project/page/flyouts/files.tsx b/src/packages/frontend/project/page/flyouts/files.tsx index aca83cbf65..b8ebdcc743 100644 --- a/src/packages/frontend/project/page/flyouts/files.tsx +++ b/src/packages/frontend/project/page/flyouts/files.tsx @@ -527,7 +527,7 @@ export function FilesFlyout({ if (typeof mtime === "number") { return ( @@ -570,7 +570,7 @@ export function FilesFlyout({ function renderListItem(index: number, item: DirectoryListingEntry) { const { mtime, mask = false } = item; - const age = typeof mtime === "number" ? 1000 * mtime : null; + const age = typeof mtime === "number" ? mtime : null; // either select by scrolling (and only scrolling!) or by clicks const isSelected = scrollIdx != null From 256affd9c6cc10d2a3ddb14446053b3fb0f72112 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 26 Jul 2025 18:43:43 +0000 Subject: [PATCH 087/798] filesystem rewrite frontend integration progress --- src/packages/conat/core/client.ts | 12 +- src/packages/conat/files/fs.ts | 30 +- .../frontend/project/directory-selector.tsx | 285 ++++++------------ .../frontend/project/listing/use-fs.ts | 11 +- src/packages/frontend/project_actions.ts | 20 +- src/packages/frontend/projects/actions.ts | 1 - 6 files changed, 154 insertions(+), 205 deletions(-) diff --git a/src/packages/conat/core/client.ts b/src/packages/conat/core/client.ts index d4349349b3..cdf1a9f89e 100644 --- a/src/packages/conat/core/client.ts +++ b/src/packages/conat/core/client.ts @@ -254,7 +254,7 @@ import { type SyncDB, type SyncDBOptions, } from "@cocalc/conat/sync-doc/syncdb"; -import { fsClient, DEFAULT_FILE_SERVICE } from "@cocalc/conat/files/fs"; +import { fsClient, fsSubject } from "@cocalc/conat/files/fs"; import TTL from "@isaacs/ttlcache"; import { ConatSocketServer, @@ -1168,7 +1168,7 @@ export class Client extends EventEmitter { if (s !== undefined) { return s; } - if (typeof name !== "string") { + if (typeof name !== "string" || name == "then") { return undefined; } return async (...args) => await call(name, args); @@ -1478,15 +1478,13 @@ export class Client extends EventEmitter { return sub; }; - fs = ({ - project_id, - service = DEFAULT_FILE_SERVICE, - }: { + fs = (opts: { project_id: string; + compute_server_id?: number; service?: string; }) => { return fsClient({ - subject: `${service}.project-${project_id}`, + subject: fsSubject(opts), client: this, }); }; diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index dc197a6ca1..bd29b7a394 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -13,6 +13,7 @@ import { type WatchIterator, } from "@cocalc/conat/files/watch"; import listing, { type Listing, type FileTypeLabel } from "./listing"; +import { isValidUUID } from "@cocalc/util/misc"; export const DEFAULT_FILE_SERVICE = "fs"; @@ -44,7 +45,7 @@ export interface Filesystem { constants: () => Promise<{ [key: string]: number }>; copyFile: (src: string, dest: string) => Promise; cp: (src: string, dest: string, options?) => Promise; - exists: (path: string) => Promise; + exists: (path: string) => Promise; link: (existingPath: string, newPath: string) => Promise; lstat: (path: string) => Promise; mkdir: (path: string, options?) => Promise; @@ -227,7 +228,7 @@ export async function fsServer({ service, fs, client }: Options) { async cp(src: string, dest: string, options?) { await (await fs(this.subject)).cp(src, dest, options); }, - async exists(path: string) { + async exists(path: string): Promise { return await (await fs(this.subject)).exists(path); }, async find(path: string, printf: string, options?: FindOptions) { @@ -333,6 +334,31 @@ export type FilesystemClient = Omit, "lstat"> & { lstat: (path: string) => Promise; }; +export function fsSubject({ + project_id, + compute_server_id = 0, + service = DEFAULT_FILE_SERVICE, +}: { + project_id: string; + compute_server_id?: number; + service?: string; +}) { + if (!isValidUUID(project_id)) { + throw Error(`project_id must be a valid uuid -- ${project_id}`); + } + if (typeof compute_server_id != "number") { + throw Error("compute_server_id must be a number"); + } + if (typeof service != "string") { + throw Error("service must be a string"); + } + if (compute_server_id) { + return `${service}/${compute_server_id}.project-${project_id}`; + } else { + return `${service}.project-${project_id}`; + } +} + export function fsClient({ client, subject, diff --git a/src/packages/frontend/project/directory-selector.tsx b/src/packages/frontend/project/directory-selector.tsx index 6ec742ccd9..bc10ca9fdb 100644 --- a/src/packages/frontend/project/directory-selector.tsx +++ b/src/packages/frontend/project/directory-selector.tsx @@ -21,13 +21,12 @@ import { } from "react"; import { Icon, Loading } from "@cocalc/frontend/components"; import { path_split } from "@cocalc/util/misc"; -import { exec } from "@cocalc/frontend/frame-editors/generic/client"; import { alert_message } from "@cocalc/frontend/alerts"; -import { delay } from "awaiting"; import { redux, useTypedRedux } from "@cocalc/frontend/app-framework"; -import useIsMountedRef from "@cocalc/frontend/app-framework/is-mounted-hook"; import { useFrameContext } from "@cocalc/frontend/frame-editors/frame-tree/frame-context"; import ShowError from "@cocalc/frontend/components/error"; +import useFs from "@cocalc/frontend/project/listing/use-fs"; +import useFiles from "@cocalc/frontend/project/listing/use-files"; const NEW_FOLDER = "New Folder"; @@ -74,11 +73,6 @@ export default function DirectorySelector({ "compute_server_id", ); const computeServerId = compute_server_id ?? fallbackComputeServerId; - const directoryListings = useTypedRedux( - { project_id }, - "directory_listings", - )?.get(computeServerId); - const isMountedRef = useIsMountedRef(); const [expandedPaths, setExpandedPaths] = useState>(() => { const expandedPaths: string[] = [""]; if (startingPath == null) { @@ -123,77 +117,6 @@ export default function DirectorySelector({ [selectedPaths, multi], ); - useEffect(() => { - // Run the loop below every 30s until project_id or expandedPaths changes (or unmount) - // in which case loop stops. If not unmount, then get new loops for new values. - if (!project_id) return; - const state = { loop: true }; - (async () => { - while (state.loop && isMountedRef.current) { - // Component is mounted, so call watch on all expanded paths. - const listings = redux - .getProjectStore(project_id) - .get_listings(computeServerId); - for (const path of expandedPaths) { - listings.watch(path); - } - await delay(30000); - } - })(); - return () => { - state.loop = false; - }; - }, [project_id, expandedPaths, computeServerId]); - - let body; - if (directoryListings == null) { - (async () => { - await delay(0); - // Ensure store gets initialized before redux - // E.g., for copy between projects you make this - // directory selector before even opening the project. - redux.getProjectStore(project_id); - })(); - body = ; - } else { - body = ( - <> - {}} - /> - - { - setShowHidden(!showHidden); - }} - > - Show hidden - - - ); - } - return ( - {body} + {}} + /> + + { + setShowHidden(!showHidden); + }} + > + Show hidden + ); } @@ -245,14 +198,10 @@ function SelectablePath({ return; // no-op } try { - await exec({ - command: "mv", - project_id, - path: path_split(path).head, - args: [tail, editedTail], - compute_server_id: computeServerId, - filesystem: true, - }); + const actions = redux.getProjectActions(project_id); + const fs = actions.fs(computeServerId); + const { head } = path_split(path); + await fs.rename(join(head, tail), join(head, editedTail)); setEditedTail(null); } catch (err) { alert_message({ type: "error", message: err.toString() }); @@ -415,83 +364,68 @@ function Directory(props) { } } +// Show the directories in path function Subdirs(props) { const { computeServerId, - directoryListings, path, project_id, showHidden, style, toggleSelection, } = props; - const x = directoryListings?.get(path); - const v = x?.toJS?.(); - if (v == null) { - (async () => { - // Must happen in a different render loop, hence the delay, because - // fetch can actually update the store in the same render loop. - await delay(0); - redux.getProjectActions(project_id)?.fetch_directory_listing({ path }); - })(); + const fs = useFs({ project_id, computeServerId }); + const { files, error, refresh } = useFiles({ + fs, + path, + cacheId: { project_id }, + }); + if (error) { + return ; + } + if (files == null) { return ; - } else { - const w: React.JSX.Element[] = []; - const base = !path ? "" : path + "/"; - const paths: string[] = []; - const newPaths: string[] = []; - for (const x of v) { - if (x?.isdir) { - if (x.name.startsWith(".") && !showHidden) continue; - if (x.name.startsWith(NEW_FOLDER)) { - newPaths.push(x.name); - } else { - paths.push(x.name); - } - } - } - paths.sort(); - newPaths.sort(); - const createProps = { - project_id, - path, - computeServerId, - directoryListings, - toggleSelection, - }; - w.push(); - for (const name of paths.concat(newPaths)) { - w.push(); - } - if (w.length > 10) { - w.push(); + } + + const w: React.JSX.Element[] = []; + const base = !path ? "" : path + "/"; + const paths: string[] = []; + const newPaths: string[] = []; + for (const name in files) { + if (!files[name].isdir) continue; + if (name.startsWith(".") && !showHidden) continue; + if (name.startsWith(NEW_FOLDER)) { + newPaths.push(name); + } else { + paths.push(name); } - return ( -
- {w} -
- ); } + paths.sort(); + newPaths.sort(); + const createProps = { + project_id, + path, + computeServerId, + toggleSelection, + }; + w.push(); + for (const name of paths.concat(newPaths)) { + w.push(); + } + if (w.length > 10) { + w.push(); + } + return ( +
+ {w} +
+ ); } -async function getValidPath( - project_id, - target, - directoryListings, - computeServerId, -) { - if ( - await pathExists(project_id, target, directoryListings, computeServerId) - ) { +async function getValidPath(project_id, target, computeServerId) { + if (await pathExists(project_id, target, computeServerId)) { let i: number = 1; - while ( - await pathExists( - project_id, - target + ` (${i})`, - directoryListings, - computeServerId, - ) - ) { + while (await pathExists(project_id, target + ` (${i})`, computeServerId)) { i += 1; } target += ` (${i})`; @@ -503,7 +437,6 @@ function CreateDirectory({ computeServerId, project_id, path, - directoryListings, toggleSelection, }) { const [error, setError] = useState(""); @@ -518,12 +451,7 @@ function CreateDirectory({ const target = path + (path != "" ? "/" : "") + value; (async () => { try { - const path1 = await getValidPath( - project_id, - target, - directoryListings, - computeServerId, - ); + const path1 = await getValidPath(project_id, target, computeServerId); setValue(path_split(path1).tail); setTimeout(() => { input_ref.current?.select(); @@ -536,18 +464,16 @@ function CreateDirectory({ const createFolder = async () => { setOpen(false); + if (!value?.trim()) return; try { - await exec({ - command: "mkdir", - args: ["-p", value], - project_id, - path, - compute_server_id: computeServerId, - filesystem: true, - }); + const actions = redux.getProjectActions(project_id); + const fs = actions.fs(computeServerId); + await fs.mkdir(join(path, value)); toggleSelection(value); } catch (err) { setError(`${err}`); + } finally { + setValue(NEW_FOLDER); } }; @@ -556,7 +482,8 @@ function CreateDirectory({ - New Folder + New + Folder } open={open} @@ -569,18 +496,19 @@ function CreateDirectory({ style={{ marginTop: "30px" }} value={value} onChange={(e) => setValue(e.target.value)} - onPressEnter={createFolder} + onPressEnter={() => createFolder()} autoFocus />
@@ -590,24 +518,9 @@ function CreateDirectory({ export async function pathExists( project_id: string, path: string, - directoryListings?, computeServerId?, ): Promise { - const { head, tail } = path_split(path); - let known = directoryListings?.get(head); - if (known == null) { - const actions = redux.getProjectActions(project_id); - await actions.fetch_directory_listing({ - path: head, - compute_server_id: computeServerId, - }); - } - known = directoryListings?.get(head); - if (known == null) { - return false; - } - for (const x of known) { - if (x.get("name") == tail) return true; - } - return false; + const actions = redux.getProjectActions(project_id); + const fs = actions.fs(computeServerId); + return await fs.exists(path); } diff --git a/src/packages/frontend/project/listing/use-fs.ts b/src/packages/frontend/project/listing/use-fs.ts index ff76d79801..1bf01ab762 100644 --- a/src/packages/frontend/project/listing/use-fs.ts +++ b/src/packages/frontend/project/listing/use-fs.ts @@ -10,11 +10,20 @@ import { useState } from "react"; // the typing for now) export default function useFs({ project_id, + compute_server_id, + computeServerId, }: { project_id: string; + compute_server_id?: number; + computeServerId?: number; }): FilesystemClient | null { const [fs] = useState(() => - webapp_client.conat_client.conat().fs({ project_id }), + webapp_client.conat_client + .conat() + .fs({ + project_id, + compute_server_id: compute_server_id ?? computeServerId, + }), ); return fs; } diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index d76df13f18..4647f056b0 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -399,6 +399,7 @@ export class ProjectActions extends Actions { this.open_files?.close(); delete this.open_files; this.state = "closed"; + this._filesystem = {}; }; private save_session(): void { @@ -1562,12 +1563,12 @@ export class ProjectActions extends Actions { // Update the directory listing cache for the given path. // Uses current path if path not provided. fetch_directory_listing = async (_opts?): Promise => { - console.log("TODO: eliminate code that uses fetch_directory_listing"); + console.trace("TODO: rewrite code that uses fetch_directory_listing"); }; public async fetch_directory_listing_directly(): Promise { - console.log( - "TODO: eliminate code that uses fetch_directory_listing_directly", + console.trace( + "TODO: rewrite code that uses fetch_directory_listing_directly", ); } @@ -2445,12 +2446,15 @@ export class ProjectActions extends Actions { } } - private _filesystem: FilesystemClient; - fs = (): FilesystemClient => { - this._filesystem ??= webapp_client.conat_client + // note: there is no need to explicitly close or await what is returned by + // fs(...) since it's just a lightweight wrapper object to format appropriate RPC calls. + private _filesystem: { [compute_server_id: number]: FilesystemClient } = {}; + fs = (compute_server_id?: number): FilesystemClient => { + compute_server_id ??= this.get_store()?.get("compute_server_id") ?? 0; + this._filesystem[compute_server_id] ??= webapp_client.conat_client .conat() - .fs({ project_id: this.project_id }); - return this._filesystem; + .fs({ project_id: this.project_id, compute_server_id }); + return this._filesystem[compute_server_id]; }; // if available in cache, this returns the filenames in the current directory, diff --git a/src/packages/frontend/projects/actions.ts b/src/packages/frontend/projects/actions.ts index 76aa6bb3c0..3a1fcf4716 100644 --- a/src/packages/frontend/projects/actions.ts +++ b/src/packages/frontend/projects/actions.ts @@ -464,7 +464,6 @@ export class ProjectsActions extends Actions { if (relation == null || ["public", "admin"].includes(relation)) { this.fetch_public_project_title(opts.project_id); } - project_actions.fetch_directory_listing(); if (opts.switch_to) { redux .getActions("page") From c52224bedf1868df4e1cacde3291c2b3ccc1ad52 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 26 Jul 2025 19:41:51 +0000 Subject: [PATCH 088/798] rewrite flyout panel directory listing to use new fs api. - wow, it's incredible how much this simplifies and cleans up the code! --- .../configuration/configuration-copying.tsx | 25 +--- .../frame-editors/code-editor/actions.ts | 11 ++ .../frame-editors/qmd-editor/actions.ts | 47 +------ .../frame-editors/rmd-editor/actions.ts | 45 +------ .../frame-editors/rmd-editor/utils.ts | 29 +++++ .../project/page/flyouts/file-list-item.tsx | 29 +++-- .../frontend/project/page/flyouts/files.tsx | 115 +++++------------- 7 files changed, 101 insertions(+), 200 deletions(-) diff --git a/src/packages/frontend/course/configuration/configuration-copying.tsx b/src/packages/frontend/course/configuration/configuration-copying.tsx index 4510ad0f23..41df5121e6 100644 --- a/src/packages/frontend/course/configuration/configuration-copying.tsx +++ b/src/packages/frontend/course/configuration/configuration-copying.tsx @@ -35,19 +35,15 @@ import { } from "antd"; import { useMemo, useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; - import { labels } from "@cocalc/frontend/i18n"; import { redux, useFrameContext, - useTypedRedux, } from "@cocalc/frontend/app-framework"; import { Icon } from "@cocalc/frontend/components"; import ShowError from "@cocalc/frontend/components/error"; import { COMMANDS } from "@cocalc/frontend/course/commands"; -import { exec } from "@cocalc/frontend/frame-editors/generic/client"; import { IntlMessage } from "@cocalc/frontend/i18n"; -import { pathExists } from "@cocalc/frontend/project/directory-selector"; import { ProjectTitle } from "@cocalc/frontend/projects/project-title"; import { isIntlMessage } from "@cocalc/util/i18n"; import { plural } from "@cocalc/util/misc"; @@ -446,10 +442,6 @@ function AddTarget({ settings, actions, project_id }) { const [path, setPath] = useState(""); const [error, setError] = useState(""); const [create, setCreate] = useState(""); - const directoryListings = useTypedRedux( - { project_id }, - "directory_listings", - )?.get(0); const add = async () => { try { @@ -458,19 +450,14 @@ function AddTarget({ settings, actions, project_id }) { throw Error(`'${path} is the current course'`); } setLoading(true); - const exists = await pathExists(project_id, path, directoryListings); - if (!exists) { + const projectActions = redux.getProjectActions(project_id); + const fs = projectActions.fs(); + if (!(await fs.exists(path))) { if (create) { - await exec({ - command: "touch", - args: [path], - project_id, - filesystem: true, - }); - } else { - setCreate(path); - return; + await fs.writeFile(path, ""); } + } else { + setCreate(path); } const copy_config_targets = getTargets(settings); copy_config_targets[`${project_id}/${path}`] = true; diff --git a/src/packages/frontend/frame-editors/code-editor/actions.ts b/src/packages/frontend/frame-editors/code-editor/actions.ts index 744008bf83..38cac66727 100644 --- a/src/packages/frontend/frame-editors/code-editor/actions.ts +++ b/src/packages/frontend/frame-editors/code-editor/actions.ts @@ -3195,4 +3195,15 @@ export class Actions< }); actions?.foldAllThreads(); } + + getComputeServerId = (): number | undefined => { + return this.redux + .getProjectActions(this.project_id) + .getComputeServerIdForFile(this.path); + }; + + fs = () => { + const a = this.redux.getProjectActions(this.project_id); + return a.fs(a.getComputeServerIdForFile(this.path)); + }; } diff --git a/src/packages/frontend/frame-editors/qmd-editor/actions.ts b/src/packages/frontend/frame-editors/qmd-editor/actions.ts index 50920aa7cc..17cea7239c 100644 --- a/src/packages/frontend/frame-editors/qmd-editor/actions.ts +++ b/src/packages/frontend/frame-editors/qmd-editor/actions.ts @@ -7,12 +7,8 @@ Quarto Editor Actions */ -import { Set } from "immutable"; import { debounce } from "lodash"; - -import { redux } from "@cocalc/frontend/app-framework"; import { markdown_to_html_frontmatter } from "@cocalc/frontend/markdown"; -import { path_split } from "@cocalc/util/misc"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { Actions as BaseActions, @@ -21,7 +17,7 @@ import { import { FrameTree } from "../frame-tree/types"; import { ExecOutput } from "../generic/client"; import { Actions as MarkdownActions } from "../markdown-editor/actions"; -import { derive_rmd_output_filename } from "../rmd-editor/utils"; +import { checkProducedFiles } from "../rmd-editor/utils"; import { convert } from "./qmd-converter"; const custom_pdf_error_message: string = ` @@ -112,44 +108,7 @@ export class Actions extends MarkdownActions { } async _check_produced_files(): Promise { - const project_actions = redux.getProjectActions(this.project_id); - if (project_actions == undefined) { - return; - } - const path = path_split(this.path).head; - await project_actions.fetch_directory_listing({ path }); - - const project_store = project_actions.get_store(); - if (project_store == undefined) { - return; - } - // TODO: change the 0 to the compute server when/if we ever support QMD on a compute server (which we don't) - const dir_listings = project_store.getIn(["directory_listings", 0]); - if (dir_listings == undefined) { - return; - } - const listing = dir_listings.get(path); - if (listing == undefined) { - return; - } - - let existing = Set(); - for (const ext of ["pdf", "html", "nb.html"]) { - // full path – basename might change - const expected_fn = derive_rmd_output_filename(this.path, ext); - const fn_exists = listing.some((entry) => { - const name = entry.get("name"); - return name === path_split(expected_fn).tail; - }); - if (fn_exists) { - existing = existing.add(ext); - } - } - - // console.log("setting derived_file_types to", existing.toJS()); - this.setState({ - derived_file_types: existing as any, - }); + await checkProducedFiles(this); } private set_log(output?: ExecOutput | undefined): void { @@ -248,4 +207,4 @@ export class Actions extends MarkdownActions { this.build(); } } -} +} \ No newline at end of file diff --git a/src/packages/frontend/frame-editors/rmd-editor/actions.ts b/src/packages/frontend/frame-editors/rmd-editor/actions.ts index 2b92b449a0..517cef0558 100644 --- a/src/packages/frontend/frame-editors/rmd-editor/actions.ts +++ b/src/packages/frontend/frame-editors/rmd-editor/actions.ts @@ -7,13 +7,9 @@ R Markdown Editor Actions */ -import { Set } from "immutable"; import { debounce } from "lodash"; - -import { redux } from "@cocalc/frontend/app-framework"; import { markdown_to_html_frontmatter } from "@cocalc/frontend/markdown"; import { open_new_tab } from "@cocalc/frontend/misc"; -import { path_split } from "@cocalc/util/misc"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { Actions as BaseActions, @@ -23,7 +19,7 @@ import { FrameTree } from "../frame-tree/types"; import { ExecOutput } from "../generic/client"; import { Actions as MarkdownActions } from "../markdown-editor/actions"; import { convert } from "./rmd-converter"; -import { derive_rmd_output_filename } from "./utils"; +import { checkProducedFiles } from "./utils"; const HELP_URL = "https://doc.cocalc.com/frame-editor.html#edit-rmd"; const MINIMAL = `--- @@ -139,44 +135,7 @@ export class Actions extends MarkdownActions { } async _check_produced_files(): Promise { - const project_actions = redux.getProjectActions(this.project_id); - if (project_actions == undefined) { - return; - } - const path = path_split(this.path).head; - await project_actions.fetch_directory_listing({ path }); - - const project_store = project_actions.get_store(); - if (project_store == undefined) { - return; - } - // TODO: change the 0 to the compute server when/if we ever support RMD on a compute server (which we don't) - const dir_listings = project_store.getIn(["directory_listings", 0]); - if (dir_listings == undefined) { - return; - } - const listing = dir_listings.get(path); - if (listing == undefined) { - return; - } - - let existing = Set(); - for (const ext of ["pdf", "html", "nb.html"]) { - // full path – basename might change - const expected_fn = derive_rmd_output_filename(this.path, ext); - const fn_exists = listing.some((entry) => { - const name = entry.get("name"); - return name === path_split(expected_fn).tail; - }); - if (fn_exists) { - existing = existing.add(ext); - } - } - - // console.log("setting derived_file_types to", existing.toJS()); - this.setState({ - derived_file_types: existing as any, - }); + await checkProducedFiles(this); } private set_log(output?: ExecOutput | undefined): void { diff --git a/src/packages/frontend/frame-editors/rmd-editor/utils.ts b/src/packages/frontend/frame-editors/rmd-editor/utils.ts index d4c129650a..63dfce78f1 100644 --- a/src/packages/frontend/frame-editors/rmd-editor/utils.ts +++ b/src/packages/frontend/frame-editors/rmd-editor/utils.ts @@ -5,6 +5,7 @@ import { change_filename_extension, path_split } from "@cocalc/util/misc"; import { join } from "path"; +import { Set } from "immutable"; // something in the rmarkdown source code replaces all spaces by dashes // [hsy] I think this is because of calling pandoc. @@ -17,3 +18,31 @@ export function derive_rmd_output_filename(path, ext) { // avoid a leading / if it's just a filename (i.e. head = '') return join(head, fn); } + +export async function checkProducedFiles(codeEditorActions) { + const project_actions = codeEditorActions.redux.getProjectActions( + codeEditorActions.project_id, + ); + if (project_actions == null) { + return; + } + + let existing = Set(); + const fs = codeEditorActions.fs(); + const f = async (ext: string) => { + const expectedFilename = derive_rmd_output_filename( + codeEditorActions.path, + ext, + ); + if (await fs.exists(expectedFilename)) { + existing = existing.add(ext); + } + }; + const v = ["pdf", "html", "nb.html"].map(f); + await Promise.all(v); + + // console.log("setting derived_file_types to", existing.toJS()); + codeEditorActions.setState({ + derived_file_types: existing as any, + }); +} diff --git a/src/packages/frontend/project/page/flyouts/file-list-item.tsx b/src/packages/frontend/project/page/flyouts/file-list-item.tsx index 348ab4dd12..341435e9d2 100644 --- a/src/packages/frontend/project/page/flyouts/file-list-item.tsx +++ b/src/packages/frontend/project/page/flyouts/file-list-item.tsx @@ -6,7 +6,6 @@ import { Button, Dropdown, MenuProps, Tooltip } from "antd"; import immutable from "immutable"; import { useIntl } from "react-intl"; - import { CSS, React, @@ -115,6 +114,7 @@ interface Item { name: string; size?: number; mask?: boolean; + link_target?: string; } interface FileListItemProps { @@ -219,7 +219,6 @@ export const FileListItem = React.memo((props: Readonly) => { function renderName(): React.JSX.Element { const name = item.name; - const path = isActive ? path_split(name).tail : name; const { name: basename, ext } = item.isdir ? { name: path, ext: "" } @@ -230,8 +229,8 @@ export const FileListItem = React.memo((props: Readonly) => { ? item.isopen ? { fontWeight: "bold" } : item.isdir - ? undefined - : { color: COLORS.FILE_EXT } + ? undefined + : { color: COLORS.FILE_EXT } : undefined; return ( @@ -252,6 +251,12 @@ export const FileListItem = React.memo((props: Readonly) => { ) ) : undefined} + {!!item.link_target && ( + <> + + {item.link_target} + + )} ); } @@ -279,8 +284,8 @@ export const FileListItem = React.memo((props: Readonly) => { ? "check-square" : "square" : item.isdir - ? "folder-open" - : file_options(item.name)?.icon ?? "file"); + ? "folder-open" + : (file_options(item.name)?.icon ?? "file")); return ( ) => { const actionNames = multiple ? ACTION_BUTTONS_MULTI : isdir - ? ACTION_BUTTONS_DIR - : ACTION_BUTTONS_FILE; + ? ACTION_BUTTONS_DIR + : ACTION_BUTTONS_FILE; for (const key of actionNames) { if (key === "download" && !item.isdir) continue; const disabled = @@ -527,10 +532,10 @@ export const FileListItem = React.memo((props: Readonly) => { ? FILE_ITEM_ACTIVE_STYLE_2 : {} : item.isopen - ? item.isactive - ? FILE_ITEM_ACTIVE_STYLE - : FILE_ITEM_OPENED_STYLE - : {}; + ? item.isactive + ? FILE_ITEM_ACTIVE_STYLE + : FILE_ITEM_OPENED_STYLE + : {}; return ( diff --git a/src/packages/frontend/project/page/flyouts/files.tsx b/src/packages/frontend/project/page/flyouts/files.tsx index b8ebdcc743..87555832a5 100644 --- a/src/packages/frontend/project/page/flyouts/files.tsx +++ b/src/packages/frontend/project/page/flyouts/files.tsx @@ -4,17 +4,14 @@ */ import { Alert, InputRef } from "antd"; -import { delay } from "awaiting"; -import { List, Map } from "immutable"; +import { List } from "immutable"; import { debounce, fromPairs } from "lodash"; import { Virtuoso, VirtuosoHandle } from "react-virtuoso"; - import { React, TypedMap, redux, useEffect, - useIsMountedRef, useLayoutEffect, useMemo, usePrevious, @@ -35,7 +32,6 @@ import { DirectoryListingEntry, FileMap, } from "@cocalc/frontend/project/explorer/types"; -import { WATCH_THROTTLE_MS } from "@cocalc/frontend/conat/listings"; import { mutate_data_to_compute_public_files } from "@cocalc/frontend/project_store"; import track from "@cocalc/frontend/user-tracking"; import { @@ -44,8 +40,6 @@ import { human_readable_size, path_split, path_to_file, - search_match, - search_split, separate_file_extension, tab_to_path, unreachable, @@ -59,6 +53,9 @@ import { FileListItem } from "./file-list-item"; import { FilesBottom } from "./files-bottom"; import { FilesHeader } from "./files-header"; import { fileItemStyle } from "./utils"; +import useFs from "@cocalc/frontend/project/listing/use-fs"; +import useListing from "@cocalc/frontend/project/listing/use-listing"; +import ShowError from "@cocalc/frontend/components/error"; type PartialClickEvent = Pick< React.MouseEvent | React.KeyboardEvent, @@ -100,7 +97,6 @@ export function FilesFlyout({ project_id, actions, } = useProjectContext(); - const isMountedRef = useIsMountedRef(); const rootRef = useRef(null as any); const refInput = useRef(null as any); const [rootHeightPx, setRootHeightPx] = useState(0); @@ -110,12 +106,6 @@ export function FilesFlyout({ const current_path = useTypedRedux({ project_id }, "current_path"); const strippedPublicPaths = useStrippedPublicPaths(project_id); const compute_server_id = useTypedRedux({ project_id }, "compute_server_id"); - const directoryListings: Map< - string, - TypedMap | null - > | null = useTypedRedux({ project_id }, "directory_listings")?.get( - compute_server_id, - ); const activeTab = useTypedRedux({ project_id }, "active_project_tab"); const activeFileSort: ActiveFileSort = useTypedRedux( { project_id }, @@ -143,25 +133,6 @@ export function FilesFlyout({ return tab_to_path(activeTab); }, [activeTab]); - // copied roughly from directory-selector.tsx - useEffect(() => { - // Run the loop below every 30s until project_id or current_path changes (or unmount) - // in which case loop stops. If not unmount, then get new loops for new values. - if (!project_id) return; - const state = { loop: true }; - (async () => { - while (state.loop && isMountedRef.current) { - // Component is mounted, so call watch on all expanded paths. - const listings = redux.getProjectStore(project_id).get_listings(); - listings.watch(current_path); - await delay(WATCH_THROTTLE_MS); - } - })(); - return () => { - state.loop = false; - }; - }, [project_id, current_path]); - // selecting files switches over to "select" mode or back to "open" useEffect(() => { if (mode === "open" && checked_files.size > 0) { @@ -172,6 +143,16 @@ export function FilesFlyout({ } }, [checked_files]); + const fs = useFs({ project_id, compute_server_id }); + const { + listing: directoryListing, + error: listingError, + refresh, + } = useListing({ + fs, + path: current_path, + }); + // active file: current editor is the file in the listing // empty: either no files, or just the ".." for the parent dir const [directoryFiles, fileMap, activeFile, isEmpty] = useMemo((): [ @@ -180,28 +161,19 @@ export function FilesFlyout({ DirectoryListingEntry | null, boolean, ] => { - if (directoryListings == null) return EMPTY_LISTING; - const filesStore = directoryListings.get(current_path); - if (filesStore == null) return EMPTY_LISTING; - - // TODO this is an error, process it - if (typeof filesStore === "string") return EMPTY_LISTING; - - const files: DirectoryListing | null = filesStore.toJS?.(); + const files = directoryListing; if (files == null) return EMPTY_LISTING; let activeFile: DirectoryListingEntry | null = null; compute_file_masks(files); - const searchWords = search_split(file_search.trim().toLowerCase()); + const searchWords = file_search.trim().toLowerCase(); - const procFiles = files + const processedFiles : DirectoryListingEntry[] = files .filter((file: DirectoryListingEntry) => { - file.name ??= ""; // sanitization - if (file_search === "") return true; - const fName = file.name.toLowerCase(); + const filename = file.name.toLowerCase(); return ( - search_match(fName, searchWords) || - ((file.isdir ?? false) && search_match(`${fName}/`, searchWords)) + filename.includes(searchWords) || + (file.isdir && `${filename}/`.includes(searchWords)) ); }) .filter( @@ -211,17 +183,17 @@ export function FilesFlyout({ (file: DirectoryListingEntry) => hidden || !file.name.startsWith("."), ); - // this shares the logic with what's in project_store.js + // this shares the logic with what's in project_store.ts mutate_data_to_compute_public_files( { - listing: procFiles, + listing: processedFiles, public: {}, }, strippedPublicPaths, current_path, ); - procFiles.sort((a, b) => { + processedFiles.sort((a, b) => { // This replicated what project_store is doing const col = activeFileSort.get("column_name"); switch (col) { @@ -245,7 +217,7 @@ export function FilesFlyout({ } }); - for (const file of procFiles) { + for (const file of processedFiles) { const fullPath = path_to_file(current_path, file.name); if (openFiles.some((path) => path == fullPath)) { file.isopen = true; @@ -257,26 +229,26 @@ export function FilesFlyout({ } if (activeFileSort.get("is_descending")) { - procFiles.reverse(); // inplace op + processedFiles.reverse(); // inplace op } - const isEmpty = procFiles.length === 0; + const isEmpty = processedFiles.length === 0; // the ".." dir does not change the isEmpty state // hide ".." if there is a search -- https://github.com/sagemathinc/cocalc/issues/6877 if (file_search === "" && current_path != "") { - procFiles.unshift({ + processedFiles.unshift({ name: "..", isdir: true, }); } // map each filename to it's entry in the directory listing - const fileMap = fromPairs(procFiles.map((file) => [file.name, file])); + const fileMap = fromPairs(processedFiles.map((file) => [file.name, file])); - return [procFiles, fileMap, activeFile, isEmpty]; + return [processedFiles, fileMap, activeFile, isEmpty]; }, [ - directoryListings, + directoryListing, activeFileSort, hidden, file_search, @@ -309,7 +281,7 @@ export function FilesFlyout({ useEffect(() => { setShowCheckboxIndex(null); - }, [directoryListings, current_path]); + }, [directoryListing, current_path]); const triggerRootResize = debounce( () => setRootHeightPx(rootRef.current?.clientHeight ?? 0), @@ -353,27 +325,6 @@ export function FilesFlyout({ return fileMap[basename]; } - if (directoryListings == null) { - (async () => { - await delay(0); - // Ensure store gets initialized before redux - // E.g., for copy between projects you make this - // directory selector before even opening the project. - redux.getProjectStore(project_id); - })(); - } - - if (directoryListings?.get(current_path) == null) { - (async () => { - // Must happen in a different render loop, hence the delay, because - // fetch can actually update the store in the same render loop. - await delay(0); - redux - .getProjectActions(project_id) - ?.fetch_directory_listing({ path: current_path }); - })(); - } - function open( e: PartialClickEvent, index: number, @@ -639,8 +590,7 @@ export function FilesFlyout({ } function renderListing(): React.JSX.Element { - const files = directoryListings?.get(current_path); - if (files == null) { + if (directoryListing == null) { return renderLoadingOrStartProject(); } @@ -683,6 +633,7 @@ export function FilesFlyout({ ref={rootRef} style={{ flex: "1 0 auto", flexDirection: "column", display: "flex" }} > + Date: Sat, 26 Jul 2025 20:25:04 +0000 Subject: [PATCH 089/798] update guide and fix some major bugs/issues - the biggest bug is cwd wasn't being updated which made it kind of useless (I think I caused this bug recently). I fixed this. - i updated the file stats thing to be much better (using our human_readable_size function) and simpler cleaner code - We should remove this or write something modern. See https://github.com/sagemathinc/cocalc/issues/8462 In particular: - the buttons look weird -- they are totally different than anywhere else in cocalc - clicking a button in there instantly does the thing, no matter how dangerous. This makes me a little scared to use this. - The files section isn't searchable and can be huge -- do I want to sort through 50 pages to find something - But of course the real thing that makes this so dates is that LLM's exist, and if there is anything they are good it, it's basic bash terminal stuff. And they have a VASTLY larger range of options for what they can do, with vastly better reasoning, than what is here. Even the cheapest old models... --- .../frame-editors/code-editor/actions.ts | 4 +- .../terminal-editor/commands-guide.tsx | 148 +++++++----------- .../terminal-editor/conat-terminal.ts | 2 +- .../terminal-editor/connected-terminal.ts | 23 ++- 4 files changed, 74 insertions(+), 103 deletions(-) diff --git a/src/packages/frontend/frame-editors/code-editor/actions.ts b/src/packages/frontend/frame-editors/code-editor/actions.ts index 38cac66727..8a7eb2c93e 100644 --- a/src/packages/frontend/frame-editors/code-editor/actions.ts +++ b/src/packages/frontend/frame-editors/code-editor/actions.ts @@ -1500,7 +1500,7 @@ export class Actions< return this.terminals.get_terminal(id, parent); } - public set_terminal_cwd(id: string, cwd: string): void { + set_terminal_cwd(id: string, cwd: string): void { this.save_editor_state(id, { cwd }); } @@ -3204,6 +3204,6 @@ export class Actions< fs = () => { const a = this.redux.getProjectActions(this.project_id); - return a.fs(a.getComputeServerIdForFile(this.path)); + return a.fs(a.getComputeServerIdForFile({ path: this.path })); }; } diff --git a/src/packages/frontend/frame-editors/terminal-editor/commands-guide.tsx b/src/packages/frontend/frame-editors/terminal-editor/commands-guide.tsx index 3f99d6a579..ba18b638f5 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/commands-guide.tsx +++ b/src/packages/frontend/frame-editors/terminal-editor/commands-guide.tsx @@ -11,8 +11,7 @@ import { Table, Typography, } from "antd"; -import { List, Map } from "immutable"; - +import { Map } from "immutable"; import { ControlOutlined, FileOutlined, @@ -20,82 +19,45 @@ import { InfoCircleOutlined, QuestionCircleOutlined, } from "@ant-design/icons"; -import { - CSS, - React, - TypedMap, - useActions, - useEffect, - useState, - useTypedRedux, -} from "@cocalc/frontend/app-framework"; +import { createContext, useEffect, useState } from "react"; import { Icon } from "@cocalc/frontend/components"; -import { plural, round1 } from "@cocalc/util/misc"; -import { DirectoryListingEntry } from "../../project/explorer/types"; +import { human_readable_size, plural } from "@cocalc/util/misc"; import { TerminalActions } from "./actions"; import { Command, SelectFile } from "./commands-guide-components"; +import useFiles from "@cocalc/frontend/project/listing/use-files"; +import ShowError from "@cocalc/frontend/components/error"; const { Panel } = Collapse; interface Props { - font_size: number; - project_id: string; actions: TerminalActions; local_view_state: Map; } -export const TerminalActionsContext = React.createContext< +export const TerminalActionsContext = createContext< TerminalActions | undefined >(undefined); const ListingStatsInit = { - total: 0, num_files: 0, num_dirs: 0, - size_mib: 0, + size: 0, }; const info = "info"; -type ListingImm = List>; - -function listing2names(listing?): string[] { - if (listing == null) { - return []; - } else { - return listing - .map((val) => val.get("name")) - .sort() - .toJS(); - } -} - function cwd2path(cwd: string): string { return cwd.charAt(0) === "/" ? ".smc/root" + cwd : cwd; } -export const CommandsGuide: React.FC = React.memo((props: Props) => { - const { /*font_size,*/ actions, local_view_state, project_id } = props; - - const project_actions = useActions({ project_id }); - // TODO: for now just assuming in the project (not a compute server) -- problem - // is that the guide is general to the whole terminal not a particular frame, - // and each frame can be on a different compute server! Not worth solving if - // nobody is using either the guide or compute servers. - const directory_listings = useTypedRedux( - { project_id }, - "directory_listings", - )?.get(0); - - const [terminal_id, set_terminal_id] = useState(); +export function CommandsGuide({ actions, local_view_state }: Props) { + const [terminal_id, setTerminalId] = useState(); const [cwd, set_cwd] = useState(""); // default home directory const [hidden, set_hidden] = useState(false); // hidden files // empty immutable js list - const [listing, set_listing] = useState(List([])); - const [listing_stats, set_listing_stats] = useState(ListingStatsInit); - const [directorynames, set_directorynames] = useState([]); + const [directoryNames, set_directoryNames] = useState([]); const [filenames, set_filenames] = useState([]); // directory and filenames const [dir1, set_dir1] = useState(undefined); @@ -103,13 +65,17 @@ export const CommandsGuide: React.FC = React.memo((props: Props) => { const [fn2, set_fn2] = useState(undefined); useEffect(() => { - const tid = actions._get_most_recent_active_frame_id_of_type("terminal"); - if (tid == null) return; - if (terminal_id != tid) set_terminal_id(tid); + const terminalId = + actions._get_most_recent_active_frame_id_of_type("terminal"); + if (terminalId == null) { + return; + } + if (terminal_id != terminalId) { + setTerminalId(terminalId); + } }, [local_view_state]); useEffect(() => { - //const terminal = actions.get_terminal(tid); const next_cwd = local_view_state.getIn([ "editor_state", terminal_id, @@ -117,50 +83,45 @@ export const CommandsGuide: React.FC = React.memo((props: Props) => { ]) as string | undefined; if (next_cwd != null && cwd != next_cwd) { set_cwd(next_cwd); - project_actions?.fetch_directory_listing({ path: cwd2path(next_cwd) }); } }, [terminal_id, local_view_state]); - // if the working directory changes or the listing itself, recompute the listing we base the files on - useEffect(() => { - if (cwd == null) return; - set_listing(directory_listings?.get(cwd2path(cwd))); - }, [directory_listings, cwd]); + const { files, error, refresh } = useFiles({ + fs: actions.fs(), + path: cwd2path(cwd), + }); // finally, if the listing really did change – or show/hide hidden files toggled – recalculate everything useEffect(() => { - // a user reported a crash "Uncaught TypeError: listing.filter is not a function". - // This was because directory_listings is a map from path to either an immutable - // listing **or** an error, as you can see where it is set in the file frontend/project_actions.ts - // The typescript there just has an "any", because it's code that was partly converted from coffeescript. - // Fixing this by just doing listing?.filter==null instead of listing==null here, since dealing with - // an error isn't necessary for this command guide. - if ( - listing == null || - typeof listing == "string" || - listing?.filter == null - ) + if (files == null) { return; - const all_files = hidden - ? listing - : listing.filter((val) => !val.get("name").startsWith(".")); - const grouped = all_files.groupBy((val) => !!val.get("isdir")); - const dirnames = [".", "..", ...listing2names(grouped.get(true))]; - const filenames = listing2names(grouped.get(false)); - set_directorynames(dirnames); + } + const dirnames: string[] = [".", ".."]; + const filenames: string[] = []; + for (const name in files) { + if (!hidden && name.startsWith(".")) { + continue; + } + if (files[name].isdir) { + dirnames.push(name); + } else { + filenames.push(name); + } + } + dirnames.sort(); + filenames.sort(); + + set_directoryNames(dirnames); set_filenames(filenames); - const total = all_files.size; - const size_red = grouped - .get(false) - ?.reduce((cur, val) => cur + val.get("size", 0), 0); - const size = (size_red ?? 0) / (1024 * 1024); + const size = filenames + .map((name) => files[name].size ?? 0) + .reduce((a, b) => a + b, 0); set_listing_stats({ - total, num_files: filenames.length, - num_dirs: dirnames.length, - size_mib: size, + num_dirs: dirnames.length - 2, + size, }); - }, [listing, hidden]); + }, [files, hidden]); // we also clear selected files if they no longer exist useEffect(() => { @@ -170,16 +131,16 @@ export const CommandsGuide: React.FC = React.memo((props: Props) => { if (fn2 != null && !filenames.includes(fn2)) { set_fn2(undefined); } - if (dir1 != null && !directorynames.includes(dir1)) { + if (dir1 != null && !directoryNames.includes(dir1)) { set_dir1(undefined); } - }, [directorynames, filenames]); + }, [directoryNames, filenames]); function render_files() { - const dirs = directorynames.map((v) => ({ key: v, name: v, type: "dir" })); + const dirs = directoryNames.map((v) => ({ key: v, name: v, type: "dir" })); const fns = filenames.map((v) => ({ key: v, name: v, type: "file" })); const data = [...dirs, ...fns]; - const style: CSS = { cursor: "pointer" }; + const style = { cursor: "pointer" } as const; const columns = [ { title: "Name", @@ -306,7 +267,7 @@ export const CommandsGuide: React.FC = React.memo((props: Props) => { {plural(listing_stats.num_files, "file")},{" "} {listing_stats.num_dirs}{" "} {plural(listing_stats.num_dirs, "directory", "directories")},{" "} - {round1(listing_stats.size_mib)} MiB + {human_readable_size(listing_stats.size)} @@ -316,7 +277,7 @@ export const CommandsGuide: React.FC = React.memo((props: Props) => { - + @@ -430,7 +391,7 @@ export const CommandsGuide: React.FC = React.memo((props: Props) => { } function render() { - const style: CSS = { overflowY: "auto" }; + const style = { overflowY: "auto" } as const; return ( } key={info}> @@ -501,7 +462,8 @@ export const CommandsGuide: React.FC = React.memo((props: Props) => { return ( + {render()} ); -}); +} diff --git a/src/packages/frontend/frame-editors/terminal-editor/conat-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/conat-terminal.ts index 5bb2c6a5bc..58e0e1d51e 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/conat-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/conat-terminal.ts @@ -24,7 +24,7 @@ export class ConatTerminal extends EventEmitter { private terminalResize; private openPaths; private closePaths; - private api: TerminalServiceApi; + public readonly api: TerminalServiceApi; private service?; private options?; private writeQueue: string = ""; diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index 95e9bf16fc..cfb7437e96 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -313,9 +313,6 @@ export class Terminal { this.conn = conn as any; conn.on("close", this.connect); conn.on("kick", this.close_request); - conn.on("cwd", (cwd) => { - this.actions.set_terminal_cwd(this.id, cwd); - }); conn.on("data", this.handleDataFromProject); conn.on("init", this.render); conn.once("ready", () => { @@ -438,7 +435,7 @@ export class Terminal { this.terminal.onTitleChange((title) => { if (title != null) { this.actions.set_title(this.id, title); - this.ask_for_cwd(); + this.update_cwd(); } }); }; @@ -720,9 +717,21 @@ export class Terminal { this.render_buffer = ""; }; - ask_for_cwd = debounce((): void => { - this.conn_write({ cmd: "cwd" }); - }); + update_cwd = debounce( + async () => { + let cwd; + try { + cwd = await this.conn?.api.cwd(); + } catch { + return; + } + if (cwd != null) { + this.actions.set_terminal_cwd(this.id, cwd); + } + }, + 1000, + { leading: true, trailing: true }, + ); kick_other_users_out(): void { // @ts-ignore From 4b36fa9bfd1b60d9e0ad2ffe8b95f3daf7c02ca3 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 26 Jul 2025 21:22:19 +0000 Subject: [PATCH 090/798] switch terminal to be ephemeral by default --- .../frame-editors/terminal-editor/connected-terminal.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index cfb7437e96..34e73fb720 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -50,6 +50,11 @@ const MAX_DELAY = 15000; const ENABLE_WEBGL = false; + +// ephemeral = faster, less load on servers, but if project and browser all +// close, the history is gone... which may be good and less confusing. +const EPHEMERAL = true; + interface Path { file?: string; directory?: string; @@ -309,6 +314,7 @@ export class Terminal { cwd: this.workingDir, env: this.actions.get_term_env(), }, + ephemeral: EPHEMERAL, }); this.conn = conn as any; conn.on("close", this.connect); From 39ed974cb540c451b01f1909b2291dac31bfba8c Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 26 Jul 2025 21:29:39 +0000 Subject: [PATCH 091/798] no need to alert about opening the target of a symlink --- src/packages/frontend/project/open-file.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/packages/frontend/project/open-file.ts b/src/packages/frontend/project/open-file.ts index d5d20be4b2..cb10653459 100644 --- a/src/packages/frontend/project/open-file.ts +++ b/src/packages/frontend/project/open-file.ts @@ -169,11 +169,6 @@ export async function open_file( } if (opts.path != realpath) { if (!actions.open_files) return; // closed - alert_message({ - type: "info", - message: `Opening normalized real path "${realpath}"`, - timeout: 10, - }); actions.open_files.delete(opts.path); opts.path = realpath; actions.open_files.set(opts.path, "component", {}); From d36bb978a0f1ab24240da337c75c9dc075836de5 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 26 Jul 2025 22:41:53 +0000 Subject: [PATCH 092/798] fix a case where I guess trying to edit a readonly sagews would crash (and generally seems good to be more careful here) --- .../frontend/project/page/file-tabs.tsx | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/packages/frontend/project/page/file-tabs.tsx b/src/packages/frontend/project/page/file-tabs.tsx index 0e1eaab893..c593217ef2 100644 --- a/src/packages/frontend/project/page/file-tabs.tsx +++ b/src/packages/frontend/project/page/file-tabs.tsx @@ -100,9 +100,11 @@ export default function FileTabs({ openFiles, project_id, activeTab }) { if (action == "add") { actions.set_active_tab("files"); } else { - const path = keyToPath(key); - // close given file - actions.close_tab(path); + if (key) { + const path = keyToPath(key); + // close given file + actions.close_tab(path); + } } }; @@ -135,11 +137,14 @@ export default function FileTabs({ openFiles, project_id, activeTab }) { function onDragStart(event) { if (actions == null) return; if (event?.active?.id != activeKey) { - actions.set_active_tab(path_to_tab(keyToPath(event?.active?.id)), { - // noFocus -- critical to not focus when dragging or codemirror focus breaks on end of drag. - // See https://github.com/sagemathinc/cocalc/issues/7029 - noFocus: true, - }); + const key = event?.active?.id; + if (key) { + actions.set_active_tab(path_to_tab(keyToPath(key)), { + // noFocus -- critical to not focus when dragging or codemirror focus breaks on end of drag. + // See https://github.com/sagemathinc/cocalc/issues/7029 + noFocus: true, + }); + } } } @@ -160,7 +165,7 @@ export default function FileTabs({ openFiles, project_id, activeTab }) { activeKey={activeKey} type={"editable-card"} onChange={(key) => { - if (actions == null) return; + if (actions == null || !key) return; actions.set_active_tab(path_to_tab(keyToPath(key))); }} popupClassName={"cocalc-files-tabs-more"} From 3ec160a75822f7e11fd770115975d346e0180220 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 01:19:24 +0000 Subject: [PATCH 093/798] jupyter: first steps on getting notebooks to start running just using an rpc call --- .../backend/conat/files/local-path.ts | 27 +++++++-- src/packages/conat/files/fs.ts | 26 +++++--- src/packages/conat/project/api/editor.ts | 14 +++++ src/packages/frontend/components/time-ago.tsx | 2 +- .../frontend/jupyter/browser-actions.ts | 47 +++++++++++++-- .../frontend/project/explorer/action-box.tsx | 1 - .../project/explorer/create-archive.tsx | 1 - .../frontend/project/explorer/download.tsx | 3 - .../frontend/project/explorer/rename-file.tsx | 1 - .../frontend/project/new/new-file-page.tsx | 11 +--- .../project/page/flyouts/files-header.tsx | 3 - .../frontend/project/page/flyouts/files.tsx | 5 +- src/packages/frontend/project_actions.ts | 16 ----- src/packages/jupyter/control.ts | 60 +++++++++++++++++++ src/packages/jupyter/kernel/kernel.ts | 13 ++-- src/packages/jupyter/redux/actions.ts | 25 ++------ src/packages/jupyter/redux/project-actions.ts | 2 +- src/packages/jupyter/zmq/index.ts | 10 ++-- src/packages/project/conat/api/editor.ts | 18 ++++++ src/packages/project/conat/files/fs.ts | 30 ++++++++++ src/packages/project/sagews/control.ts | 10 ++++ src/packages/util/redux/Actions.ts | 10 +--- 22 files changed, 237 insertions(+), 98 deletions(-) create mode 100644 src/packages/jupyter/control.ts create mode 100644 src/packages/project/conat/files/fs.ts create mode 100644 src/packages/project/sagews/control.ts diff --git a/src/packages/backend/conat/files/local-path.ts b/src/packages/backend/conat/files/local-path.ts index bc905561a6..3fa1c4a83a 100644 --- a/src/packages/backend/conat/files/local-path.ts +++ b/src/packages/backend/conat/files/local-path.ts @@ -10,22 +10,37 @@ export async function localPathFileserver({ path, service = DEFAULT_FILE_SERVICE, client, + project_id, + unsafeMode, }: { path: string; service?: string; client?: Client; + // if project_id is specified, use single project mode. + project_id?: string; + unsafeMode?: boolean; }) { client ??= conat(); + + const singleProjectFilesystem = project_id + ? new SandboxedFilesystem(path, { unsafeMode }) + : undefined; + const server = await fsServer({ service, client, + project_id, fs: async (subject: string) => { - const project_id = getProjectId(subject); - const p = join(path, project_id); - try { - await mkdir(p); - } catch {} - return new SandboxedFilesystem(p); + if (project_id) { + return singleProjectFilesystem!; + } else { + const project_id = getProjectId(subject); + const p = join(path, project_id); + try { + await mkdir(p); + } catch {} + return new SandboxedFilesystem(p, { unsafeMode }); + } }, }); return { server, client, path, service, close: () => server.close() }; diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index bd29b7a394..c7ce7280f8 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -206,11 +206,17 @@ interface Options { service: string; client?: Client; fs: (subject?: string) => Promise; + // project-id: if given, ONLY serve files for this one project, and the + // path must be the home of the project + // If not given, + project_id?: string; } -export async function fsServer({ service, fs, client }: Options) { +export async function fsServer({ service, fs, client, project_id }: Options) { client ??= conat(); - const subject = `${service}.*`; + const subject = project_id + ? `${service}.project-${project_id}` + : `${service}.*`; const watches: { [subject: string]: any } = {}; const sub = await client.service(subject, { async appendFile(path: string, data: string | Buffer, encoding?) { @@ -334,6 +340,16 @@ export type FilesystemClient = Omit, "lstat"> & { lstat: (path: string) => Promise; }; +export function getService({ + compute_server_id, + service = DEFAULT_FILE_SERVICE, +}: { + compute_server_id?: number; + service?: string; +}) { + return compute_server_id ? `${service}/${compute_server_id}` : service; +} + export function fsSubject({ project_id, compute_server_id = 0, @@ -352,11 +368,7 @@ export function fsSubject({ if (typeof service != "string") { throw Error("service must be a string"); } - if (compute_server_id) { - return `${service}/${compute_server_id}.project-${project_id}`; - } else { - return `${service}.project-${project_id}`; - } + return `${getService({ service, compute_server_id })}.project-${project_id}`; } export function fsClient({ diff --git a/src/packages/conat/project/api/editor.ts b/src/packages/conat/project/api/editor.ts index 9db2b45471..e17d597e8a 100644 --- a/src/packages/conat/project/api/editor.ts +++ b/src/packages/conat/project/api/editor.ts @@ -5,13 +5,21 @@ import type { KernelSpec } from "@cocalc/util/jupyter/types"; export const editor = { newFile: true, + + jupyterStart: true, + jupyterStop: true, jupyterStripNotebook: true, jupyterNbconvert: true, jupyterRunNotebook: true, jupyterKernelLogo: true, jupyterKernels: true, + formatString: true, + printSageWS: true, + sagewsStart: true, + sagewsStop: true, + createTerminalService: true, }; @@ -35,6 +43,10 @@ export interface Editor { jupyterStripNotebook: (path_ipynb: string) => Promise; + // path = the syncdb path (not *.ipynb) + jupyterStart: (path: string) => Promise; + jupyterStop: (path: string) => Promise; + jupyterNbconvert: (opts: NbconvertParams) => Promise; jupyterRunNotebook: (opts: RunNotebookOptions) => Promise; @@ -54,6 +66,8 @@ export interface Editor { }) => Promise; printSageWS: (opts) => Promise; + sagewsStart: (path_sagews: string) => Promise; + sagewsStop: (path_sagews: string) => Promise; createTerminalService: ( termPath: string, diff --git a/src/packages/frontend/components/time-ago.tsx b/src/packages/frontend/components/time-ago.tsx index 7259a5a75a..784747c0d3 100644 --- a/src/packages/frontend/components/time-ago.tsx +++ b/src/packages/frontend/components/time-ago.tsx @@ -203,7 +203,7 @@ export const TimeAgo: React.FC = React.memo( }: TimeAgoElementProps) => { const { timeAgoAbsolute } = useAppContext(); - if (date == null) { + if (!date || date.valueOf()) { return <>; } diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 97f2e1f341..2779fccf87 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -56,6 +56,7 @@ import { syncdbPath } from "@cocalc/util/jupyter/names"; import getKernelSpec from "@cocalc/frontend/jupyter/kernelspecs"; import { get as getUsageInfo } from "@cocalc/conat/project/usage-info"; import { delay } from "awaiting"; +import { until } from "@cocalc/util/async-utils"; // local cache: map project_id (string) -> kernels (immutable) let jupyter_kernels = Map(); @@ -70,6 +71,7 @@ export class JupyterActions extends JupyterActions0 { protected init2(): void { this.syncdbPath = syncdbPath(this.path); + this.initBackend(); this.update_contents = debounce(this.update_contents.bind(this), 2000); this.setState({ toolbar: !this.get_local_storage("hide_toolbar"), @@ -172,6 +174,44 @@ export class JupyterActions extends JupyterActions0 { } } + // if the project or compute server is running and listening, this call + // tells them to open this jupyter notebook, so it can provide the compute + // functionality. + + private conatApi = async () => { + const compute_server_id = await this.getComputeServerId(); + const api = webapp_client.project_client.conatApi( + this.project_id, + compute_server_id, + ); + return api; + }; + + initBackend = async () => { + await until( + async () => { + if (this.is_closed()) { + return true; + } + try { + const api = await this.conatApi(); + await api.editor.jupyterStart(this.syncdbPath); + console.log("initialized ", this.path); + return true; + } catch (err) { + console.log("failed to initialize ", this.path, err); + return false; + } + }, + { min: 3000 }, + ); + }; + + stopBackend = async () => { + const api = await this.conatApi(); + await api.editor.jupyterStop(this.syncdbPath); + }; + initOpenLog = () => { // Put an entry in the project log once the jupyter notebook gets opened and // shows cells. @@ -354,10 +394,9 @@ export class JupyterActions extends JupyterActions0 { }; protected close_client_only(): void { - const account = this.redux.getStore("account"); - if (account != null) { - account.removeListener("change", this.account_change); - } + const account = this.redux + ?.getStore("account") + ?.removeListener("change", this.account_change); } private syncdb_cursor_activity = (): void => { diff --git a/src/packages/frontend/project/explorer/action-box.tsx b/src/packages/frontend/project/explorer/action-box.tsx index 949a87c17b..d52c5e9e18 100644 --- a/src/packages/frontend/project/explorer/action-box.tsx +++ b/src/packages/frontend/project/explorer/action-box.tsx @@ -122,7 +122,6 @@ export function ActionBox(props: ReactProps) { props.actions.delete_files({ paths }); props.actions.set_file_action(); props.actions.set_all_files_unchecked(); - props.actions.fetch_directory_listing(); } function render_delete_warning(): React.JSX.Element | undefined { diff --git a/src/packages/frontend/project/explorer/create-archive.tsx b/src/packages/frontend/project/explorer/create-archive.tsx index a18c2d9fa0..b599de0761 100644 --- a/src/packages/frontend/project/explorer/create-archive.tsx +++ b/src/packages/frontend/project/explorer/create-archive.tsx @@ -46,7 +46,6 @@ export default function CreateArchive({}) { dest: target + ".zip", path, }); - await actions.fetch_directory_listing({ path }); } catch (err) { setLoading(false); setError(err); diff --git a/src/packages/frontend/project/explorer/download.tsx b/src/packages/frontend/project/explorer/download.tsx index 24654ec1d1..ddaa75c793 100644 --- a/src/packages/frontend/project/explorer/download.tsx +++ b/src/packages/frontend/project/explorer/download.tsx @@ -89,9 +89,6 @@ export default function Download({}) { dest = files[0]; } actions.download_file({ path: dest, log: files }); - await actions.fetch_directory_listing({ - path: store.get("current_path"), - }); } catch (err) { console.log(err); setLoading(false); diff --git a/src/packages/frontend/project/explorer/rename-file.tsx b/src/packages/frontend/project/explorer/rename-file.tsx index 2b819ba9d5..327427126f 100644 --- a/src/packages/frontend/project/explorer/rename-file.tsx +++ b/src/packages/frontend/project/explorer/rename-file.tsx @@ -79,7 +79,6 @@ export default function RenameFile({ duplicate }: Props) { } else { await actions.rename_file(opts); } - await actions.fetch_directory_listing({ path: renameDir }); } catch (err) { setLoading(false); setError(err); diff --git a/src/packages/frontend/project/new/new-file-page.tsx b/src/packages/frontend/project/new/new-file-page.tsx index 015c97b069..d8f9ace744 100644 --- a/src/packages/frontend/project/new/new-file-page.tsx +++ b/src/packages/frontend/project/new/new-file-page.tsx @@ -250,11 +250,6 @@ export default function NewFilePage(props: Props) { { - getActions().fetch_directory_listing(); - }, - }} project_id={project_id} current_path={current_path} show_header={false} @@ -355,11 +350,7 @@ export default function NewFilePage(props: Props) { } values={{ upload: ( - getActions().fetch_directory_listing()} - /> + ), folder: (txt) => ( ): React.JSX.Element { actions?.fetch_directory_listing(), - }} config={{ clickable: `.${uploadClassName}` }} className="smc-vfill" > diff --git a/src/packages/frontend/project/page/flyouts/files.tsx b/src/packages/frontend/project/page/flyouts/files.tsx index 87555832a5..4ed6d1f2a7 100644 --- a/src/packages/frontend/project/page/flyouts/files.tsx +++ b/src/packages/frontend/project/page/flyouts/files.tsx @@ -167,7 +167,7 @@ export function FilesFlyout({ compute_file_masks(files); const searchWords = file_search.trim().toLowerCase(); - const processedFiles : DirectoryListingEntry[] = files + const processedFiles: DirectoryListingEntry[] = files .filter((file: DirectoryListingEntry) => { if (file_search === "") return true; const filename = file.name.toLowerCase(); @@ -661,9 +661,6 @@ export function FilesFlyout({ actions?.fetch_directory_listing(), - }} style={{ flex: "1 0 auto", display: "flex", diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index 4647f056b0..f038821e6a 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -1519,7 +1519,6 @@ export class ProjectActions extends Actions { compute_server_id, checked_files: store.get("checked_files").clear(), // always clear on compute_server_id change }); - this.fetch_directory_listing({ compute_server_id }); set_local_storage( store.computeServerIdLocalStorageKey, `${compute_server_id}`, @@ -1560,18 +1559,6 @@ export class ProjectActions extends Actions { }); } - // Update the directory listing cache for the given path. - // Uses current path if path not provided. - fetch_directory_listing = async (_opts?): Promise => { - console.trace("TODO: rewrite code that uses fetch_directory_listing"); - }; - - public async fetch_directory_listing_directly(): Promise { - console.trace( - "TODO: rewrite code that uses fetch_directory_listing_directly", - ); - } - // Sets the active file_sort to next_column_name set_sorted_file_column(column_name): void { let is_descending; @@ -2819,8 +2806,6 @@ export class ProjectActions extends Actions { explicit: true, compute_server_id, }); - } else { - this.fetch_directory_listing(); } }; @@ -2846,7 +2831,6 @@ export class ProjectActions extends Actions { alert: true, }); } finally { - this.fetch_directory_listing(); this.set_activity({ id, stop: "" }); this.setState({ downloading_file: false }); this.set_active_tab("files", { update_file_listing: false }); diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts new file mode 100644 index 0000000000..49f5d8e1d2 --- /dev/null +++ b/src/packages/jupyter/control.ts @@ -0,0 +1,60 @@ +import { SyncDB } from "@cocalc/sync/editor/db/sync"; +import { SYNCDB_OPTIONS } from "@cocalc/jupyter/redux/sync"; +import { type Filesystem } from "@cocalc/conat/files/fs"; +import { getLogger } from "@cocalc/backend/logger"; +import { initJupyterRedux, removeJupyterRedux } from "@cocalc/jupyter/kernel"; +import { original_path } from "@cocalc/util/misc"; + +const logger = getLogger("jupyter:control"); + +const sessions: { [path: string]: SyncDB } = {}; +let project_id: string = ""; + +export function jupyterStart({ + path, + client, + project_id: project_id0, + fs, +}: { + path: string; + client; + project_id: string; + fs: Filesystem; +}) { + project_id = project_id0; + if (sessions[path] != null) { + logger.debug("jupyterStart: ", path, " - already running"); + return; + } + logger.debug("jupyterStart: ", path, " - starting it"); + const syncdb = new SyncDB({ + ...SYNCDB_OPTIONS, + project_id, + path, + client, + fs, + }); + sessions[path] = syncdb; + // [ ] TODO: some way to convey this to clients (?) + syncdb.on("error", (err) => { + logger.debug(`syncdb error -- ${err}`, path); + jupyterStop({ path }); + }); + syncdb.on("close", () => { + jupyterStop({ path }); + }); + initJupyterRedux(syncdb, client); +} + +export function jupyterStop({ path }: { path: string }) { + const syncdb = sessions[path]; + if (syncdb == null) { + logger.debug("jupyterStop: ", path, " - not running"); + } else { + logger.debug("jupyterStop: ", path, " - stopping it"); + syncdb.close(); + delete sessions[path]; + const path_ipynb = original_path(path); + removeJupyterRedux(path_ipynb, project_id); + } +} diff --git a/src/packages/jupyter/kernel/kernel.ts b/src/packages/jupyter/kernel/kernel.ts index 9f12677ccd..d22e206c50 100644 --- a/src/packages/jupyter/kernel/kernel.ts +++ b/src/packages/jupyter/kernel/kernel.ts @@ -121,7 +121,7 @@ const SAGE_JUPYTER_ENV = merge(copy(process.env), { // the ipynb file, and this function creates the corresponding // actions and store, which make it possible to work with this // notebook. -export async function initJupyterRedux(syncdb: SyncDB, client: Client) { +export function initJupyterRedux(syncdb: SyncDB, client: Client) { const project_id = syncdb.project_id; if (project_id == null) { throw Error("project_id must be defined"); @@ -147,7 +147,7 @@ export async function initJupyterRedux(syncdb: SyncDB, client: Client) { // Having two at once basically results in things feeling hung. // This should never happen, but we ensure it // See https://github.com/sagemathinc/cocalc/issues/4331 - await removeJupyterRedux(path, project_id); + removeJupyterRedux(path, project_id); } const store = redux.createStore(name, JupyterStore); const actions = redux.createActions(name, JupyterActions); @@ -171,14 +171,11 @@ export async function getJupyterRedux(syncdb: SyncDB) { // Remove the store/actions for a given Jupyter notebook, // and also close the kernel if it is running. -export async function removeJupyterRedux( - path: string, - project_id: string, -): Promise { +export function removeJupyterRedux(path: string, project_id: string): void { logger.debug("removeJupyterRedux", path); // if there is a kernel, close it try { - await kernels.get(path)?.close(); + kernels.get(path)?.close(); } catch (_err) { // ignore } @@ -186,7 +183,7 @@ export async function removeJupyterRedux( const actions = redux.getActions(name); if (actions != null) { try { - await actions.close(); + actions.close(); } catch (err) { logger.debug( "removeJupyterRedux", diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index be7a437acf..2c8975435a 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -213,30 +213,15 @@ export abstract class JupyterActions extends Actions { } public is_closed(): boolean { - return this._state === "closed" || this._state === undefined; + return (this._state ?? "closed") === "closed"; } - public async close({ noSave }: { noSave?: boolean } = {}): Promise { + public close() { if (this.is_closed()) { return; } - // ensure save to disk happens: - // - it will automatically happen for the sync-doc file, but - // we also need it for the ipynb file... as ipynb is unique - // in having two formats. - if (!noSave) { - await this.save(); - } - if (this.is_closed()) { - return; - } - - if (this.syncdb != null) { - this.syncdb.close(); - } - if (this._file_watcher != null) { - this._file_watcher.close(); - } + this.syncdb?.close(); + this._file_watcher?.close(); if (this.is_project || this.is_compute_server) { this.close_project_only(); } else { @@ -246,7 +231,7 @@ export abstract class JupyterActions extends Actions { // since otherwise this.redux and this.name are gone, // which makes destroying the actions properly impossible. this.destroy(); - this.store.destroy(); + this.store?.destroy(); close(this); this._state = "closed"; } diff --git a/src/packages/jupyter/redux/project-actions.ts b/src/packages/jupyter/redux/project-actions.ts index ee1300deec..87eca214ed 100644 --- a/src/packages/jupyter/redux/project-actions.ts +++ b/src/packages/jupyter/redux/project-actions.ts @@ -1303,7 +1303,7 @@ export class JupyterActions extends JupyterActions0 { // we should terminate and clean up everything. if (this.isDeleted()) { dbg("ipynb file is deleted, so NOT saving to disk and closing"); - this.close({ noSave: true }); + this.close(); return; } diff --git a/src/packages/jupyter/zmq/index.ts b/src/packages/jupyter/zmq/index.ts index 3a18b162ba..632a3e5ff0 100644 --- a/src/packages/jupyter/zmq/index.ts +++ b/src/packages/jupyter/zmq/index.ts @@ -1,10 +1,10 @@ import { EventEmitter } from "events"; import { Dealer, Subscriber } from "zeromq"; import { Message } from "./message"; -import { getLogger } from "@cocalc/backend/logger"; import type { JupyterMessage } from "./types"; -const logger = getLogger("jupyter:zmq"); +//import { getLogger } from "@cocalc/backend/logger"; +//const logger = getLogger("jupyter:zmq"); type JupyterSocketName = "iopub" | "shell" | "stdin" | "control"; @@ -76,7 +76,7 @@ export class JupyterSockets extends EventEmitter { throw Error(`invalid socket name '${name}'`); } - logger.debug("send message", message); + //logger.debug("send message", message); const jMessage = new Message(message); socket.send( jMessage._encode( @@ -119,9 +119,9 @@ export class JupyterSockets extends EventEmitter { private listen = async (name: JupyterSocketName, socket) => { if (ZMQ_TYPE[name] == "sub") { - // subscribe to everything -- + // subscribe to everything -- // https://zeromq.github.io/zeromq.js/classes/Subscriber.html#subscribe - socket.subscribe(); + socket.subscribe(); } for await (const data of socket) { const mesg = Message._decode( diff --git a/src/packages/project/conat/api/editor.ts b/src/packages/project/conat/api/editor.ts index c8fde5365a..22cad3d183 100644 --- a/src/packages/project/conat/api/editor.ts +++ b/src/packages/project/conat/api/editor.ts @@ -7,6 +7,8 @@ export { get_kernel_data as jupyterKernels } from "@cocalc/jupyter/kernel/kernel export { newFile } from "@cocalc/backend/misc/new-file"; import { printSageWS as printSageWS0 } from "@cocalc/project/print_to_pdf"; +export { sagewsStart, sagewsStop } from "@cocalc/project/sagews/control"; + import { filename_extension } from "@cocalc/util/misc"; export async function printSageWS(opts): Promise { let pdf; @@ -32,3 +34,19 @@ export async function printSageWS(opts): Promise { } export { createTerminalService } from "@cocalc/project/conat/terminal"; + +import { getClient } from "@cocalc/project/client"; +import { project_id } from "@cocalc/project/data"; +import * as control from "@cocalc/jupyter/control"; +import { SandboxedFilesystem } from "@cocalc/backend/files/sandbox"; + +export async function jupyterStart(path: string) { + const fs = new SandboxedFilesystem(process.env.HOME ?? "/tmp", { + unsafeMode: true, + }); + await control.jupyterStart({ project_id, path, client: getClient(), fs }); +} + +export async function jupyterStop(path: string) { + await control.jupyterStop({ path }); +} diff --git a/src/packages/project/conat/files/fs.ts b/src/packages/project/conat/files/fs.ts new file mode 100644 index 0000000000..718d011ace --- /dev/null +++ b/src/packages/project/conat/files/fs.ts @@ -0,0 +1,30 @@ +/* +Fileserver with all safety off for the project. This is run inside the project by the project, +so the security is off. +*/ + +import { localPathFileserver } from "@cocalc/backend/conat/files/local-path"; +import { getService } from "@cocalc/conat/files/fs"; +import { compute_server_id, project_id } from "@cocalc/project/data"; +import { connectToConat } from "@cocalc/project/conat/connection"; + +let server: any = undefined; +export async function init() { + if (server) { + return; + } + const client = connectToConat(); + const service = getService({ compute_server_id }); + server = await localPathFileserver({ + client, + service, + path: process.env.HOME ?? "/tmp", + unsafeMode: true, + project_id, + }); +} + +export function close() { + server?.close(); + server = undefined; +} diff --git a/src/packages/project/sagews/control.ts b/src/packages/project/sagews/control.ts new file mode 100644 index 0000000000..fa31dd09f8 --- /dev/null +++ b/src/packages/project/sagews/control.ts @@ -0,0 +1,10 @@ +import { getLogger } from "@cocalc/backend/logger"; +const logger = getLogger("project:sagews:control"); + +export async function sagewsStart(path_ipynb: string) { + logger.debug("sagewsStart: ", path_ipynb); +} + +export async function sagewsStop(path_ipynb: string) { + logger.debug("sagewsStop: ", path_ipynb); +} diff --git a/src/packages/util/redux/Actions.ts b/src/packages/util/redux/Actions.ts index 27d9ffc098..032303919e 100644 --- a/src/packages/util/redux/Actions.ts +++ b/src/packages/util/redux/Actions.ts @@ -39,13 +39,9 @@ export class Actions { }; destroy = (): void => { - if (this.name == null) { - throw Error("unable to destroy actions because this.name is not defined"); - } - if (this.redux == null) { - throw Error( - `unable to destroy actions '${this.name}' since this.redux is not defined`, - ); + if (this.name == null || this.redux == null) { + // already closed + return; } // On the share server this.redux can be undefined at this point. this.redux.removeActions(this.name); From 986f664ec067ff2c91b3620c0ff0e7210c3a1c84 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 02:17:24 +0000 Subject: [PATCH 094/798] wire up proof of concept to show we can do code evaluation in the ping time --- src/packages/conat/project/api/editor.ts | 2 + .../frontend/jupyter/browser-actions.ts | 13 ++++- src/packages/jupyter/control.ts | 52 +++++++++++++++++-- src/packages/jupyter/kernel/kernel.ts | 2 + src/packages/jupyter/redux/sync.ts | 4 +- src/packages/project/conat/api/editor.ts | 5 ++ 6 files changed, 70 insertions(+), 8 deletions(-) diff --git a/src/packages/conat/project/api/editor.ts b/src/packages/conat/project/api/editor.ts index e17d597e8a..2771f927b9 100644 --- a/src/packages/conat/project/api/editor.ts +++ b/src/packages/conat/project/api/editor.ts @@ -13,6 +13,7 @@ export const editor = { jupyterRunNotebook: true, jupyterKernelLogo: true, jupyterKernels: true, + jupyterRun: true, formatString: true, @@ -46,6 +47,7 @@ export interface Editor { // path = the syncdb path (not *.ipynb) jupyterStart: (path: string) => Promise; jupyterStop: (path: string) => Promise; + jupyterRun: (path: string, ids: string[]) => Promise; jupyterNbconvert: (opts: NbconvertParams) => Promise; diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 2779fccf87..89be281203 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -212,6 +212,17 @@ export class JupyterActions extends JupyterActions0 { await api.editor.jupyterStop(this.syncdbPath); }; + // temporary proof of concept + runCell = async (id: string) => { + const api = await this.conatApi(); + const resp = await api.editor.jupyterRun(this.syncdbPath, [id]); + const output = {}; + for (let i = 0; i < resp.length; i++) { + output[i] = resp[i]; + } + this.syncdb.set({ id, type: "cell", output }); + }; + initOpenLog = () => { // Put an entry in the project log once the jupyter notebook gets opened and // shows cells. @@ -394,7 +405,7 @@ export class JupyterActions extends JupyterActions0 { }; protected close_client_only(): void { - const account = this.redux + this.redux ?.getStore("account") ?.removeListener("change", this.account_change); } diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts index 49f5d8e1d2..d610d3807e 100644 --- a/src/packages/jupyter/control.ts +++ b/src/packages/jupyter/control.ts @@ -4,10 +4,11 @@ import { type Filesystem } from "@cocalc/conat/files/fs"; import { getLogger } from "@cocalc/backend/logger"; import { initJupyterRedux, removeJupyterRedux } from "@cocalc/jupyter/kernel"; import { original_path } from "@cocalc/util/misc"; +import { once } from "@cocalc/util/async-utils"; const logger = getLogger("jupyter:control"); -const sessions: { [path: string]: SyncDB } = {}; +const sessions: { [path: string]: { syncdb: SyncDB; actions; store } } = {}; let project_id: string = ""; export function jupyterStart({ @@ -34,7 +35,6 @@ export function jupyterStart({ client, fs, }); - sessions[path] = syncdb; // [ ] TODO: some way to convey this to clients (?) syncdb.on("error", (err) => { logger.debug(`syncdb error -- ${err}`, path); @@ -43,14 +43,56 @@ export function jupyterStart({ syncdb.on("close", () => { jupyterStop({ path }); }); - initJupyterRedux(syncdb, client); + const { actions, store } = initJupyterRedux(syncdb, client); + sessions[path] = { syncdb, actions, store }; +} + +// run the cells with given id... +export async function jupyterRun({ + path, + ids, +}: { + path: string; + ids: string[]; +}) { + logger.debug("jupyterRun", { path, ids }); + const session = sessions[path]; + if (session == null) { + throw Error(`${path} not running`); + } + const { syncdb, actions, store } = session; + if (syncdb.isClosed()) { + // shouldn't be possible + throw Error("syncdb is closed"); + } + if (!syncdb.isReady()) { + logger.debug("jupyterRun: waiting until ready"); + await once(syncdb, "ready"); + } + // for (let i = 0; i < ids.length; i++) { + // actions.run_cell(ids[i], false); + // } + logger.debug("jupyterRun: running"); + if (ids.length == 1) { + const code = store.get("cells").get(ids[0])?.get("input")?.trim(); + if (code) { + const result: any[] = []; + for (const x of await actions.jupyter_kernel.execute_code_now({ code })) { + if (x.msg_type == "execute_result") { + result.push(x.content); + } + } + return result; + } + } } export function jupyterStop({ path }: { path: string }) { - const syncdb = sessions[path]; - if (syncdb == null) { + const session = sessions[path]; + if (session == null) { logger.debug("jupyterStop: ", path, " - not running"); } else { + const { syncdb } = session; logger.debug("jupyterStop: ", path, " - stopping it"); syncdb.close(); delete sessions[path]; diff --git a/src/packages/jupyter/kernel/kernel.ts b/src/packages/jupyter/kernel/kernel.ts index d22e206c50..a3e67c6e9a 100644 --- a/src/packages/jupyter/kernel/kernel.ts +++ b/src/packages/jupyter/kernel/kernel.ts @@ -160,6 +160,8 @@ export function initJupyterRedux(syncdb: SyncDB, client: Client) { syncdb.once("ready", () => logger.debug("initJupyterRedux", path, "syncdb ready"), ); + + return { actions, store }; } export async function getJupyterRedux(syncdb: SyncDB) { diff --git a/src/packages/jupyter/redux/sync.ts b/src/packages/jupyter/redux/sync.ts index d36ef1a971..b94935c196 100644 --- a/src/packages/jupyter/redux/sync.ts +++ b/src/packages/jupyter/redux/sync.ts @@ -1,6 +1,6 @@ export const SYNCDB_OPTIONS = { - change_throttle: 50, // our UI/React can handle more rapid updates; plus we want output FAST. - patch_interval: 50, + change_throttle: 25, + patch_interval: 25, primary_keys: ["type", "id"], string_cols: ["input"], cursors: true, diff --git a/src/packages/project/conat/api/editor.ts b/src/packages/project/conat/api/editor.ts index 22cad3d183..f1ec6b82fa 100644 --- a/src/packages/project/conat/api/editor.ts +++ b/src/packages/project/conat/api/editor.ts @@ -47,6 +47,11 @@ export async function jupyterStart(path: string) { await control.jupyterStart({ project_id, path, client: getClient(), fs }); } +export async function jupyterRun(path: string, ids: string[]) { + await jupyterStart(path); + return await control.jupyterRun({ path, ids }); +} + export async function jupyterStop(path: string) { await control.jupyterStop({ path }); } From 80e70f4f5da75b71665f373f3acef3d23a989149 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 05:20:01 +0000 Subject: [PATCH 095/798] sometimes err is a string so we can't do err.message in that case --- src/packages/frontend/conat/client.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/packages/frontend/conat/client.ts b/src/packages/frontend/conat/client.ts index 674aa05ac5..516c138143 100644 --- a/src/packages/frontend/conat/client.ts +++ b/src/packages/frontend/conat/client.ts @@ -314,7 +314,13 @@ export class ConatClient extends EventEmitter { const resp = await cn.request(subject, data, { timeout }); return resp.data; } catch (err) { - err.message = `${err.message} - callHub: subject='${subject}', name='${name}', `; + try { + err.message = `${err.message} - callHub: subject='${subject}', name='${name}', `; + } catch { + err = new Error( + `${err.message} - callHub: subject='${subject}', name='${name}', `, + ); + } throw err; } }; From bef4fff56baa7cfcbf20477f9c8e3e1426d76aca Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 05:38:35 +0000 Subject: [PATCH 096/798] add better typing to the jupyter output handler --- .../jupyter/execute/output-handler.ts | 86 +++++++++++-------- src/packages/jupyter/ipynb/export-to-ipynb.ts | 10 ++- 2 files changed, 58 insertions(+), 38 deletions(-) diff --git a/src/packages/jupyter/execute/output-handler.ts b/src/packages/jupyter/execute/output-handler.ts index 80312d1e6b..61536a3f77 100644 --- a/src/packages/jupyter/execute/output-handler.ts +++ b/src/packages/jupyter/execute/output-handler.ts @@ -28,49 +28,65 @@ OutputHandler emits these events: import { callback } from "awaiting"; import { EventEmitter } from "events"; -import { - close, - defaults, - required, - server_time, - len, - to_json, - is_object, -} from "@cocalc/util/misc"; +import { close, server_time, len, to_json, is_object } from "@cocalc/util/misc"; +import { type TypedMap } from "@cocalc/util/types/typed-map"; const now = () => server_time().valueOf() - 0; const MIN_SAVE_INTERVAL_MS = 500; const MAX_SAVE_INTERVAL_MS = 45000; +import { type Cell } from "@cocalc/jupyter/ipynb/export-to-ipynb"; + +interface Message { + execution_state?; + execution_count?: number; + exec_count?: number | null; + code?: string; + status?; + source?; + name?: string; + opts?; + more_output?: boolean; + text?: string; + data?: { [mimeType: string]: any }; +} + +interface Options { + // object; the cell whose output (etc.) will get mutated + cell: Cell; + // If given, used to truncate, discard output messages; extra + // messages are saved and made available. + max_output_length?: number; + max_output_messages?: number; + // If no messages for this many ms, then we update via set to indicate + // that cell is being run. + report_started_ms?: number; + dbg?; +} + +type State = "ready" | "closed"; + export class OutputHandler extends EventEmitter { - private _opts: any; + private _opts: Options; private _n: number; private _clear_before_next_output: boolean; private _output_length: number; - private _in_more_output_mode: any; - private _state: any; - private _stdin_cb: any; + private _in_more_output_mode: boolean; + private _state: State; + private _stdin_cb?: Function; - // Never commit output to send to the frontend more frequently than this.saveIntervalMs + // Never commit output to send to the frontend more frequently + // than this.saveIntervalMs // Otherwise, we'll end up with a large number of patches. // We start out with MIN_SAVE_INTERVAL_MS and exponentially back it off to // MAX_SAVE_INTERVAL_MS. private lastSave: number = 0; private saveIntervalMs = MIN_SAVE_INTERVAL_MS; - constructor(opts: any) { + constructor(opts: Options) { super(); - this._opts = defaults(opts, { - cell: required, // object; the cell whose output (etc.) will get mutated - // If given, used to truncate, discard output messages; extra - // messages are saved and made available. - max_output_length: undefined, - max_output_messages: undefined, - report_started_ms: undefined, // If no messages for this many ms, then we update via set to indicate - // that cell is being run. - dbg: undefined, - }); + this._opts = opts; const { cell } = this._opts; cell.output = null; cell.exec_count = null; @@ -177,7 +193,7 @@ export class OutputHandler extends EventEmitter { this._clear_output(); }; - _clean_mesg = (mesg: any): void => { + _clean_mesg = (mesg: Message): void => { delete mesg.execution_state; delete mesg.code; delete mesg.status; @@ -190,7 +206,7 @@ export class OutputHandler extends EventEmitter { } }; - private _push_mesg = (mesg: any, save?: boolean): void => { + private _push_mesg = (mesg: Message, save?: boolean): void => { if (this._state === "closed") { return; } @@ -209,7 +225,7 @@ export class OutputHandler extends EventEmitter { this.lastSave = now(); } - if (this._opts.cell.output === null) { + if (this._opts.cell.output == null) { this._opts.cell.output = {}; } this._opts.cell.output[`${this._n}`] = mesg; @@ -217,7 +233,7 @@ export class OutputHandler extends EventEmitter { this.emit("change", save); }; - set_input = (input: any, save = true): void => { + set_input = (input: string, save = true): void => { if (this._state === "closed") { return; } @@ -226,8 +242,8 @@ export class OutputHandler extends EventEmitter { }; // Process incoming messages. This may mutate mesg. - message = (mesg: any): void => { - let has_exec_count: any; + message = (mesg: Message): void => { + let has_exec_count: boolean; if (this._state === "closed") { return; } @@ -299,7 +315,7 @@ export class OutputHandler extends EventEmitter { }; async stdin(prompt: string, password: boolean): Promise { - // See docs for stdin option to execute_code in backend jupyter.coffee + // See docs for stdin option to execute_code in backend. this._push_mesg({ name: "input", opts: { prompt, password } }); // Now we wait until the output message we just included has its // value set. Then we call cb with that value. @@ -310,14 +326,14 @@ export class OutputHandler extends EventEmitter { } // Call this when the cell changes; only used for stdin right now. - cell_changed = (cell: any, get_password: any): void => { + cell_changed = (cell: TypedMap, get_password: () => string): void => { if (this._state === "closed") { return; } if (this._stdin_cb == null) { return; } - const output = cell != null ? cell.get("output") : undefined; + const output = cell?.get("output"); if (output == null) { return; } @@ -346,7 +362,7 @@ export class OutputHandler extends EventEmitter { } }; - payload = (payload: any): void => { + payload = (payload: { source?; text: string }): void => { if (this._state === "closed") { return; } diff --git a/src/packages/jupyter/ipynb/export-to-ipynb.ts b/src/packages/jupyter/ipynb/export-to-ipynb.ts index 8c7ec57886..ebd57e53d1 100644 --- a/src/packages/jupyter/ipynb/export-to-ipynb.ts +++ b/src/packages/jupyter/ipynb/export-to-ipynb.ts @@ -13,7 +13,7 @@ type CellType = "code" | "markdown" | "raw"; type Tags = { [key: string]: boolean }; -interface Cell { +export interface Cell { cell_type?: CellType; input?: string; collapsed?: boolean; @@ -21,9 +21,13 @@ interface Cell { slide?; attachments?; tags?: Tags; - output?: { [n: string]: OutputMessage }; + output?: { [n: string]: OutputMessage } | null; metadata?: Metadata; - exec_count?: number; + exec_count?: number | null; + + start?: number | null; + end?: number | null; + state?: "done" | "busy" | "run"; } type OutputMessage = any; From 1ccb0997b5d3b4b06ecee917dbba40e4e9e53a73 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 06:22:28 +0000 Subject: [PATCH 097/798] jupyter: faster execution proof of concept --- src/packages/conat/project/api/editor.ts | 5 +- .../frontend/jupyter/browser-actions.ts | 16 ++--- src/packages/frontend/jupyter/run-cell.ts | 30 +++++++++ src/packages/jupyter/control.ts | 28 +++------ .../jupyter/execute/output-handler.ts | 62 ++++++++++++++++--- src/packages/jupyter/package.json | 1 + src/packages/jupyter/redux/project-actions.ts | 49 +-------------- src/packages/pnpm-lock.yaml | 3 + src/packages/project/conat/api/editor.ts | 7 ++- 9 files changed, 116 insertions(+), 85 deletions(-) create mode 100644 src/packages/frontend/jupyter/run-cell.ts diff --git a/src/packages/conat/project/api/editor.ts b/src/packages/conat/project/api/editor.ts index 2771f927b9..c18d7d00f8 100644 --- a/src/packages/conat/project/api/editor.ts +++ b/src/packages/conat/project/api/editor.ts @@ -47,7 +47,10 @@ export interface Editor { // path = the syncdb path (not *.ipynb) jupyterStart: (path: string) => Promise; jupyterStop: (path: string) => Promise; - jupyterRun: (path: string, ids: string[]) => Promise; + jupyterRun: ( + path: string, + cells: { id: string; input: string }[], + ) => Promise; jupyterNbconvert: (opts: NbconvertParams) => Promise; diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 89be281203..932bee24a3 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -57,6 +57,7 @@ import getKernelSpec from "@cocalc/frontend/jupyter/kernelspecs"; import { get as getUsageInfo } from "@cocalc/conat/project/usage-info"; import { delay } from "awaiting"; import { until } from "@cocalc/util/async-utils"; +import { runCell } from "./run-cell"; // local cache: map project_id (string) -> kernels (immutable) let jupyter_kernels = Map(); @@ -67,7 +68,7 @@ export class JupyterActions extends JupyterActions0 { private cursor_manager: CursorManager; private account_change_editor_settings: any; private update_keyboard_shortcuts: any; - private syncdbPath: string; + public syncdbPath: string; protected init2(): void { this.syncdbPath = syncdbPath(this.path); @@ -178,7 +179,7 @@ export class JupyterActions extends JupyterActions0 { // tells them to open this jupyter notebook, so it can provide the compute // functionality. - private conatApi = async () => { + conatApi = async () => { const compute_server_id = await this.getComputeServerId(); const api = webapp_client.project_client.conatApi( this.project_id, @@ -214,13 +215,7 @@ export class JupyterActions extends JupyterActions0 { // temporary proof of concept runCell = async (id: string) => { - const api = await this.conatApi(); - const resp = await api.editor.jupyterRun(this.syncdbPath, [id]); - const output = {}; - for (let i = 0; i < resp.length; i++) { - output[i] = resp[i]; - } - this.syncdb.set({ id, type: "cell", output }); + await runCell({ actions: this, id }); }; initOpenLog = () => { @@ -278,7 +273,8 @@ export class JupyterActions extends JupyterActions0 { this.clear_cell(id, save); return; } - this.run_code_cell(id, save, no_halt); + this.runCell(id); + //this.run_code_cell(id, save, no_halt); if (save) { this.save_asap(); } diff --git a/src/packages/frontend/jupyter/run-cell.ts b/src/packages/frontend/jupyter/run-cell.ts new file mode 100644 index 0000000000..6c818e5582 --- /dev/null +++ b/src/packages/frontend/jupyter/run-cell.ts @@ -0,0 +1,30 @@ +import { OutputHandler } from "@cocalc/jupyter/execute/output-handler"; +import { type JupyterActions } from "./browser-actions"; + +export async function runCell({ + actions, + id, +}: { + actions: JupyterActions; + id: string; +}) { + const cell = actions.store.getIn(["cells", id])?.toJS(); + if (cell == null) { + // nothing to do + return; + } + cell.output = null; + actions._set(cell); + const handler = new OutputHandler({ cell }); + const api = await actions.conatApi(); + const mesgs = await api.editor.jupyterRun(actions.syncdbPath, [ + { id: cell.id, input: cell.input }, + ]); + console.log(mesgs); + for (const mesg of mesgs) { + handler.process(mesg); + } + cell.state = "done"; + console.log(cell); + actions._set(cell); +} diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts index d610d3807e..499e830945 100644 --- a/src/packages/jupyter/control.ts +++ b/src/packages/jupyter/control.ts @@ -50,17 +50,17 @@ export function jupyterStart({ // run the cells with given id... export async function jupyterRun({ path, - ids, + cells, }: { path: string; - ids: string[]; + cells: { id: string; input: string }[]; }) { - logger.debug("jupyterRun", { path, ids }); + logger.debug("jupyterRun", { path, cells }); const session = sessions[path]; if (session == null) { throw Error(`${path} not running`); } - const { syncdb, actions, store } = session; + const { syncdb, actions } = session; if (syncdb.isClosed()) { // shouldn't be possible throw Error("syncdb is closed"); @@ -69,22 +69,14 @@ export async function jupyterRun({ logger.debug("jupyterRun: waiting until ready"); await once(syncdb, "ready"); } - // for (let i = 0; i < ids.length; i++) { - // actions.run_cell(ids[i], false); - // } logger.debug("jupyterRun: running"); - if (ids.length == 1) { - const code = store.get("cells").get(ids[0])?.get("input")?.trim(); - if (code) { - const result: any[] = []; - for (const x of await actions.jupyter_kernel.execute_code_now({ code })) { - if (x.msg_type == "execute_result") { - result.push(x.content); - } - } - return result; - } + let v = []; + for (const cell of cells) { + v = v.concat( + await actions.jupyter_kernel.execute_code_now({ code: cell.input }), + ); } + return v; } export function jupyterStop({ path }: { path: string }) { diff --git a/src/packages/jupyter/execute/output-handler.ts b/src/packages/jupyter/execute/output-handler.ts index 61536a3f77..0d6bec0dea 100644 --- a/src/packages/jupyter/execute/output-handler.ts +++ b/src/packages/jupyter/execute/output-handler.ts @@ -28,7 +28,7 @@ OutputHandler emits these events: import { callback } from "awaiting"; import { EventEmitter } from "events"; -import { close, server_time, len, to_json, is_object } from "@cocalc/util/misc"; +import { close, server_time, len, is_object } from "@cocalc/util/misc"; import { type TypedMap } from "@cocalc/util/types/typed-map"; const now = () => server_time().valueOf() - 0; @@ -62,7 +62,6 @@ interface Options { // If no messages for this many ms, then we update via set to indicate // that cell is being run. report_started_ms?: number; - dbg?; } type State = "ready" | "closed"; @@ -107,6 +106,57 @@ export class OutputHandler extends EventEmitter { this.stdin = this.stdin.bind(this); } + // mesg = from the kernel + process = (mesg) => { + if (mesg == null) { + // can't possibly happen, + return; + } + if (mesg.done) { + // done is a special internal cocalc message. + this.done(); + return; + } + if (mesg.content?.transient?.display_id != null) { + //this.handleTransientUpdate(mesg); + if (mesg.msg_type == "update_display_data") { + // don't also create a new output + return; + } + } + + if (mesg.msg_type === "clear_output") { + this.clear(mesg.content.wait); + return; + } + + if (mesg.content.comm_id != null) { + // ignore any comm/widget related messages here + return; + } + + if (mesg.content.execution_state === "busy") { + this.start(); + } + + if (mesg.content.payload != null) { + if (mesg.content.payload.length > 0) { + // payload shell message: + // Despite https://ipython.org/ipython-doc/3/development/messaging.html#payloads saying + // ""Payloads are considered deprecated, though their replacement is not yet implemented." + // we fully have to implement them, since they are used to implement (crazy, IMHO) + // things like %load in the python2 kernel! + for (const p of mesg.content.payload) { + this.payload(p); + } + return; + } + } else { + // Normal iopub output message + this.message(mesg.content); + } + }; + close = (): void => { if (this._state == "closed") return; this._state = "closed"; @@ -241,7 +291,8 @@ export class OutputHandler extends EventEmitter { this.emit("change", save); }; - // Process incoming messages. This may mutate mesg. + // Process incoming messages. **This may mutate mesg** and + // definitely mutates this.cell. message = (mesg: Message): void => { let has_exec_count: boolean; if (this._state === "closed") { @@ -375,10 +426,7 @@ export class OutputHandler extends EventEmitter { // https://github.com/sagemathinc/cocalc/issues/1933 this.message(payload); } else { - // No idea what to do with this... - if (typeof this._opts.dbg === "function") { - this._opts.dbg(`Unknown PAYLOAD: ${to_json(payload)}`); - } + // TODO: No idea what to do with this... } }; } diff --git a/src/packages/jupyter/package.json b/src/packages/jupyter/package.json index a109440ca8..7c4bd52173 100644 --- a/src/packages/jupyter/package.json +++ b/src/packages/jupyter/package.json @@ -47,6 +47,7 @@ "@cocalc/util": "workspace:*", "awaiting": "^3.0.0", "debug": "^4.4.0", + "events": "3.3.0", "expect": "^26.6.2", "he": "^1.2.0", "immutable": "^4.3.0", diff --git a/src/packages/jupyter/redux/project-actions.ts b/src/packages/jupyter/redux/project-actions.ts index 87eca214ed..92d28879b6 100644 --- a/src/packages/jupyter/redux/project-actions.ts +++ b/src/packages/jupyter/redux/project-actions.ts @@ -718,7 +718,6 @@ export class JupyterActions extends JupyterActions0 { max_output_length: this.store.get("max_output_length"), max_output_messages: MAX_OUTPUT_MESSAGES, report_started_ms: 250, - dbg, }); dbg("setting up jupyter_kernel.once('closed', ...) handler"); @@ -885,63 +884,19 @@ export class JupyterActions extends JupyterActions0 { exec.on("output", (mesg) => { // uncomment only for specific low level debugging -- see https://github.com/sagemathinc/cocalc/issues/7022 // dbg(`got mesg='${JSON.stringify(mesg)}'`); // !!!☡ ☡ ☡ -- EXTREME DANGER ☡ ☡ ☡ !!!! - - if (mesg == null) { - // can't possibly happen, of course. - const err = "empty mesg"; - dbg(`got error='${err}'`); - handler.error(err); - return; - } - if (mesg.done) { - // done is a special internal cocalc message. - handler.done(); - return; - } if (mesg.content?.transient?.display_id != null) { // See https://github.com/sagemathinc/cocalc/issues/2132 // We find any other outputs in the document with // the same transient.display_id, and set their output to // this mesg's output. this.handleTransientUpdate(mesg); - if (mesg.msg_type == "update_display_data") { - // don't also create a new output - return; - } } - - if (mesg.msg_type === "clear_output") { - handler.clear(mesg.content.wait); - return; - } - - if (mesg.content.comm_id != null) { - // ignore any comm/widget related messages - return; - } - if (mesg.content.execution_state === "idle") { this.store.removeListener("cell_change", cell_change); return; } - if (mesg.content.execution_state === "busy") { - handler.start(); - } - if (mesg.content.payload != null) { - if (mesg.content.payload.length > 0) { - // payload shell message: - // Despite https://ipython.org/ipython-doc/3/development/messaging.html#payloads saying - // ""Payloads are considered deprecated, though their replacement is not yet implemented." - // we fully have to implement them, since they are used to implement (crazy, IMHO) - // things like %load in the python2 kernel! - mesg.content.payload.map((p) => handler.payload(p)); - return; - } - } else { - // Normal iopub output message - handler.message(mesg.content); - return; - } + + handler.process(mesg); }); exec.on("error", (err) => { diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 901cc56769..20bdac5535 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -853,6 +853,9 @@ importers: debug: specifier: ^4.4.0 version: 4.4.1 + events: + specifier: 3.3.0 + version: 3.3.0 expect: specifier: ^26.6.2 version: 26.6.2 diff --git a/src/packages/project/conat/api/editor.ts b/src/packages/project/conat/api/editor.ts index f1ec6b82fa..5783ddae67 100644 --- a/src/packages/project/conat/api/editor.ts +++ b/src/packages/project/conat/api/editor.ts @@ -47,9 +47,12 @@ export async function jupyterStart(path: string) { await control.jupyterStart({ project_id, path, client: getClient(), fs }); } -export async function jupyterRun(path: string, ids: string[]) { +export async function jupyterRun( + path: string, + cells: { id: string; input: string }[], +) { await jupyterStart(path); - return await control.jupyterRun({ path, ids }); + return await control.jupyterRun({ path, cells }); } export async function jupyterStop(path: string) { From 979b272d2104f71640b2fff672d1c24ed90c7198 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 14:49:05 +0000 Subject: [PATCH 098/798] conat: basic framework for running jupyter code --- .../test/project/jupyter/run-code.test.ts | 122 ++++++++++++++++ .../conat/project/jupyter/run-code.ts | 135 ++++++++++++++++++ 2 files changed, 257 insertions(+) create mode 100644 src/packages/backend/conat/test/project/jupyter/run-code.test.ts create mode 100644 src/packages/conat/project/jupyter/run-code.ts diff --git a/src/packages/backend/conat/test/project/jupyter/run-code.test.ts b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts new file mode 100644 index 0000000000..6dbb84f2e7 --- /dev/null +++ b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts @@ -0,0 +1,122 @@ +/* + +DEVELOPMENT: + +pnpm test `pwd`/run-code.test.ts + +*/ + +import { before, after, connect } from "@cocalc/backend/conat/test/setup"; +import { + jupyterClient, + jupyterServer, +} from "@cocalc/conat/project/jupyter/run-code"; +import { uuid } from "@cocalc/util/misc"; + +beforeAll(before); + +describe("create very simple mocked jupyter runner and test evaluating code", () => { + let client1, client2; + it("create two clients", async () => { + client1 = connect(); + client2 = connect(); + }); + + let server; + const project_id = uuid(); + it("create jupyter code run server", () => { + // running code with this just results in two responses: the path and the cells + async function jupyterRun({ path, cells }) { + async function* runner() { + yield [{ path }]; + yield [{ cells }]; + } + return runner(); + } + + server = jupyterServer({ client: client1, project_id, jupyterRun }); + }); + + let client; + const path = "a.ipynb"; + const cells = [{ id: "a", input: "2+3" }]; + it("create a jupyter client, then run some code", async () => { + client = jupyterClient({ path, project_id, client: client2 }); + const iter = await client.run(cells); + const v: any[] = []; + for await (const output of iter) { + v.push(output); + } + expect(v).toEqual([[{ path }], [{ cells }]]); + }); + + const count = 100; + it(`run ${count} evaluations to ensure that the speed is reasonable (and also everything is kept properly ordered, etc.)`, async () => { + const start = Date.now(); + for (let i = 0; i < count; i++) { + const v: any[] = []; + const cells = [{ id: `${i}`, input: `${i} + ${i}` }]; + for await (const output of await client.run(cells)) { + v.push(output); + } + expect(v).toEqual([[{ path }], [{ cells }]]); + } + const evalsPerSecond = Math.floor((1000 * count) / (Date.now() - start)); + if (process.env.BENCH) { + console.log({ evalsPerSecond }); + } + expect(evalsPerSecond).toBeGreaterThan(25); + }); + + it("cleans up", () => { + server.close(); + client.close(); + }); +}); + +describe("create simple mocked jupyter runner that does actually eval an expression", () => { + let client1, client2; + it("create two clients", async () => { + client1 = connect(); + client2 = connect(); + }); + + let server; + const project_id = uuid(); + it("create jupyter code run server", () => { + // running code with this just results in two responses: the path and the cells + async function jupyterRun({ cells }) { + async function* runner() { + for (const { id, input } of cells) { + yield [{ id, output: eval(input) }]; + } + } + return runner(); + } + + server = jupyterServer({ client: client1, project_id, jupyterRun }); + }); + + let client; + const path = "b.ipynb"; + const cells = [ + { id: "a", input: "2+3" }, + { id: "b", input: "3**5" }, + ]; + it("create a jupyter client, then run some code", async () => { + client = jupyterClient({ path, project_id, client: client2 }); + const iter = await client.run(cells); + const v: any[] = []; + for await (const output of iter) { + v.push(output); + } + expect(v).toEqual([[{ id: "a", output: 5 }], [{ id: "b", output: 243 }]]); + }); + + it("cleans up", () => { + server.close(); + client.close(); + }); +}); + +afterAll(after); diff --git a/src/packages/conat/project/jupyter/run-code.ts b/src/packages/conat/project/jupyter/run-code.ts new file mode 100644 index 0000000000..1cc87d3873 --- /dev/null +++ b/src/packages/conat/project/jupyter/run-code.ts @@ -0,0 +1,135 @@ +/* +A conat socket server that takes as input + +Tests are in + +packages/backend/conat/test/juypter/run-code.test.s + +*/ + +import { type Client as ConatClient } from "@cocalc/conat/core/client"; +import { + type ConatSocketServer, + type ServerSocket, +} from "@cocalc/conat/socket"; +import { EventIterator } from "@cocalc/util/event-iterator"; +import { getLogger } from "@cocalc/conat/client"; + +const logger = getLogger("conat:project:jupyter:run-code"); + +function getSubject({ + project_id, + compute_server_id = 0, +}: { + project_id: string; + compute_server_id?: number; +}) { + return `jupyter.project-${project_id}.${compute_server_id}`; +} + +interface InputCell { + id: string; + input: string; +} + +type OutputMessage = any; + +type JupyterCodeRunner = (opts: { + // syncdb path + path: string; + // array of input cells to run + cells: InputCell[]; +}) => Promise>; + +export function jupyterServer({ + client, + project_id, + compute_server_id = 0, + jupyterRun, +}: { + client: ConatClient; + project_id: string; + compute_server_id?: number; + jupyterRun: JupyterCodeRunner; +}) { + const subject = getSubject({ project_id, compute_server_id }); + const server: ConatSocketServer = client.socket.listen(subject); + logger.debug("server: listening on ", { subject }); + + server.on("connection", (socket: ServerSocket) => { + logger.debug("server: got new connection", { + id: socket.id, + subject: socket.subject, + }); + + socket.on("request", async (mesg) => { + const { path, cells } = mesg.data; + try { + mesg.respondSync(null); + await handleRequest({ socket, jupyterRun, path, cells }); + } catch (err) { + console.log(err); + logger.debug("server: failed response -- ", err); + } + }); + }); + + return server; +} + +async function handleRequest({ socket, jupyterRun, path, cells }) { + const runner = await jupyterRun({ path, cells }); + for await (const result of runner) { + socket.write(result); + } + socket.write(null); +} + +class JupyterClient { + private iter?: EventIterator; + private socket; + constructor( + private client: ConatClient, + private subject: string, + private path: string, + ) { + this.socket = this.client.socket.connect(this.subject); + this.socket.once("close", () => this.iter?.end()); + } + + close = () => { + this.iter?.end(); + delete this.iter; + this.socket.close(); + }; + + run = async (cells: InputCell[]) => { + if (this.iter) { + // one evaluation at a time. + this.iter.end(); + delete this.iter; + } + this.iter = new EventIterator(this.socket, "data", { + map: (args) => { + if (args[0] == null) { + this.iter?.end(); + return null; + } else { + return args[0]; + } + }, + }); + await this.socket.request({ path: this.path, cells }); + return this.iter; + }; +} + +export function jupyterClient(opts: { + path: string; + project_id: string; + compute_server_id?: number; + client: ConatClient; +}) { + const subject = getSubject(opts); + return new JupyterClient(opts.client, subject, opts.path); +} From 4f6bb4140d346a881087e90406e6db31dbec5b0b Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 16:22:25 +0000 Subject: [PATCH 099/798] conat jupyter code runner: streaming output now working for one cell --- .../test/project/jupyter/run-code.test.ts | 39 +++++++++++--- src/packages/conat/project/api/editor.ts | 5 -- .../conat/project/jupyter/run-code.ts | 13 +++-- .../frontend/jupyter/browser-actions.ts | 1 + src/packages/frontend/jupyter/run-cell.ts | 35 +++++++++---- src/packages/jupyter/control.ts | 52 +++++++++++-------- src/packages/project/conat/api/editor.ts | 15 +++--- src/packages/project/conat/index.ts | 2 + src/packages/project/conat/jupyter.ts | 20 +++++++ 9 files changed, 131 insertions(+), 51 deletions(-) create mode 100644 src/packages/project/conat/jupyter.ts diff --git a/src/packages/backend/conat/test/project/jupyter/run-code.test.ts b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts index 6dbb84f2e7..f13e0249cb 100644 --- a/src/packages/backend/conat/test/project/jupyter/run-code.test.ts +++ b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts @@ -13,6 +13,9 @@ import { } from "@cocalc/conat/project/jupyter/run-code"; import { uuid } from "@cocalc/util/misc"; +// it's really 100+, but tests fails if less than this. +const MIN_EVALS_PER_SECOND = 10; + beforeAll(before); describe("create very simple mocked jupyter runner and test evaluating code", () => { @@ -28,8 +31,8 @@ describe("create very simple mocked jupyter runner and test evaluating code", () // running code with this just results in two responses: the path and the cells async function jupyterRun({ path, cells }) { async function* runner() { - yield [{ path }]; - yield [{ cells }]; + yield { path }; + yield { cells }; } return runner(); } @@ -65,7 +68,7 @@ describe("create very simple mocked jupyter runner and test evaluating code", () if (process.env.BENCH) { console.log({ evalsPerSecond }); } - expect(evalsPerSecond).toBeGreaterThan(25); + expect(evalsPerSecond).toBeGreaterThan(MIN_EVALS_PER_SECOND); }); it("cleans up", () => { @@ -83,18 +86,24 @@ describe("create simple mocked jupyter runner that does actually eval an expres let server; const project_id = uuid(); + const compute_server_id = 3; it("create jupyter code run server", () => { // running code with this just results in two responses: the path and the cells async function jupyterRun({ cells }) { async function* runner() { for (const { id, input } of cells) { - yield [{ id, output: eval(input) }]; + yield { id, output: eval(input) }; } } return runner(); } - server = jupyterServer({ client: client1, project_id, jupyterRun }); + server = jupyterServer({ + client: client1, + project_id, + jupyterRun, + compute_server_id, + }); }); let client; @@ -104,7 +113,12 @@ describe("create simple mocked jupyter runner that does actually eval an expres { id: "b", input: "3**5" }, ]; it("create a jupyter client, then run some code", async () => { - client = jupyterClient({ path, project_id, client: client2 }); + client = jupyterClient({ + path, + project_id, + client: client2, + compute_server_id, + }); const iter = await client.run(cells); const v: any[] = []; for await (const output of iter) { @@ -113,6 +127,19 @@ describe("create simple mocked jupyter runner that does actually eval an expres expect(v).toEqual([[{ id: "a", output: 5 }], [{ id: "b", output: 243 }]]); }); + it("run code that FAILS and see error is visible to client properly", async () => { + const iter = await client.run([ + { id: "a", input: "2+3" }, + { id: "b", input: "2+invalid" }, + ]); + try { + for await (const _ of iter) { + } + } catch (err) { + expect(`${err}`).toContain("ReferenceError: invalid is not defined"); + } + }); + it("cleans up", () => { server.close(); client.close(); diff --git a/src/packages/conat/project/api/editor.ts b/src/packages/conat/project/api/editor.ts index c18d7d00f8..e17d597e8a 100644 --- a/src/packages/conat/project/api/editor.ts +++ b/src/packages/conat/project/api/editor.ts @@ -13,7 +13,6 @@ export const editor = { jupyterRunNotebook: true, jupyterKernelLogo: true, jupyterKernels: true, - jupyterRun: true, formatString: true, @@ -47,10 +46,6 @@ export interface Editor { // path = the syncdb path (not *.ipynb) jupyterStart: (path: string) => Promise; jupyterStop: (path: string) => Promise; - jupyterRun: ( - path: string, - cells: { id: string; input: string }[], - ) => Promise; jupyterNbconvert: (opts: NbconvertParams) => Promise; diff --git a/src/packages/conat/project/jupyter/run-code.ts b/src/packages/conat/project/jupyter/run-code.ts index 1cc87d3873..98e1844282 100644 --- a/src/packages/conat/project/jupyter/run-code.ts +++ b/src/packages/conat/project/jupyter/run-code.ts @@ -39,7 +39,7 @@ type JupyterCodeRunner = (opts: { path: string; // array of input cells to run cells: InputCell[]; -}) => Promise>; +}) => Promise>; export function jupyterServer({ client, @@ -68,8 +68,9 @@ export function jupyterServer({ mesg.respondSync(null); await handleRequest({ socket, jupyterRun, path, cells }); } catch (err) { - console.log(err); + //console.log(err); logger.debug("server: failed response -- ", err); + socket.write(null, { headers: { error: `${err}` } }); } }); }); @@ -79,8 +80,8 @@ export function jupyterServer({ async function handleRequest({ socket, jupyterRun, path, cells }) { const runner = await jupyterRun({ path, cells }); - for await (const result of runner) { - socket.write(result); + for await (const mesg of runner) { + socket.write([mesg]); } socket.write(null); } @@ -111,6 +112,10 @@ class JupyterClient { } this.iter = new EventIterator(this.socket, "data", { map: (args) => { + if (args[1]?.error) { + this.iter?.throw(Error(args[1].error)); + return; + } if (args[0] == null) { this.iter?.end(); return null; diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 932bee24a3..02f806b221 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -214,6 +214,7 @@ export class JupyterActions extends JupyterActions0 { }; // temporary proof of concept + public jupyterClient?; runCell = async (id: string) => { await runCell({ actions: this, id }); }; diff --git a/src/packages/frontend/jupyter/run-cell.ts b/src/packages/frontend/jupyter/run-cell.ts index 6c818e5582..977533e5f1 100644 --- a/src/packages/frontend/jupyter/run-cell.ts +++ b/src/packages/frontend/jupyter/run-cell.ts @@ -1,5 +1,7 @@ import { OutputHandler } from "@cocalc/jupyter/execute/output-handler"; import { type JupyterActions } from "./browser-actions"; +import { jupyterClient } from "@cocalc/conat/project/jupyter/run-code"; +import { webapp_client } from "@cocalc/frontend/webapp-client"; export async function runCell({ actions, @@ -13,18 +15,33 @@ export async function runCell({ // nothing to do return; } + + if (actions.jupyterClient == null) { + // [ ] **TODO: Must invalidate this when compute server changes!!!!!** + // and + const compute_server_id = await actions.getComputeServerId(); + actions.jupyterClient = jupyterClient({ + path: actions.syncdbPath, + client: webapp_client.conat_client.conat(), + project_id: actions.project_id, + compute_server_id, + }); + } + const client = actions.jupyterClient; + if (client == null) { + throw Error("bug"); + } + cell.output = null; actions._set(cell); const handler = new OutputHandler({ cell }); - const api = await actions.conatApi(); - const mesgs = await api.editor.jupyterRun(actions.syncdbPath, [ - { id: cell.id, input: cell.input }, - ]); - console.log(mesgs); - for (const mesg of mesgs) { - handler.process(mesg); + const runner = await client.run([cell]); + for await (const mesgs of runner) { + for (const mesg of mesgs) { + handler.process(mesg); + actions._set(cell, false); + } } cell.state = "done"; - console.log(cell); - actions._set(cell); + actions._set(cell, true); } diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts index 499e830945..27726da352 100644 --- a/src/packages/jupyter/control.ts +++ b/src/packages/jupyter/control.ts @@ -47,7 +47,21 @@ export function jupyterStart({ sessions[path] = { syncdb, actions, store }; } -// run the cells with given id... +export function jupyterStop({ path }: { path: string }) { + const session = sessions[path]; + if (session == null) { + logger.debug("jupyterStop: ", path, " - not running"); + } else { + const { syncdb } = session; + logger.debug("jupyterStop: ", path, " - stopping it"); + syncdb.close(); + delete sessions[path]; + const path_ipynb = original_path(path); + removeJupyterRedux(path_ipynb, project_id); + } +} + +// Returns async iterator over outputs export async function jupyterRun({ path, cells, @@ -56,6 +70,7 @@ export async function jupyterRun({ cells: { id: string; input: string }[]; }) { logger.debug("jupyterRun", { path, cells }); + const session = sessions[path]; if (session == null) { throw Error(`${path} not running`); @@ -70,25 +85,20 @@ export async function jupyterRun({ await once(syncdb, "ready"); } logger.debug("jupyterRun: running"); - let v = []; - for (const cell of cells) { - v = v.concat( - await actions.jupyter_kernel.execute_code_now({ code: cell.input }), - ); - } - return v; -} - -export function jupyterStop({ path }: { path: string }) { - const session = sessions[path]; - if (session == null) { - logger.debug("jupyterStop: ", path, " - not running"); - } else { - const { syncdb } = session; - logger.debug("jupyterStop: ", path, " - stopping it"); - syncdb.close(); - delete sessions[path]; - const path_ipynb = original_path(path); - removeJupyterRedux(path_ipynb, project_id); + async function* run() { + for (const cell of cells) { + const output = actions.jupyter_kernel.execute_code({ + halt_on_error: true, + code: cell.input, + }); + for await (const mesg of output.iter()) { + yield mesg; + } + if (actions.jupyter_kernel.failedError) { + // kernel failed during call + throw Error(actions.jupyter_kernel.failedError); + } + } } + return await run(); } diff --git a/src/packages/project/conat/api/editor.ts b/src/packages/project/conat/api/editor.ts index 5783ddae67..c858c7d124 100644 --- a/src/packages/project/conat/api/editor.ts +++ b/src/packages/project/conat/api/editor.ts @@ -47,12 +47,15 @@ export async function jupyterStart(path: string) { await control.jupyterStart({ project_id, path, client: getClient(), fs }); } -export async function jupyterRun( - path: string, - cells: { id: string; input: string }[], -) { - await jupyterStart(path); - return await control.jupyterRun({ path, cells }); +// IMPORTANT: jupyterRun is NOT used directly by the API, but instead by packages/project/conat/jupyter.ts +// It is convenient to have it here so it can call jupyterStart above, etc. The reason is because +// this returns an async iterator managed using a dedicated socket, and the api is request/response.. +export async function jupyterRun(opts: { + path: string; + cells: { id: string; input: string }[]; +}) { + await jupyterStart(opts.path); + return await control.jupyterRun(opts); } export async function jupyterStop(path: string) { diff --git a/src/packages/project/conat/index.ts b/src/packages/project/conat/index.ts index 056c9c8c7b..a93de46d88 100644 --- a/src/packages/project/conat/index.ts +++ b/src/packages/project/conat/index.ts @@ -17,12 +17,14 @@ import { init as initRead } from "./files/read"; import { init as initWrite } from "./files/write"; import { init as initProjectStatus } from "@cocalc/project/project-status/server"; import { init as initUsageInfo } from "@cocalc/project/usage-info"; +import { init as initJupyter } from "./jupyter"; const logger = getLogger("project:conat:index"); export default async function init() { logger.debug("starting Conat project services"); await initAPI(); + await initJupyter(); await initOpenFiles(); initWebsocketApi(); await initListings(); diff --git a/src/packages/project/conat/jupyter.ts b/src/packages/project/conat/jupyter.ts new file mode 100644 index 0000000000..f64859571b --- /dev/null +++ b/src/packages/project/conat/jupyter.ts @@ -0,0 +1,20 @@ +import { jupyterRun } from "@cocalc/project/conat/api/editor"; +import { jupyterServer } from "@cocalc/conat/project/jupyter/run-code"; +import { connectToConat } from "@cocalc/project/conat/connection"; +import { compute_server_id, project_id } from "@cocalc/project/data"; +import { getLogger } from "@cocalc/project/logger"; + +const logger = getLogger("project:conat:jupyter"); + +let server: any = null; +export function init() { + logger.debug("initializing jupyter run server"); + const client = connectToConat(); + server = jupyterServer({ client, project_id, compute_server_id, jupyterRun }); +} + +export function close() { + logger.debug("closing jupyter run server"); + server?.close(); + server = null; +} From 16b799406b767d7d0ee82d2552780630f7b71031 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 16:27:21 +0000 Subject: [PATCH 100/798] depcheck fix --- src/packages/jupyter/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/packages/jupyter/package.json b/src/packages/jupyter/package.json index 7c4bd52173..24c0d73b8f 100644 --- a/src/packages/jupyter/package.json +++ b/src/packages/jupyter/package.json @@ -24,7 +24,7 @@ "build": "../node_modules/.bin/tsc --build", "clean": "rm -rf node_modules dist", "test": "pnpm exec jest --forceExit --maxWorkers=1", - "depcheck": "pnpx depcheck", + "depcheck": "pnpx depcheck --ignores events", "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput" }, "files": [ From e02db38fbb1388ba3965221ad16fb4e6c385372d Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 17:13:00 +0000 Subject: [PATCH 101/798] ts --- src/packages/frontend/jupyter/browser-actions.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 02f806b221..2e34745151 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -215,7 +215,7 @@ export class JupyterActions extends JupyterActions0 { // temporary proof of concept public jupyterClient?; - runCell = async (id: string) => { + runCell = async (id: string, _noHalt) => { await runCell({ actions: this, id }); }; @@ -274,7 +274,7 @@ export class JupyterActions extends JupyterActions0 { this.clear_cell(id, save); return; } - this.runCell(id); + this.runCell(id, no_halt); //this.run_code_cell(id, save, no_halt); if (save) { this.save_asap(); From a43db52701c7729b8e26349a78b97def2c77918e Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 18:21:55 +0000 Subject: [PATCH 102/798] jupyter eval: organizing code; removing flicker --- .../conat/project/jupyter/run-code.ts | 15 +- src/packages/conat/socket/server-socket.ts | 10 +- .../frontend/jupyter/browser-actions.ts | 135 ++++++++++++------ src/packages/frontend/jupyter/cell-list.tsx | 2 +- src/packages/frontend/jupyter/run-cell.ts | 47 ------ src/packages/jupyter/control.ts | 2 +- 6 files changed, 113 insertions(+), 98 deletions(-) delete mode 100644 src/packages/frontend/jupyter/run-cell.ts diff --git a/src/packages/conat/project/jupyter/run-code.ts b/src/packages/conat/project/jupyter/run-code.ts index 98e1844282..4df926c29d 100644 --- a/src/packages/conat/project/jupyter/run-code.ts +++ b/src/packages/conat/project/jupyter/run-code.ts @@ -53,7 +53,10 @@ export function jupyterServer({ jupyterRun: JupyterCodeRunner; }) { const subject = getSubject({ project_id, compute_server_id }); - const server: ConatSocketServer = client.socket.listen(subject); + const server: ConatSocketServer = client.socket.listen(subject, { + keepAlive: 5000, + keepAliveTimeout: 5000, + }); logger.debug("server: listening on ", { subject }); server.on("connection", (socket: ServerSocket) => { @@ -73,6 +76,10 @@ export function jupyterServer({ socket.write(null, { headers: { error: `${err}` } }); } }); + + socket.on("closed", () => { + logger.debug("socket closed", { id: socket.id }); + }); }); return server; @@ -81,7 +88,11 @@ export function jupyterServer({ async function handleRequest({ socket, jupyterRun, path, cells }) { const runner = await jupyterRun({ path, cells }); for await (const mesg of runner) { - socket.write([mesg]); + if (socket.state == "closed") { + logger.debug("socket closed -- server is now handling output!", mesg); + } else { + socket.write([mesg]); + } } socket.write(null); } diff --git a/src/packages/conat/socket/server-socket.ts b/src/packages/conat/socket/server-socket.ts index edfdc225a1..531e151949 100644 --- a/src/packages/conat/socket/server-socket.ts +++ b/src/packages/conat/socket/server-socket.ts @@ -51,6 +51,7 @@ export class ServerSocket extends EventEmitter { this.initKeepAlive(); } + private firstPing = true; private initKeepAlive = () => { this.alive?.close(); this.alive = keepAlive({ @@ -59,10 +60,15 @@ export class ServerSocket extends EventEmitter { await this.request(null, { headers: { [SOCKET_HEADER_CMD]: "ping" }, timeout: this.conatSocket.keepAliveTimeout, - // waitForInterest is very important in a cluster -- also, obviously + // waitForInterest for the *first ping* is very important + // in a cluster -- also, obviously // if somebody just opened a socket, they probably exist. - waitForInterest: true, + // However, after the first ping, we want to fail + // very quickly if the client disappears (and hence no + // more interest). + waitForInterest: this.firstPing, }); + this.firstPing = false; }, disconnect: this.close, keepAlive: this.conatSocket.keepAlive, diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 2e34745151..0cc92b136e 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -57,7 +57,9 @@ import getKernelSpec from "@cocalc/frontend/jupyter/kernelspecs"; import { get as getUsageInfo } from "@cocalc/conat/project/usage-info"; import { delay } from "awaiting"; import { until } from "@cocalc/util/async-utils"; -import { runCell } from "./run-cell"; +import { jupyterClient } from "@cocalc/conat/project/jupyter/run-code"; +import { OutputHandler } from "@cocalc/jupyter/execute/output-handler"; +import { throttle } from "lodash"; // local cache: map project_id (string) -> kernels (immutable) let jupyter_kernels = Map(); @@ -175,50 +177,6 @@ export class JupyterActions extends JupyterActions0 { } } - // if the project or compute server is running and listening, this call - // tells them to open this jupyter notebook, so it can provide the compute - // functionality. - - conatApi = async () => { - const compute_server_id = await this.getComputeServerId(); - const api = webapp_client.project_client.conatApi( - this.project_id, - compute_server_id, - ); - return api; - }; - - initBackend = async () => { - await until( - async () => { - if (this.is_closed()) { - return true; - } - try { - const api = await this.conatApi(); - await api.editor.jupyterStart(this.syncdbPath); - console.log("initialized ", this.path); - return true; - } catch (err) { - console.log("failed to initialize ", this.path, err); - return false; - } - }, - { min: 3000 }, - ); - }; - - stopBackend = async () => { - const api = await this.conatApi(); - await api.editor.jupyterStop(this.syncdbPath); - }; - - // temporary proof of concept - public jupyterClient?; - runCell = async (id: string, _noHalt) => { - await runCell({ actions: this, id }); - }; - initOpenLog = () => { // Put an entry in the project log once the jupyter notebook gets opened and // shows cells. @@ -377,6 +335,7 @@ export class JupyterActions extends JupyterActions0 { public async close(): Promise { if (this.is_closed()) return; + this.jupyterClient?.close(); await super.close(); } @@ -1494,4 +1453,90 @@ export class JupyterActions extends JupyterActions0 { } return; }; + + // if the project or compute server is running and listening, this call + // tells them to open this jupyter notebook, so it can provide the compute + // functionality. + + conatApi = async () => { + const compute_server_id = await this.getComputeServerId(); + const api = webapp_client.project_client.conatApi( + this.project_id, + compute_server_id, + ); + return api; + }; + + initBackend = async () => { + await until( + async () => { + if (this.is_closed()) { + return true; + } + try { + const api = await this.conatApi(); + await api.editor.jupyterStart(this.syncdbPath); + return true; + } catch (err) { + console.log("failed to initialize ", this.path, err); + return false; + } + }, + { min: 3000 }, + ); + }; + + stopBackend = async () => { + const api = await this.conatApi(); + await api.editor.jupyterStop(this.syncdbPath); + }; + + // temporary proof of concept + private jupyterClient?; + runCell = async (id: string, _noHalt) => { + const cell = this.store.getIn(["cells", id])?.toJS(); + if (cell == null) { + // nothing to do + return; + } + + if (this.jupyterClient == null) { + // [ ] **TODO: Must invalidate this when compute server changes!!!!!** + // and + const compute_server_id = await this.getComputeServerId(); + this.jupyterClient = jupyterClient({ + path: this.syncdbPath, + client: webapp_client.conat_client.conat(), + project_id: this.project_id, + compute_server_id, + }); + } + const client = this.jupyterClient; + if (client == null) { + throw Error("bug"); + } + + if (cell.output) { + // trick to avoid flicker + for (const n in cell.output) { + if (n == "0") continue; + cell.output[n] = null; + } + this._set(cell, false); + } + const handler = new OutputHandler({ cell }); + const f = throttle(() => this._set(cell, false), 1000 / 24, { + leading: false, + trailing: true, + }); + handler.on("change", f); + const runner = await client.run([cell]); + for await (const mesgs of runner) { + for (const mesg of mesgs) { + handler.process(mesg); + } + } + handler.done(); + this._set(cell, true); + }; } diff --git a/src/packages/frontend/jupyter/cell-list.tsx b/src/packages/frontend/jupyter/cell-list.tsx index d8a717943e..3812b636b0 100644 --- a/src/packages/frontend/jupyter/cell-list.tsx +++ b/src/packages/frontend/jupyter/cell-list.tsx @@ -434,7 +434,7 @@ export const CellList: React.FC = (props: CellListProps) => { if (index == null) { index = cell_list.indexOf(id) ?? 0; } - const dragHandle = actions?.store.is_cell_editable(id) ? ( + const dragHandle = actions?.store?.is_cell_editable(id) ? ( Date: Sun, 27 Jul 2025 19:38:35 +0000 Subject: [PATCH 103/798] jupyter: evaluation when client vanishes (with one cell) is working --- .../conat/project/jupyter/run-code.ts | 59 +++++++++++++++++-- src/packages/jupyter/control.ts | 37 +++++++++--- .../jupyter/execute/output-handler.ts | 2 + src/packages/jupyter/ipynb/export-to-ipynb.ts | 2 + src/packages/project/conat/jupyter.ts | 9 ++- 5 files changed, 94 insertions(+), 15 deletions(-) diff --git a/src/packages/conat/project/jupyter/run-code.ts b/src/packages/conat/project/jupyter/run-code.ts index 4df926c29d..f9b29fa207 100644 --- a/src/packages/conat/project/jupyter/run-code.ts +++ b/src/packages/conat/project/jupyter/run-code.ts @@ -34,23 +34,44 @@ interface InputCell { type OutputMessage = any; -type JupyterCodeRunner = (opts: { +export interface RunOptions { // syncdb path path: string; // array of input cells to run cells: InputCell[]; -}) => Promise>; +} + +type JupyterCodeRunner = ( + opts: RunOptions, +) => Promise>; + +interface OutputHandler { + process: (mesg: OutputMessage) => void; + done: () => void; +} +type CreateOutputHandler = (opts: { + path: string; + cells: InputCell[]; +}) => OutputHandler; export function jupyterServer({ client, project_id, compute_server_id = 0, + // jupyterRun takes a path and cells to run and returns an async iterator + // over the outputs. jupyterRun, + // outputHandler takes a path and returns an OutputHandler, which can be + // used to process the output and include it in the notebook. It is used + // as a fallback in case the client that initiated running cells is + // disconnected, so output won't be lost. + outputHandler, }: { client: ConatClient; project_id: string; compute_server_id?: number; jupyterRun: JupyterCodeRunner; + outputHandler?: CreateOutputHandler; }) { const subject = getSubject({ project_id, compute_server_id }); const server: ConatSocketServer = client.socket.listen(subject, { @@ -69,11 +90,13 @@ export function jupyterServer({ const { path, cells } = mesg.data; try { mesg.respondSync(null); - await handleRequest({ socket, jupyterRun, path, cells }); + await handleRequest({ socket, jupyterRun, outputHandler, path, cells }); } catch (err) { //console.log(err); logger.debug("server: failed response -- ", err); - socket.write(null, { headers: { error: `${err}` } }); + if (socket.state != "closed") { + socket.write(null, { headers: { error: `${err}` } }); + } } }); @@ -85,15 +108,39 @@ export function jupyterServer({ return server; } -async function handleRequest({ socket, jupyterRun, path, cells }) { +async function handleRequest({ + socket, + jupyterRun, + outputHandler, + path, + cells, +}) { const runner = await jupyterRun({ path, cells }); + const output: OutputMessage[] = []; + let handler: OutputHandler | null = null; for await (const mesg of runner) { if (socket.state == "closed") { - logger.debug("socket closed -- server is now handling output!", mesg); + if (handler == null) { + logger.debug("socket closed -- server must handle output"); + if (outputHandler == null) { + throw Error("no output handler available"); + } + handler = outputHandler({ path, cells }); + if (handler == null) { + throw Error("bug -- outputHandler must return a handler"); + } + for (const prev of output) { + handler.process(prev); + } + output.length = 0; + } + handler.process(mesg); } else { + output.push(mesg); socket.write([mesg]); } } + handler?.done(); socket.write(null); } diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts index 2641a5fd24..9b4ac09fbe 100644 --- a/src/packages/jupyter/control.ts +++ b/src/packages/jupyter/control.ts @@ -5,6 +5,9 @@ import { getLogger } from "@cocalc/backend/logger"; import { initJupyterRedux, removeJupyterRedux } from "@cocalc/jupyter/kernel"; import { original_path } from "@cocalc/util/misc"; import { once } from "@cocalc/util/async-utils"; +import { OutputHandler } from "@cocalc/jupyter/execute/output-handler"; +import { throttle } from "lodash"; +import { type RunOptions } from "@cocalc/conat/project/jupyter/run-code"; const logger = getLogger("jupyter:control"); @@ -62,14 +65,8 @@ export function jupyterStop({ path }: { path: string }) { } // Returns async iterator over outputs -export async function jupyterRun({ - path, - cells, -}: { - path: string; - cells: { id: string; input: string }[]; -}) { - logger.debug("jupyterRun", { path }) // , cells }); +export async function jupyterRun({ path, cells }: RunOptions) { + logger.debug("jupyterRun", { path }); // , cells }); const session = sessions[path]; if (session == null) { @@ -102,3 +99,27 @@ export async function jupyterRun({ } return await run(); } + +const BACKEND_OUTPUT_FPS = 8; +export function outputHandler({ path, cells }: RunOptions) { + if (sessions[path] == null) { + throw Error(`session '${path}' not available`); + } + const { actions } = sessions[path]; + // todo: need to handle multiple cells + const cell = { type: "cell" as "cell", ...cells[0] }; + const handler = new OutputHandler({ cell }); + const f = throttle( + () => { + logger.debug("outputHandler", path, cell); + actions._set(cell, true); + }, + 1000 / BACKEND_OUTPUT_FPS, + { + leading: false, + trailing: true, + }, + ); + handler.on("change", f); + return handler; +} diff --git a/src/packages/jupyter/execute/output-handler.ts b/src/packages/jupyter/execute/output-handler.ts index 0d6bec0dea..f1ef5a8eff 100644 --- a/src/packages/jupyter/execute/output-handler.ts +++ b/src/packages/jupyter/execute/output-handler.ts @@ -38,6 +38,8 @@ const MAX_SAVE_INTERVAL_MS = 45000; import { type Cell } from "@cocalc/jupyter/ipynb/export-to-ipynb"; +export { type Cell }; + interface Message { execution_state?; execution_count?: number; diff --git a/src/packages/jupyter/ipynb/export-to-ipynb.ts b/src/packages/jupyter/ipynb/export-to-ipynb.ts index ebd57e53d1..f3dd354351 100644 --- a/src/packages/jupyter/ipynb/export-to-ipynb.ts +++ b/src/packages/jupyter/ipynb/export-to-ipynb.ts @@ -14,6 +14,8 @@ type CellType = "code" | "markdown" | "raw"; type Tags = { [key: string]: boolean }; export interface Cell { + type?: "cell"; + id?: string; cell_type?: CellType; input?: string; collapsed?: boolean; diff --git a/src/packages/project/conat/jupyter.ts b/src/packages/project/conat/jupyter.ts index f64859571b..1405684198 100644 --- a/src/packages/project/conat/jupyter.ts +++ b/src/packages/project/conat/jupyter.ts @@ -1,4 +1,5 @@ import { jupyterRun } from "@cocalc/project/conat/api/editor"; +import { outputHandler } from "@cocalc/jupyter/control"; import { jupyterServer } from "@cocalc/conat/project/jupyter/run-code"; import { connectToConat } from "@cocalc/project/conat/connection"; import { compute_server_id, project_id } from "@cocalc/project/data"; @@ -10,7 +11,13 @@ let server: any = null; export function init() { logger.debug("initializing jupyter run server"); const client = connectToConat(); - server = jupyterServer({ client, project_id, compute_server_id, jupyterRun }); + server = jupyterServer({ + client, + project_id, + compute_server_id, + jupyterRun, + outputHandler, + }); } export function close() { From 61a82f1db261273f58c822a3583fa6a82e73d84a Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 20:42:13 +0000 Subject: [PATCH 104/798] jupyter run: add unit test for "client closes connection" --- .../test/project/jupyter/run-code.test.ts | 94 ++++++++++++++++++- .../conat/project/jupyter/run-code.ts | 1 + 2 files changed, 93 insertions(+), 2 deletions(-) diff --git a/src/packages/backend/conat/test/project/jupyter/run-code.test.ts b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts index f13e0249cb..39c3fb0035 100644 --- a/src/packages/backend/conat/test/project/jupyter/run-code.test.ts +++ b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts @@ -6,12 +6,13 @@ pnpm test `pwd`/run-code.test.ts */ -import { before, after, connect } from "@cocalc/backend/conat/test/setup"; +import { before, after, connect, wait } from "@cocalc/backend/conat/test/setup"; import { jupyterClient, jupyterServer, } from "@cocalc/conat/project/jupyter/run-code"; import { uuid } from "@cocalc/util/misc"; +import { delay } from "awaiting"; // it's really 100+, but tests fails if less than this. const MIN_EVALS_PER_SECOND = 10; @@ -77,7 +78,7 @@ describe("create very simple mocked jupyter runner and test evaluating code", () }); }); -describe("create simple mocked jupyter runner that does actually eval an expression", () => { +describe("create simple mocked jupyter runner that does actually eval an expression", () => { let client1, client2; it("create two clients", async () => { client1 = connect(); @@ -146,4 +147,93 @@ describe("create simple mocked jupyter runner that does actually eval an expres }); }); +describe("create mocked jupyter runner that does failover to backend output management when client disconnects", () => { + let client1, client2; + it("create two clients", async () => { + client1 = connect(); + client2 = connect(); + }); + + const path = "b.ipynb"; + const cells = [ + { id: "a", input: "10*(2+3)" }, + { id: "b", input: "100" }, + ]; + let server; + const project_id = uuid(); + let handler: any = null; + + it("create jupyter code run server that also takes as long as the output to run", () => { + async function jupyterRun({ cells }) { + async function* runner() { + for (const { id, input } of cells) { + const output = eval(input); + await delay(output); + yield { id, output }; + } + } + return runner(); + } + + class OutputHandler { + messages: any[] = []; + + constructor(public cells) {} + + process = (mesg: any) => { + this.messages.push(mesg); + }; + done = () => { + this.messages.push({ done: true }); + }; + } + + function outputHandler({ path: path0, cells }) { + if (path0 != path) { + throw Error(`path must be ${path}`); + } + handler = new OutputHandler(cells); + return handler; + } + + server = jupyterServer({ + client: client1, + project_id, + jupyterRun, + outputHandler, + }); + }); + + let client; + it("create a jupyter client, then run some code (doesn't use output handler)", async () => { + client = jupyterClient({ + path, + project_id, + client: client2, + }); + const iter = await client.run(cells); + const v: any[] = []; + for await (const output of iter) { + v.push(output); + } + expect(v).toEqual([[{ id: "a", output: 50 }], [{ id: "b", output: 100 }]]); + }); + + it("starts code running then closes the client, which causes output to have to be placed in the handler instead.", async () => { + const iter = await client.run(cells); + client.close(); + await wait({ until: () => handler.messages.length >= 3 }); + expect(handler.messages).toEqual([ + { id: "a", output: 50 }, + { id: "b", output: 100 }, + { done: true }, + ]); + }); + + it("cleans up", () => { + server.close(); + client.close(); + }); +}); + afterAll(after); diff --git a/src/packages/conat/project/jupyter/run-code.ts b/src/packages/conat/project/jupyter/run-code.ts index f9b29fa207..f7fdbf8076 100644 --- a/src/packages/conat/project/jupyter/run-code.ts +++ b/src/packages/conat/project/jupyter/run-code.ts @@ -49,6 +49,7 @@ interface OutputHandler { process: (mesg: OutputMessage) => void; done: () => void; } + type CreateOutputHandler = (opts: { path: string; cells: InputCell[]; From 235d983270394667afc0840e44314a738bcbbb4f Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 21:53:16 +0000 Subject: [PATCH 105/798] jupyter cell runner: handle multiple input cells --- .../test/project/jupyter/run-code.test.ts | 10 +-- .../conat/project/jupyter/run-code.ts | 18 ++++- .../frontend/jupyter/browser-actions.ts | 70 ++++++++++++------- src/packages/jupyter/control.ts | 52 +++++++++----- .../jupyter/execute/output-handler.ts | 10 ++- 5 files changed, 111 insertions(+), 49 deletions(-) diff --git a/src/packages/backend/conat/test/project/jupyter/run-code.test.ts b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts index 39c3fb0035..fdb960d326 100644 --- a/src/packages/backend/conat/test/project/jupyter/run-code.test.ts +++ b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts @@ -32,8 +32,8 @@ describe("create very simple mocked jupyter runner and test evaluating code", () // running code with this just results in two responses: the path and the cells async function jupyterRun({ path, cells }) { async function* runner() { - yield { path }; - yield { cells }; + yield { path, id: "0" }; + yield { cells, id: "0" }; } return runner(); } @@ -51,7 +51,7 @@ describe("create very simple mocked jupyter runner and test evaluating code", () for await (const output of iter) { v.push(output); } - expect(v).toEqual([[{ path }], [{ cells }]]); + expect(v).toEqual([[{ path, id: "0" }], [{ cells, id: "0" }]]); }); const count = 100; @@ -63,7 +63,7 @@ describe("create very simple mocked jupyter runner and test evaluating code", () for await (const output of await client.run(cells)) { v.push(output); } - expect(v).toEqual([[{ path }], [{ cells }]]); + expect(v).toEqual([[{ path, id: "0" }], [{ cells, id: "0" }]]); } const evalsPerSecond = Math.floor((1000 * count) / (Date.now() - start)); if (process.env.BENCH) { @@ -220,7 +220,7 @@ describe("create mocked jupyter runner that does failover to backend output mana }); it("starts code running then closes the client, which causes output to have to be placed in the handler instead.", async () => { - const iter = await client.run(cells); + await client.run(cells); client.close(); await wait({ until: () => handler.messages.length >= 3 }); expect(handler.messages).toEqual([ diff --git a/src/packages/conat/project/jupyter/run-code.ts b/src/packages/conat/project/jupyter/run-code.ts index f7fdbf8076..ed9f0cbe6b 100644 --- a/src/packages/conat/project/jupyter/run-code.ts +++ b/src/packages/conat/project/jupyter/run-code.ts @@ -32,7 +32,16 @@ interface InputCell { input: string; } -type OutputMessage = any; +export interface OutputMessage { + // id = id of the cell + id: string; + // everything below is exactly from Jupyter + metadata?; + content?; + buffers?; + msg_type?: string; + done?: boolean; +} export interface RunOptions { // syncdb path @@ -183,7 +192,12 @@ class JupyterClient { } }, }); - await this.socket.request({ path: this.path, cells }); + // get rid of any fields except id and input from the cells, since, e.g., + // if there is a lot of output in a cell, there is no need to send that to the backend. + const cells1 = cells.map(({ id, input }) => { + return { id, input }; + }); + await this.socket.request({ path: this.path, cells: cells1 }); return this.iter; }; } diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 0cc92b136e..6ddf7e0d9b 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -61,6 +61,8 @@ import { jupyterClient } from "@cocalc/conat/project/jupyter/run-code"; import { OutputHandler } from "@cocalc/jupyter/execute/output-handler"; import { throttle } from "lodash"; +const OUTPUT_FPS = 29; + // local cache: map project_id (string) -> kernels (immutable) let jupyter_kernels = Map(); @@ -232,7 +234,7 @@ export class JupyterActions extends JupyterActions0 { this.clear_cell(id, save); return; } - this.runCell(id, no_halt); + this.runCells([id], no_halt); //this.run_code_cell(id, save, no_halt); if (save) { this.save_asap(); @@ -1491,15 +1493,18 @@ export class JupyterActions extends JupyterActions0 { await api.editor.jupyterStop(this.syncdbPath); }; - // temporary proof of concept - private jupyterClient?; - runCell = async (id: string, _noHalt) => { - const cell = this.store.getIn(["cells", id])?.toJS(); - if (cell == null) { - // nothing to do - return; - } + getOutputHandler = (cell) => { + const handler = new OutputHandler({ cell }); + const f = throttle(() => this._set(cell, false), 1000 / OUTPUT_FPS, { + leading: false, + trailing: true, + }); + handler.on("change", f); + return handler; + }; + private jupyterClient?; + runCells = async (ids: string[], _noHalt) => { if (this.jupyterClient == null) { // [ ] **TODO: Must invalidate this when compute server changes!!!!!** // and @@ -1515,28 +1520,43 @@ export class JupyterActions extends JupyterActions0 { if (client == null) { throw Error("bug"); } - - if (cell.output) { - // trick to avoid flicker - for (const n in cell.output) { - if (n == "0") continue; - cell.output[n] = null; + const cells: any[] = []; + for (const id of ids) { + const cell = this.store.getIn(["cells", id])?.toJS(); + if (!cell?.input?.trim()) { + // nothing to do + continue; + } + if (cell.output) { + // trick to avoid flicker + for (const n in cell.output) { + if (n == "0") continue; + cell.output[n] = null; + } + this._set(cell, false); } - this._set(cell, false); + cells.push(cell); } - const handler = new OutputHandler({ cell }); - const f = throttle(() => this._set(cell, false), 1000 / 24, { - leading: false, - trailing: true, - }); - handler.on("change", f); - const runner = await client.run([cell]); + + const runner = await client.run(cells); + let handler: null | OutputHandler = null; + let id: null | string = null; for await (const mesgs of runner) { for (const mesg of mesgs) { + if (mesg.id !== id || handler == null) { + id = mesg.id; + let cell = this.store.getIn(["cells", mesg.id])?.toJS(); + if (cell == null) { + // cell removed? + cell = { id }; + } + handler?.done(); + handler = this.getOutputHandler(cell); + } handler.process(mesg); } } - handler.done(); - this._set(cell, true); + handler?.done(); + this.save_asap(); }; } diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts index 9b4ac09fbe..8b3dffba0c 100644 --- a/src/packages/jupyter/control.ts +++ b/src/packages/jupyter/control.ts @@ -89,6 +89,7 @@ export async function jupyterRun({ path, cells }: RunOptions) { code: cell.input, }); for await (const mesg of output.iter()) { + mesg.id = cell.id; yield mesg; } if (actions.jupyter_kernel.failedError) { @@ -100,26 +101,45 @@ export async function jupyterRun({ path, cells }: RunOptions) { return await run(); } +class MulticellOutputHandler { + private id: string | null = null; + private handler: OutputHandler | null = null; + + constructor( + private cells: RunOptions["cells"], + private actions, + ) {} + + process = (mesg) => { + if (mesg.id !== this.id || this.handler == null) { + this.id = mesg.id; + let cell = this.cells[mesg.id] ?? { id: mesg.id }; + this.handler?.done(); + this.handler = new OutputHandler({ cell }); + const f = throttle( + () => this.actions._set({ ...cell, type: "cell" }, true), + 1000 / BACKEND_OUTPUT_FPS, + { + leading: true, + trailing: true, + }, + ); + this.handler.on("change", f); + } + this.handler!.process(mesg); + }; + + done = () => { + this.handler?.done(); + this.handler = null; + }; +} + const BACKEND_OUTPUT_FPS = 8; export function outputHandler({ path, cells }: RunOptions) { if (sessions[path] == null) { throw Error(`session '${path}' not available`); } const { actions } = sessions[path]; - // todo: need to handle multiple cells - const cell = { type: "cell" as "cell", ...cells[0] }; - const handler = new OutputHandler({ cell }); - const f = throttle( - () => { - logger.debug("outputHandler", path, cell); - actions._set(cell, true); - }, - 1000 / BACKEND_OUTPUT_FPS, - { - leading: false, - trailing: true, - }, - ); - handler.on("change", f); - return handler; + return new MulticellOutputHandler(cells, actions); } diff --git a/src/packages/jupyter/execute/output-handler.ts b/src/packages/jupyter/execute/output-handler.ts index f1ef5a8eff..f26b075cd8 100644 --- a/src/packages/jupyter/execute/output-handler.ts +++ b/src/packages/jupyter/execute/output-handler.ts @@ -54,6 +54,14 @@ interface Message { data?: { [mimeType: string]: any }; } +interface JupyterMessage { + metadata?; + content?; + buffers?; + msg_type?: string; + done?: boolean; +} + interface Options { // object; the cell whose output (etc.) will get mutated cell: Cell; @@ -109,7 +117,7 @@ export class OutputHandler extends EventEmitter { } // mesg = from the kernel - process = (mesg) => { + process = (mesg: JupyterMessage) => { if (mesg == null) { // can't possibly happen, return; From 8a5911b171163b1d1e674ec6975b00d060a12c5b Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 22:02:32 +0000 Subject: [PATCH 106/798] ensure cells run in order --- src/packages/frontend/jupyter/browser-actions.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 6ddf7e0d9b..e5a3f9ab7f 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -27,6 +27,7 @@ import { base64ToBuffer, bufferToBase64 } from "@cocalc/util/base64"; import { Config as FormatterConfig, Syntax } from "@cocalc/util/code-formatter"; import { closest_kernel_match, + field_cmp, from_json, history_path, merge_copy, @@ -1537,6 +1538,8 @@ export class JupyterActions extends JupyterActions0 { } cells.push(cell); } + // ensures cells run in order: + cells.sort(field_cmp("pos")); const runner = await client.run(cells); let handler: null | OutputHandler = null; From e6fc7176258d420fe55c2d597cb92a7db762199c Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 22:35:10 +0000 Subject: [PATCH 107/798] jupyter eval: support noHalt --- .../conat/project/jupyter/run-code.ts | 24 +++++++++++++++---- src/packages/frontend/components/time-ago.tsx | 2 +- .../frontend/jupyter/browser-actions.ts | 8 +++---- .../frontend/jupyter/cell-output-time.tsx | 16 ++++++++++--- src/packages/jupyter/control.ts | 10 +++++--- .../jupyter/execute/output-handler.ts | 4 ++-- 6 files changed, 46 insertions(+), 18 deletions(-) diff --git a/src/packages/conat/project/jupyter/run-code.ts b/src/packages/conat/project/jupyter/run-code.ts index ed9f0cbe6b..6efe51fbdf 100644 --- a/src/packages/conat/project/jupyter/run-code.ts +++ b/src/packages/conat/project/jupyter/run-code.ts @@ -48,6 +48,8 @@ export interface RunOptions { path: string; // array of input cells to run cells: InputCell[]; + // if true do not halt running the cells, even if one fails with an error + noHalt?: boolean; } type JupyterCodeRunner = ( @@ -97,10 +99,17 @@ export function jupyterServer({ }); socket.on("request", async (mesg) => { - const { path, cells } = mesg.data; + const { path, cells, noHalt } = mesg.data; try { mesg.respondSync(null); - await handleRequest({ socket, jupyterRun, outputHandler, path, cells }); + await handleRequest({ + socket, + jupyterRun, + outputHandler, + path, + cells, + noHalt, + }); } catch (err) { //console.log(err); logger.debug("server: failed response -- ", err); @@ -124,8 +133,9 @@ async function handleRequest({ outputHandler, path, cells, + noHalt, }) { - const runner = await jupyterRun({ path, cells }); + const runner = await jupyterRun({ path, cells, noHalt }); const output: OutputMessage[] = []; let handler: OutputHandler | null = null; for await (const mesg of runner) { @@ -172,7 +182,7 @@ class JupyterClient { this.socket.close(); }; - run = async (cells: InputCell[]) => { + run = async (cells: InputCell[], opts: { noHalt?: boolean } = {}) => { if (this.iter) { // one evaluation at a time. this.iter.end(); @@ -197,7 +207,11 @@ class JupyterClient { const cells1 = cells.map(({ id, input }) => { return { id, input }; }); - await this.socket.request({ path: this.path, cells: cells1 }); + await this.socket.request({ + path: this.path, + cells: cells1, + noHalt: opts.noHalt, + }); return this.iter; }; } diff --git a/src/packages/frontend/components/time-ago.tsx b/src/packages/frontend/components/time-ago.tsx index 784747c0d3..58010616ab 100644 --- a/src/packages/frontend/components/time-ago.tsx +++ b/src/packages/frontend/components/time-ago.tsx @@ -203,7 +203,7 @@ export const TimeAgo: React.FC = React.memo( }: TimeAgoElementProps) => { const { timeAgoAbsolute } = useAppContext(); - if (!date || date.valueOf()) { + if (!date?.valueOf()) { return <>; } diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index e5a3f9ab7f..f8000170ac 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -219,7 +219,7 @@ export class JupyterActions extends JupyterActions0 { public run_cell( id: string, save: boolean = true, - no_halt: boolean = false, + noHalt: boolean = false, ): void { if (this.store.get("read_only")) return; const cell = this.store.getIn(["cells", id]); @@ -235,7 +235,7 @@ export class JupyterActions extends JupyterActions0 { this.clear_cell(id, save); return; } - this.runCells([id], no_halt); + this.runCells([id], { noHalt }); //this.run_code_cell(id, save, no_halt); if (save) { this.save_asap(); @@ -1505,7 +1505,7 @@ export class JupyterActions extends JupyterActions0 { }; private jupyterClient?; - runCells = async (ids: string[], _noHalt) => { + runCells = async (ids: string[], opts: { noHalt?: boolean } = {}) => { if (this.jupyterClient == null) { // [ ] **TODO: Must invalidate this when compute server changes!!!!!** // and @@ -1541,7 +1541,7 @@ export class JupyterActions extends JupyterActions0 { // ensures cells run in order: cells.sort(field_cmp("pos")); - const runner = await client.run(cells); + const runner = await client.run(cells, opts); let handler: null | OutputHandler = null; let id: null | string = null; for await (const mesgs of runner) { diff --git a/src/packages/frontend/jupyter/cell-output-time.tsx b/src/packages/frontend/jupyter/cell-output-time.tsx index e4e7c031dc..333e3cde11 100644 --- a/src/packages/frontend/jupyter/cell-output-time.tsx +++ b/src/packages/frontend/jupyter/cell-output-time.tsx @@ -23,6 +23,14 @@ interface CellTimingProps { // make this small so smooth. const DELAY_MS = 100; +function humanReadableSeconds(s) { + if (s >= 0.9) { + return seconds2hms(s, true); + } else { + return `${Math.round(s * 1000)} ms`; + } +} + export default function CellTiming({ start, end, @@ -53,10 +61,12 @@ export default function CellTiming({ - Evaluated using {capitalize(kernel)} and took - about {seconds2hms(ms / 1000, true)}. + Took about {humanReadableSeconds(ms / 1000)}. Evaluated{" "} + + {kernel ? " using " : ""} + {capitalize(kernel)}. {last != null ? ( - <> Previous run took {seconds2hms(last / 1000, true)}. + <> Previous run took {humanReadableSeconds(last / 1000)}. ) : undefined} } diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts index 8b3dffba0c..b581dd9a1f 100644 --- a/src/packages/jupyter/control.ts +++ b/src/packages/jupyter/control.ts @@ -65,8 +65,8 @@ export function jupyterStop({ path }: { path: string }) { } // Returns async iterator over outputs -export async function jupyterRun({ path, cells }: RunOptions) { - logger.debug("jupyterRun", { path }); // , cells }); +export async function jupyterRun({ path, cells, noHalt }: RunOptions) { + logger.debug("jupyterRun", { path, noHalt }); const session = sessions[path]; if (session == null) { @@ -85,12 +85,16 @@ export async function jupyterRun({ path, cells }: RunOptions) { async function* run() { for (const cell of cells) { const output = actions.jupyter_kernel.execute_code({ - halt_on_error: true, + halt_on_error: !noHalt, code: cell.input, }); for await (const mesg of output.iter()) { mesg.id = cell.id; yield mesg; + if (!noHalt && mesg.msg_type == "error") { + // done running code because there was an error. + return; + } } if (actions.jupyter_kernel.failedError) { // kernel failed during call diff --git a/src/packages/jupyter/execute/output-handler.ts b/src/packages/jupyter/execute/output-handler.ts index f26b075cd8..47974f325a 100644 --- a/src/packages/jupyter/execute/output-handler.ts +++ b/src/packages/jupyter/execute/output-handler.ts @@ -28,10 +28,10 @@ OutputHandler emits these events: import { callback } from "awaiting"; import { EventEmitter } from "events"; -import { close, server_time, len, is_object } from "@cocalc/util/misc"; +import { close, len, is_object } from "@cocalc/util/misc"; import { type TypedMap } from "@cocalc/util/types/typed-map"; -const now = () => server_time().valueOf() - 0; +const now = () => Date.now(); const MIN_SAVE_INTERVAL_MS = 500; const MAX_SAVE_INTERVAL_MS = 45000; From 3ca8aa96035f359f65295aa0d3f68d59c3ce6918 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 22:41:22 +0000 Subject: [PATCH 108/798] jupyter runner: include the kernel (not bothering with this if exec moves to backend, since that should be rare, and this isn't super important functionality) --- src/packages/frontend/jupyter/browser-actions.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index f8000170ac..a7f90d9c13 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -1522,6 +1522,7 @@ export class JupyterActions extends JupyterActions0 { throw Error("bug"); } const cells: any[] = []; + const kernel = this.store.get("kernel"); for (const id of ids) { const cell = this.store.getIn(["cells", id])?.toJS(); if (!cell?.input?.trim()) { @@ -1553,6 +1554,7 @@ export class JupyterActions extends JupyterActions0 { // cell removed? cell = { id }; } + cell.kernel = kernel; handler?.done(); handler = this.getOutputHandler(cell); } From 198f0e2584f4bca657c2e6005692b0ddd4601cf6 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 23:44:25 +0000 Subject: [PATCH 109/798] jupyter run: record last evaluation time --- src/packages/frontend/jupyter/browser-actions.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index a7f90d9c13..1660b7b8d7 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -1535,6 +1535,8 @@ export class JupyterActions extends JupyterActions0 { if (n == "0") continue; cell.output[n] = null; } + // time last evaluation took + cell.last = cell.start && cell.end ? cell.end - cell.start : null; this._set(cell, false); } cells.push(cell); From d908e2663bd8a503550d08a359e65e6f248b2334 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 28 Jul 2025 01:15:24 +0000 Subject: [PATCH 110/798] jupyter run: queuing up cells to run --- .../frontend/jupyter/browser-actions.ts | 178 ++++++++++++------ src/packages/frontend/jupyter/cell-input.tsx | 4 +- src/packages/frontend/jupyter/cell-list.tsx | 3 + src/packages/frontend/jupyter/cell.tsx | 7 +- src/packages/frontend/jupyter/main.tsx | 10 +- .../frontend/jupyter/prompt/input.tsx | 1 - src/packages/jupyter/redux/actions.ts | 5 + src/packages/jupyter/redux/store.ts | 5 + 8 files changed, 152 insertions(+), 61 deletions(-) diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 1660b7b8d7..a4475db41d 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -8,7 +8,7 @@ browser-actions: additional actions that are only available in the web browser frontend. */ import * as awaiting from "awaiting"; -import { fromJS, Map } from "immutable"; +import { fromJS, Map, Set as iSet } from "immutable"; import { debounce, isEqual } from "lodash"; import { jupyter, labels } from "@cocalc/frontend/i18n"; import { getIntl } from "@cocalc/frontend/i18n/get-intl"; @@ -1496,74 +1496,140 @@ export class JupyterActions extends JupyterActions0 { getOutputHandler = (cell) => { const handler = new OutputHandler({ cell }); - const f = throttle(() => this._set(cell, false), 1000 / OUTPUT_FPS, { - leading: false, - trailing: true, - }); + let first = true; + const f = throttle( + () => { + // save first so that other clients know this cell is running. + this._set(cell, first); + first = false; + }, + 1000 / OUTPUT_FPS, + { + leading: false, + trailing: true, + }, + ); handler.on("change", f); return handler; }; - private jupyterClient?; - runCells = async (ids: string[], opts: { noHalt?: boolean } = {}) => { - if (this.jupyterClient == null) { - // [ ] **TODO: Must invalidate this when compute server changes!!!!!** - // and - const compute_server_id = await this.getComputeServerId(); - this.jupyterClient = jupyterClient({ - path: this.syncdbPath, - client: webapp_client.conat_client.conat(), - project_id: this.project_id, - compute_server_id, - }); + private addPendingCells = (ids: string[]) => { + let pendingCells = this.store.get("pendingCells") ?? iSet(); + for (const id of ids) { + pendingCells = pendingCells.add(id); } - const client = this.jupyterClient; - if (client == null) { - throw Error("bug"); + this.store.setState({ pendingCells }); + }; + private deletePendingCells = (ids: string[]) => { + let pendingCells = this.store.get("pendingCells"); + if (pendingCells == null) { + return; } - const cells: any[] = []; - const kernel = this.store.get("kernel"); for (const id of ids) { - const cell = this.store.getIn(["cells", id])?.toJS(); - if (!cell?.input?.trim()) { - // nothing to do - continue; + pendingCells = pendingCells.delete(id); + } + this.store.setState({ pendingCells }); + }; + + // uses inheritence so NOT arrow function + protected clearRunQueue() { + this.store?.setState({ pendingCells: iSet() }); + this.runQueue.length = 0; + } + + private jupyterClient?; + private runQueue: any[] = []; + private runningNow = false; + runCells = async (ids: string[], opts: { noHalt?: boolean } = {}) => { + if (this.runningNow) { + this.runQueue.push([ids, opts]); + this.addPendingCells(ids); + return; + } + try { + this.runningNow = true; + if (this.jupyterClient == null) { + // [ ] **TODO: Must invalidate this when compute server changes!!!!!** + // and + const compute_server_id = await this.getComputeServerId(); + this.jupyterClient = jupyterClient({ + path: this.syncdbPath, + client: webapp_client.conat_client.conat(), + project_id: this.project_id, + compute_server_id, + }); + } + const client = this.jupyterClient; + if (client == null) { + throw Error("bug"); } - if (cell.output) { - // trick to avoid flicker - for (const n in cell.output) { - if (n == "0") continue; - cell.output[n] = null; + const cells: any[] = []; + const kernel = this.store.get("kernel"); + + for (const id of ids) { + const cell = this.store.getIn(["cells", id])?.toJS(); + if (!cell?.input?.trim()) { + // nothing to do + continue; + } + if (!kernel) { + this._set({ type: "cell", id, state: "done" }); + continue; + } + if (cell.output) { + // trick to avoid flicker + for (const n in cell.output) { + if (n == "0") continue; + cell.output[n] = null; + } + // time last evaluation took + cell.last = cell.start && cell.end ? cell.end - cell.start : null; + this._set(cell, false); } - // time last evaluation took - cell.last = cell.start && cell.end ? cell.end - cell.start : null; - this._set(cell, false); + cells.push(cell); } - cells.push(cell); - } - // ensures cells run in order: - cells.sort(field_cmp("pos")); - - const runner = await client.run(cells, opts); - let handler: null | OutputHandler = null; - let id: null | string = null; - for await (const mesgs of runner) { - for (const mesg of mesgs) { - if (mesg.id !== id || handler == null) { - id = mesg.id; - let cell = this.store.getIn(["cells", mesg.id])?.toJS(); - if (cell == null) { - // cell removed? - cell = { id }; + this.addPendingCells(ids); + + // ensures cells run in order: + cells.sort(field_cmp("pos")); + + const runner = await client.run(cells, opts); + let handler: null | OutputHandler = null; + let id: null | string = null; + for await (const mesgs of runner) { + for (const mesg of mesgs) { + if (!opts.noHalt && mesg.msg_type == "error") { + this.clearRunQueue(); + } + if (mesg.id !== id || handler == null) { + id = mesg.id; + if (id == null) { + continue; + } + this.deletePendingCells([id]); + let cell = this.store.getIn(["cells", mesg.id])?.toJS(); + if (cell == null) { + // cell removed? + cell = { id }; + } + cell.kernel = kernel; + handler?.done(); + handler = this.getOutputHandler(cell); } - cell.kernel = kernel; - handler?.done(); - handler = this.getOutputHandler(cell); + handler.process(mesg); } - handler.process(mesg); + } + console.log("exited the runner loop"); + handler?.done(); + this.save_asap(); + } catch (err) { + console.log("runCells", err); + } finally { + this.runningNow = false; + if (this.runQueue.length > 0) { + const [ids, opts] = this.runQueue.shift(); + this.runCells(ids, opts); } } - handler?.done(); - this.save_asap(); }; } diff --git a/src/packages/frontend/jupyter/cell-input.tsx b/src/packages/frontend/jupyter/cell-input.tsx index d3f740ec5c..25f0409df8 100644 --- a/src/packages/frontend/jupyter/cell-input.tsx +++ b/src/packages/frontend/jupyter/cell-input.tsx @@ -73,6 +73,7 @@ export interface CellInputProps { computeServerId?: number; setShowAICellGen?: (show: Position) => void; dragHandle?: React.JSX.Element; + isPending?: boolean; } export const CellInput: React.FC = React.memo( @@ -96,7 +97,7 @@ export const CellInput: React.FC = React.memo( = React.memo( next.index !== cur.index || next.computeServerId != cur.computeServerId || next.dragHandle !== cur.dragHandle || + next.isPending !== cur.isPending || (next.cell_toolbar === "slideshow" && next.cell.get("slide") !== cur.cell.get("slide")) ), diff --git a/src/packages/frontend/jupyter/cell-list.tsx b/src/packages/frontend/jupyter/cell-list.tsx index 3812b636b0..e6342f78a2 100644 --- a/src/packages/frontend/jupyter/cell-list.tsx +++ b/src/packages/frontend/jupyter/cell-list.tsx @@ -91,6 +91,7 @@ interface CellListProps { llmTools?: LLMTools; computeServerId?: number; read_only?: boolean; + pendingCells?: immutable.Set; } export const CellList: React.FC = (props: CellListProps) => { @@ -121,6 +122,7 @@ export const CellList: React.FC = (props: CellListProps) => { llmTools, computeServerId, read_only, + pendingCells, } = props; const cellListDivRef = useRef(null); @@ -478,6 +480,7 @@ export const CellList: React.FC = (props: CellListProps) => { dragHandle={dragHandle} read_only={read_only} isDragging={isDragging} + isPending={pendingCells?.has(id)} /> ); diff --git a/src/packages/frontend/jupyter/cell.tsx b/src/packages/frontend/jupyter/cell.tsx index 3d31309d07..251f407db7 100644 --- a/src/packages/frontend/jupyter/cell.tsx +++ b/src/packages/frontend/jupyter/cell.tsx @@ -37,7 +37,6 @@ interface Props { font_size: number; id?: string; // redundant, since it's in the cell. actions?: JupyterActions; - name?: string; index?: number; // position of cell in the list of all cells; just used to optimize rendering and for no other reason. is_current?: boolean; is_selected?: boolean; @@ -62,6 +61,8 @@ interface Props { dragHandle?: React.JSX.Element; read_only?: boolean; isDragging?: boolean; + isPending?: boolean; + name?: string; } function areEqual(props: Props, nextProps: Props): boolean { @@ -91,7 +92,8 @@ function areEqual(props: Props, nextProps: Props): boolean { (nextProps.is_current || props.is_current)) || nextProps.dragHandle !== props.dragHandle || nextProps.read_only !== props.read_only || - nextProps.isDragging !== props.isDragging + nextProps.isDragging !== props.isDragging || + nextProps.isPending !== props.isPending ); } @@ -138,6 +140,7 @@ export const Cell: React.FC = React.memo((props: Props) => { computeServerId={props.computeServerId} setShowAICellGen={setShowAICellGen} dragHandle={props.dragHandle} + isPending={props.isPending} /> ); } diff --git a/src/packages/frontend/jupyter/main.tsx b/src/packages/frontend/jupyter/main.tsx index 819b519e69..aca7b02f86 100644 --- a/src/packages/frontend/jupyter/main.tsx +++ b/src/packages/frontend/jupyter/main.tsx @@ -183,6 +183,10 @@ export const JupyterEditor: React.FC = React.memo((props: Props) => { name, "check_select_kernel_init", ]); + const pendingCells: undefined | immutable.Set = useRedux([ + name, + "pendingCells", + ]); const computeServerId = path ? useTypedRedux({ project_id }, "compute_server_ids")?.get(syncdbPath(path)) @@ -318,6 +322,7 @@ export const JupyterEditor: React.FC = React.memo((props: Props) => { use_windowed_list={useWindowedListRef.current} llmTools={llmTools} computeServerId={computeServerId} + pendingCells={pendingCells} /> ); } @@ -451,7 +456,10 @@ export const JupyterEditor: React.FC = React.memo((props: Props) => { overflowY: "hidden", }} > - + {!read_only && } {render_error()} {render_modals()} diff --git a/src/packages/frontend/jupyter/prompt/input.tsx b/src/packages/frontend/jupyter/prompt/input.tsx index fb055aa8f5..7993720633 100644 --- a/src/packages/frontend/jupyter/prompt/input.tsx +++ b/src/packages/frontend/jupyter/prompt/input.tsx @@ -13,7 +13,6 @@ src/packages/frontend/frame-editors/whiteboard-editor/elements/code/input-prompt */ import React from "react"; - import { Icon } from "@cocalc/frontend/components/icon"; import { TimeAgo } from "@cocalc/frontend/components/time-ago"; import { Tip } from "@cocalc/frontend/components/tip"; diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index 2c8975435a..29f931273e 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -1049,7 +1049,12 @@ export abstract class JupyterActions extends Actions { this.save_asap(); }; + protected clearRunQueue() { + // implemented in frontend browser actions + } + clear_all_cell_run_state = (): void => { + this.clearRunQueue(); const { store } = this; if (!store) { return; diff --git a/src/packages/jupyter/redux/store.ts b/src/packages/jupyter/redux/store.ts index 1d62316bdf..e611b59eec 100644 --- a/src/packages/jupyter/redux/store.ts +++ b/src/packages/jupyter/redux/store.ts @@ -103,6 +103,11 @@ export interface JupyterStoreState { // run progress = Percent (0-100) of runnable cells that have been run since the last // kernel restart. (Thus markdown and empty cells are excluded.) runProgress?: number; + + // cells that this particular client has queued up to run. This is + // only known to this client, goes away on browser refresh, and is used + // only visually for the user to see. + pendingCells: Set; } export const initial_jupyter_store_state: { From a95ed82c7725fb07e9413990733ccd979383e83a Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 28 Jul 2025 01:22:27 +0000 Subject: [PATCH 111/798] delete jupyter exec_state sync, since it's no longer possible --- src/packages/jupyter/redux/project-actions.ts | 52 ------------------- 1 file changed, 52 deletions(-) diff --git a/src/packages/jupyter/redux/project-actions.ts b/src/packages/jupyter/redux/project-actions.ts index 92d28879b6..c6abcde4ba 100644 --- a/src/packages/jupyter/redux/project-actions.ts +++ b/src/packages/jupyter/redux/project-actions.ts @@ -181,7 +181,6 @@ export class JupyterActions extends JupyterActions0 { dbg("initializing blob store"); await this.initBlobStore(); - this.sync_exec_state = debounce(this.sync_exec_state, 2000); this._throttled_ensure_positions_are_unique = debounce( this.ensure_positions_are_unique, 5000, @@ -318,7 +317,6 @@ export class JupyterActions extends JupyterActions0 { this.ensure_there_is_a_cell(); this._throttled_ensure_positions_are_unique(); - this.sync_exec_state(); }; // ensure_backend_kernel_setup ensures that we have a connection @@ -562,56 +560,6 @@ export class JupyterActions extends JupyterActions0 { } } - // Ensure that the cells listed as running *are* exactly the - // ones actually running or queued up to run. - sync_exec_state = () => { - // sync_exec_state is debounced, so it is *expected* to get called - // after actions have been closed. - if (this.store == null || this._state !== "ready") { - // not initialized, so we better not - // mess with cell state (that is somebody else's responsibility). - return; - } - - const dbg = this.dbg("sync_exec_state"); - let change = false; - const cells = this.store.get("cells"); - // First verify that all actual cells that are said to be running - // (according to the store) are in fact running. - if (cells != null) { - cells.forEach((cell, id) => { - const state = cell.get("state"); - if ( - state != null && - state != "done" && - state != "start" && // regarding "start", see https://github.com/sagemathinc/cocalc/issues/5467 - !this._running_cells?.[id] - ) { - dbg(`set cell ${id} with state "${state}" to done`); - this._set({ type: "cell", id, state: "done" }, false); - change = true; - } - }); - } - if (this._running_cells != null) { - const cells = this.store.get("cells"); - // Next verify that every cell actually running is still in the document - // and listed as running. TimeTravel, deleting cells, etc., can - // certainly lead to this being necessary. - for (const id in this._running_cells) { - const state = cells.getIn([id, "state"]); - if (state == null || state === "done") { - // cell no longer exists or isn't in a running state - dbg(`tell kernel to not run ${id}`); - this._cancel_run(id); - } - } - } - if (change) { - return this._sync(); - } - }; - _cancel_run = (id: any) => { const dbg = this.dbg(`_cancel_run ${id}`); // All these checks are so we only cancel if it is actually running From 65ffcb0b80e05b46b61b424225ebc02d0b4c0be2 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 28 Jul 2025 01:38:54 +0000 Subject: [PATCH 112/798] jupyter: fix opening timetravel --- .../frame-editors/code-editor/actions.ts | 95 ++++++++++--------- .../frame-editors/generic/syncstring-fake.ts | 2 + .../frame-editors/jupyter-editor/actions.ts | 7 +- .../time-travel-editor/actions.ts | 7 +- 4 files changed, 63 insertions(+), 48 deletions(-) diff --git a/src/packages/frontend/frame-editors/code-editor/actions.ts b/src/packages/frontend/frame-editors/code-editor/actions.ts index 8a7eb2c93e..e0adf71f0e 100644 --- a/src/packages/frontend/frame-editors/code-editor/actions.ts +++ b/src/packages/frontend/frame-editors/code-editor/actions.ts @@ -299,55 +299,58 @@ export class Actions< } protected _init_syncstring(): void { - if (this.doctype == "none") { - this._syncstring = syncstring({ - project_id: this.project_id, - path: this.path, - cursors: !this.disable_cursors, - before_change_hook: () => this.set_syncstring_to_codemirror(), - after_change_hook: () => this.set_codemirror_to_syncstring(), - fake: true, - patch_interval: 500, - }) as SyncString; - } else if (this.doctype == "syncstring") { - this._syncstring = syncstring2({ - project_id: this.project_id, - path: this.path, - cursors: !this.disable_cursors, - }); - } else if (this.doctype == "syncdb") { - if ( - this.primary_keys == null || - this.primary_keys.length == null || - this.primary_keys.length <= 0 - ) { - throw Error("primary_keys must be array of positive length"); - } - this._syncstring = syncdb2({ - project_id: this.project_id, - path: this.path, - primary_keys: this.primary_keys, - string_cols: this.string_cols, - cursors: !this.disable_cursors, - }); - if (this.searchEmbeddings != null) { - if (!this.primary_keys.includes(this.searchEmbeddings.primaryKey)) { - throw Error( - `search embedding primaryKey must be in ${JSON.stringify( - this.primary_keys, - )}`, - ); + if (this._syncstring == null) { + // this._syncstring wasn't set in derived class so we set it here + if (this.doctype == "none") { + this._syncstring = syncstring({ + project_id: this.project_id, + path: this.path, + cursors: !this.disable_cursors, + before_change_hook: () => this.set_syncstring_to_codemirror(), + after_change_hook: () => this.set_codemirror_to_syncstring(), + fake: true, + patch_interval: 500, + }) as SyncString; + } else if (this.doctype == "syncstring") { + this._syncstring = syncstring2({ + project_id: this.project_id, + path: this.path, + cursors: !this.disable_cursors, + }); + } else if (this.doctype == "syncdb") { + if ( + this.primary_keys == null || + this.primary_keys.length == null || + this.primary_keys.length <= 0 + ) { + throw Error("primary_keys must be array of positive length"); } - if (!this.string_cols.includes(this.searchEmbeddings.textColumn)) { - throw Error( - `search embedding textColumn must be in ${JSON.stringify( - this.string_cols, - )}`, - ); + this._syncstring = syncdb2({ + project_id: this.project_id, + path: this.path, + primary_keys: this.primary_keys, + string_cols: this.string_cols, + cursors: !this.disable_cursors, + }); + if (this.searchEmbeddings != null) { + if (!this.primary_keys.includes(this.searchEmbeddings.primaryKey)) { + throw Error( + `search embedding primaryKey must be in ${JSON.stringify( + this.primary_keys, + )}`, + ); + } + if (!this.string_cols.includes(this.searchEmbeddings.textColumn)) { + throw Error( + `search embedding textColumn must be in ${JSON.stringify( + this.string_cols, + )}`, + ); + } } + } else { + throw Error(`invalid doctype="${this.doctype}"`); } - } else { - throw Error(`invalid doctype="${this.doctype}"`); } this._syncstring.once("deleted", () => { diff --git a/src/packages/frontend/frame-editors/generic/syncstring-fake.ts b/src/packages/frontend/frame-editors/generic/syncstring-fake.ts index 3e5b4e87cf..f22ef04532 100644 --- a/src/packages/frontend/frame-editors/generic/syncstring-fake.ts +++ b/src/packages/frontend/frame-editors/generic/syncstring-fake.ts @@ -25,6 +25,8 @@ export class FakeSyncstring extends EventEmitter { this.emit("ready"); } + hasFullHistory = () => true; + close() {} from_str() {} diff --git a/src/packages/frontend/frame-editors/jupyter-editor/actions.ts b/src/packages/frontend/frame-editors/jupyter-editor/actions.ts index 4d5f394c2f..e15e797f68 100644 --- a/src/packages/frontend/frame-editors/jupyter-editor/actions.ts +++ b/src/packages/frontend/frame-editors/jupyter-editor/actions.ts @@ -42,8 +42,13 @@ export class JupyterEditorActions extends BaseActions { return { type: "jupyter_cell_notebook" }; } - _init2(): void { + protected _init_syncstring(): void { this.create_jupyter_actions(); + this._syncstring = this.jupyter_actions.syncdb; + super._init_syncstring(); + } + + _init2(): void { this.init_new_frame(); this.init_changes_state(); diff --git a/src/packages/frontend/frame-editors/time-travel-editor/actions.ts b/src/packages/frontend/frame-editors/time-travel-editor/actions.ts index 2ef79fc2c6..cf13516678 100644 --- a/src/packages/frontend/frame-editors/time-travel-editor/actions.ts +++ b/src/packages/frontend/frame-editors/time-travel-editor/actions.ts @@ -151,7 +151,12 @@ export class TimeTravelActions extends CodeEditorActions { } this.syncdoc = mainFileActions._syncstring; - if (this.syncdoc == null || this.syncdoc.get_state() == "closed") { + if ( + this.syncdoc == null || + this.syncdoc.get_state() == "closed" || + // @ts-ignore + this.syncdoc.is_fake + ) { return; } if (this.syncdoc.get_state() != "ready") { From 0afbd270f458bd436b4e201d024b018c77554772 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 28 Jul 2025 02:59:24 +0000 Subject: [PATCH 113/798] disabling/deleting a lot of jupyter backend code that will no longer be needed --- .../frontend/jupyter/browser-actions.ts | 4 +- src/packages/jupyter/control.ts | 1 + .../jupyter/execute/output-handler.ts | 2 + src/packages/jupyter/redux/actions.ts | 66 +--------------- src/packages/jupyter/redux/project-actions.ts | 75 +------------------ src/packages/project/conat/api/index.ts | 5 ++ src/packages/project/conat/open-files.ts | 3 + 7 files changed, 21 insertions(+), 135 deletions(-) diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index a4475db41d..c3fe57d307 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -1541,6 +1541,9 @@ export class JupyterActions extends JupyterActions0 { private runQueue: any[] = []; private runningNow = false; runCells = async (ids: string[], opts: { noHalt?: boolean } = {}) => { + if (this.store?.get("read_only")) { + return; + } if (this.runningNow) { this.runQueue.push([ids, opts]); this.addPendingCells(ids); @@ -1619,7 +1622,6 @@ export class JupyterActions extends JupyterActions0 { handler.process(mesg); } } - console.log("exited the runner loop"); handler?.done(); this.save_asap(); } catch (err) { diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts index b581dd9a1f..6072920581 100644 --- a/src/packages/jupyter/control.ts +++ b/src/packages/jupyter/control.ts @@ -84,6 +84,7 @@ export async function jupyterRun({ path, cells, noHalt }: RunOptions) { logger.debug("jupyterRun: running"); async function* run() { for (const cell of cells) { + actions.ensure_backend_kernel_setup(); const output = actions.jupyter_kernel.execute_code({ halt_on_error: !noHalt, code: cell.input, diff --git a/src/packages/jupyter/execute/output-handler.ts b/src/packages/jupyter/execute/output-handler.ts index 47974f325a..94d5ae0cd0 100644 --- a/src/packages/jupyter/execute/output-handler.ts +++ b/src/packages/jupyter/execute/output-handler.ts @@ -99,6 +99,8 @@ export class OutputHandler extends EventEmitter { const { cell } = this._opts; cell.output = null; cell.exec_count = null; + // running a cell always de-collapses it: + cell.collapsed = false; cell.state = "run"; cell.start = null; cell.end = null; diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index 29f931273e..8cc0eac611 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -69,7 +69,6 @@ export abstract class JupyterActions extends Actions { public is_compute_server?: boolean; readonly path: string; readonly project_id: string; - private _last_start?: number; public jupyter_kernel?: JupyterKernelInterface; private last_cursor_move_time: Date = new Date(0); private _cursor_locs?: any; @@ -526,7 +525,6 @@ export abstract class JupyterActions extends Actions { } } - this.onCellChange(id, new_cell, old_cell); this.store.emit("cell_change", id, new_cell, old_cell); return cell_list_needs_recompute; @@ -698,11 +696,6 @@ export abstract class JupyterActions extends Actions { // things in project, browser, etc. } - protected onCellChange(_id: string, _new_cell: any, _old_cell: any) { - // no-op in base class. This is a hook though - // for potentially doing things when any cell changes. - } - ensure_backend_kernel_setup() { // nontrivial in the project, but not in client or here. } @@ -741,7 +734,6 @@ export abstract class JupyterActions extends Actions { } } } - //@dbg("_set")("obj=#{misc.to_json(obj)}") this.syncdb.set(obj); if (save) { this.syncdb.commit(); @@ -957,61 +949,11 @@ export abstract class JupyterActions extends Actions { } public run_code_cell( - id: string, - save: boolean = true, - no_halt: boolean = false, + _id: string, + _save: boolean = true, + _no_halt: boolean = false, ): void { - const cell = this.store.getIn(["cells", id]); - if (cell == null) { - // it is trivial to run a cell that does not exist -- nothing needs to be done. - return; - } - const kernel = this.store.get("kernel"); - if (kernel == null || kernel === "") { - // just in case, we clear any "running" indicators - this._set({ type: "cell", id, state: "done" }); - // don't attempt to run a code-cell if there is no kernel defined - this.set_error( - "No kernel set for running cells. Therefore it is not possible to run a code cell. You have to select a kernel!", - ); - return; - } - - if (cell.get("state", "done") != "done") { - // already running -- stop it first somehow if you want to run it again... - return; - } - - // We mark the start timestamp uniquely, so that the backend can sort - // multiple cells with a simultaneous time to start request. - - let start: number = this._client.server_time().valueOf(); - if (this._last_start != null && start <= this._last_start) { - start = this._last_start + 1; - } - this._last_start = start; - this.set_jupyter_metadata(id, "outputs_hidden", undefined, false); - - this._set( - { - type: "cell", - id, - state: "start", - start, - end: null, - // time last evaluation took - last: - cell.get("start") != null && cell.get("end") != null - ? cell.get("end") - cell.get("start") - : cell.get("last"), - output: null, - exec_count: null, - collapsed: null, - no_halt: no_halt ? no_halt : null, - }, - save, - ); - this.set_trust_notebook(true, save); + console.log("run_code_cell: deprecated"); } clear_cell = (id: string, save = true) => { diff --git a/src/packages/jupyter/redux/project-actions.ts b/src/packages/jupyter/redux/project-actions.ts index c6abcde4ba..7cf4d0085c 100644 --- a/src/packages/jupyter/redux/project-actions.ts +++ b/src/packages/jupyter/redux/project-actions.ts @@ -73,25 +73,7 @@ export class JupyterActions extends JupyterActions0 { save: boolean = true, no_halt: boolean = false, ): void { - if (this.store.get("read_only")) { - return; - } - const cell = this.store.getIn(["cells", id]); - if (cell == null) { - // it is trivial to run a cell that does not exist -- nothing needs to be done. - return; - } - const cell_type = cell.get("cell_type", "code"); - if (cell_type == "code") { - // when the backend is running code, just don't worry about - // trying to parse things like "foo?" out. We can't do - // it without CodeMirror, and it isn't worth it for that - // application. - this.run_code_cell(id, save, no_halt); - } - if (save) { - this.save_asap(); - } + console.log("run_cell: DEPRECATED"); } private set_backend_state(backend_state: BackendState): void { @@ -389,19 +371,6 @@ export class JupyterActions extends JupyterActions0 { this.set_kernel_error(error); }); - // Since we just made a new kernel, clearly no cells are running on the backend: - this._running_cells = {}; - - const toStart: string[] = []; - this.store?.get_cell_list().forEach((id) => { - if (this.store.getIn(["cells", id, "state"]) == "start") { - toStart.push(id); - } - }); - - dbg("clear cell run state"); - this.clear_all_cell_run_state(); - this.restartKernelOnClose = () => { // When the kernel closes, make sure a new kernel gets setup. if (this.store == null || this._state !== "ready") { @@ -426,11 +395,8 @@ export class JupyterActions extends JupyterActions0 { case "off": case "closed": // things went wrong. - this._running_cells = {}; - this.clear_all_cell_run_state(); this.set_backend_state("ready"); this.jupyter_kernel?.close(); - this.running_manager_run_cell_process_queue = false; delete this.jupyter_kernel; return; case "spawning": @@ -446,15 +412,6 @@ export class JupyterActions extends JupyterActions0 { this.handle_all_cell_attachments(); dbg("ready"); this.set_backend_state("ready"); - - // Run cells that the user explicitly set to be running before the - // kernel actually had finished starting up. - // This must be done after the state is ready. - if (toStart.length > 0) { - for (const id of toStart) { - this.run_cell(id); - } - } }; set_connection_file = () => { @@ -510,34 +467,6 @@ export class JupyterActions extends JupyterActions0 { await this.syncdb.wait(is_running, 60); } - // onCellChange is called after a cell change has been - // incorporated into the store after the syncdb change event. - // - If we are responsible for running cells, then it ensures - // that cell gets computed. - // - We also handle attachments for markdown cells. - protected onCellChange(id: string, new_cell: any, old_cell: any) { - const dbg = this.dbg(`onCellChange(id='${id}')`); - dbg(); - // this logging could be expensive due to toJS, so only uncomment - // if really needed - // dbg("new_cell=", new_cell?.toJS(), "old_cell", old_cell?.toJS()); - - if ( - new_cell?.get("state") === "start" && - old_cell?.get("state") !== "start" - ) { - this.manager_run_cell_enqueue(id); - // attachments below only happen for markdown cells, which don't get run, - // we can return here: - return; - } - - const attachments = new_cell?.get("attachments"); - if (attachments != null && attachments !== old_cell?.get("attachments")) { - this.handle_cell_attachments(new_cell); - } - } - protected __syncdb_change_post_hook(doInit: boolean) { if (doInit) { // Since just opening the actions in the project, definitely the kernel @@ -704,6 +633,8 @@ export class JupyterActions extends JupyterActions0 { manager_run_cell = (id: string) => { const dbg = this.dbg(`manager_run_cell(id='${id}')`); + console.log("manager_run_cell: DEPRECATED"); + return; dbg(JSON.stringify(misc.keys(this._running_cells))); if (this._running_cells == null) { diff --git a/src/packages/project/conat/api/index.ts b/src/packages/project/conat/api/index.ts index 7ce9561c64..a523d594d0 100644 --- a/src/packages/project/conat/api/index.ts +++ b/src/packages/project/conat/api/index.ts @@ -58,6 +58,7 @@ import { close as closeListings } from "@cocalc/project/conat/listings"; import { project_id } from "@cocalc/project/data"; import { close as closeFilesRead } from "@cocalc/project/conat/files/read"; import { close as closeFilesWrite } from "@cocalc/project/conat/files/write"; +import { close as closeJupyter } from "@cocalc/project/conat/jupyter"; import { getLogger } from "@cocalc/project/logger"; const logger = getLogger("conat:api"); @@ -109,6 +110,10 @@ async function handleMessage(api, subject, mesg) { closeListings(); await mesg.respond({ status: "terminated", service }); return; + } else if (service == "jupyter") { + closeJupyter(); + await mesg.respond({ status: "terminated", service }); + return; } else if (service == "files:read") { await closeFilesRead(); await mesg.respond({ status: "terminated", service }); diff --git a/src/packages/project/conat/open-files.ts b/src/packages/project/conat/open-files.ts index 11dab2fe09..1b93b8c3e5 100644 --- a/src/packages/project/conat/open-files.ts +++ b/src/packages/project/conat/open-files.ts @@ -227,6 +227,8 @@ async function handleChange({ doctype, id, }: OpenFileEntry & { id?: number }) { + // DEPRECATED! + return; if (!hasBackendState(path)) { return; } @@ -262,6 +264,7 @@ async function handleChange({ } } + // @ts-ignore if (time != null && time >= getCutoff()) { if (!isOpenHere) { logger.debug("handleChange: opening", { path }); From 9924e931c7d5c684c3766f75da403a57ca16664f Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 28 Jul 2025 04:20:05 +0000 Subject: [PATCH 114/798] ts --- src/packages/jupyter/redux/project-actions.ts | 160 +----------------- 1 file changed, 4 insertions(+), 156 deletions(-) diff --git a/src/packages/jupyter/redux/project-actions.ts b/src/packages/jupyter/redux/project-actions.ts index 7cf4d0085c..742f4d6315 100644 --- a/src/packages/jupyter/redux/project-actions.ts +++ b/src/packages/jupyter/redux/project-actions.ts @@ -35,9 +35,6 @@ import { type DKV, dkv } from "@cocalc/conat/sync/dkv"; import { computeServerManager } from "@cocalc/conat/compute/manager"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -// see https://github.com/sagemathinc/cocalc/issues/8060 -const MAX_OUTPUT_SAVE_DELAY = 30000; - // refuse to open an ipynb that is bigger than this: const MAX_SIZE_IPYNB_MB = 150; @@ -69,9 +66,9 @@ export class JupyterActions extends JupyterActions0 { // } public run_cell( - id: string, - save: boolean = true, - no_halt: boolean = false, + _id: string, + _save: boolean = true, + _no_halt: boolean = false, ): void { console.log("run_cell: DEPRECATED"); } @@ -631,157 +628,8 @@ export class JupyterActions extends JupyterActions0 { return handler; } - manager_run_cell = (id: string) => { - const dbg = this.dbg(`manager_run_cell(id='${id}')`); + manager_run_cell = (_id: string) => { console.log("manager_run_cell: DEPRECATED"); - return; - dbg(JSON.stringify(misc.keys(this._running_cells))); - - if (this._running_cells == null) { - this._running_cells = {}; - } - - if (this._running_cells[id]) { - dbg("cell already queued to run in kernel"); - return; - } - - // It's important to set this._running_cells[id] to be true so that - // sync_exec_state doesn't declare this cell done. The kernel identity - // will get set properly below in case it changes. - this._running_cells[id] = this.jupyter_kernel?.identity ?? "none"; - - const orig_cell = this.store.get("cells").get(id); - if (orig_cell == null) { - // nothing to do -- cell deleted - return; - } - - let input: string | undefined = orig_cell.get("input", ""); - if (input == null) { - input = ""; - } else { - input = input.trim(); - } - - const halt_on_error: boolean = !orig_cell.get("no_halt", false); - - if (this.jupyter_kernel == null) { - throw Error("bug -- this is guaranteed by the above"); - } - this._running_cells[id] = this.jupyter_kernel.identity; - - const cell: any = { - id, - type: "cell", - kernel: this.store.get("kernel"), - }; - - dbg(`using max_output_length=${this.store.get("max_output_length")}`); - const handler = this._output_handler(cell); - - // exponentiallyThrottledSaved calls this.syncdb?.save, but - // it throttles the calls, and does so using exponential backoff - // up to MAX_OUTPUT_SAVE_DELAY milliseconds. Basically every - // time exponentiallyThrottledSaved is called it increases the - // interval used for throttling by multiplying saveThrottleMs by 1.3 - // until saveThrottleMs gets to MAX_OUTPUT_SAVE_DELAY. There is no - // need at all to do a trailing call, since other code handles that. - let saveThrottleMs = 1; - let lastCall = 0; - const exponentiallyThrottledSaved = () => { - const now = Date.now(); - if (now - lastCall < saveThrottleMs) { - return; - } - lastCall = now; - saveThrottleMs = Math.min(1.3 * saveThrottleMs, MAX_OUTPUT_SAVE_DELAY); - this.syncdb?.save(); - }; - - handler.on("change", (save) => { - if (!this.store.getIn(["cells", id])) { - // The cell was deleted, but we just got some output - // NOTE: client shouldn't allow deleting running or queued - // cells, but we still want to do something useful/sensible. - // We put cell back where it was with same input. - cell.input = orig_cell.get("input"); - cell.pos = orig_cell.get("pos"); - } - this.syncdb.set(cell); - // This is potentially very verbose -- don't due it unless - // doing low level debugging: - //dbg(`change (save=${save}): cell='${JSON.stringify(cell)}'`); - if (save) { - exponentiallyThrottledSaved(); - } - }); - - handler.once("done", () => { - dbg("handler is done"); - this.store.removeListener("cell_change", cell_change); - exec.close(); - if (this._running_cells != null) { - delete this._running_cells[id]; - } - this.syncdb?.save(); - setTimeout(() => this.syncdb?.save(), 100); - }); - - if (this.jupyter_kernel == null) { - handler.error("Unable to start Jupyter"); - return; - } - - const get_password = (): string => { - if (this.jupyter_kernel == null) { - dbg("get_password", id, "no kernel"); - return ""; - } - const password = this.jupyter_kernel.store.get(id); - dbg("get_password", id, password); - this.jupyter_kernel.store.delete(id); - return password; - }; - - // This is used only for stdin right now. - const cell_change = (cell_id, new_cell) => { - if (id === cell_id) { - dbg("cell_change"); - handler.cell_changed(new_cell, get_password); - } - }; - this.store.on("cell_change", cell_change); - - const exec = this.jupyter_kernel.execute_code({ - code: input, - id, - stdin: handler.stdin, - halt_on_error, - }); - - exec.on("output", (mesg) => { - // uncomment only for specific low level debugging -- see https://github.com/sagemathinc/cocalc/issues/7022 - // dbg(`got mesg='${JSON.stringify(mesg)}'`); // !!!☡ ☡ ☡ -- EXTREME DANGER ☡ ☡ ☡ !!!! - if (mesg.content?.transient?.display_id != null) { - // See https://github.com/sagemathinc/cocalc/issues/2132 - // We find any other outputs in the document with - // the same transient.display_id, and set their output to - // this mesg's output. - this.handleTransientUpdate(mesg); - } - if (mesg.content.execution_state === "idle") { - this.store.removeListener("cell_change", cell_change); - return; - } - - handler.process(mesg); - }); - - exec.on("error", (err) => { - dbg(`got error='${err}'`); - handler.error(err); - }); }; reset_more_output = (id: string) => { From b4f060503f9febcaf2592890caa0909e7dab87d1 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 28 Jul 2025 05:53:43 +0000 Subject: [PATCH 115/798] jupyter run: run all cells and also deleting old code --- .../jupyter-editor/cell-notebook/actions.ts | 37 ++- .../frontend/jupyter/browser-actions.ts | 38 +-- .../frontend/jupyter/cell-buttonbar.tsx | 2 +- .../jupyter/output-messages/ipywidget.tsx | 2 +- src/packages/jupyter/redux/actions.ts | 17 +- src/packages/jupyter/redux/project-actions.ts | 279 +----------------- 6 files changed, 40 insertions(+), 335 deletions(-) diff --git a/src/packages/frontend/frame-editors/jupyter-editor/cell-notebook/actions.ts b/src/packages/frontend/frame-editors/jupyter-editor/cell-notebook/actions.ts index 74cfc894fd..8158f8cb40 100644 --- a/src/packages/frontend/frame-editors/jupyter-editor/cell-notebook/actions.ts +++ b/src/packages/frontend/frame-editors/jupyter-editor/cell-notebook/actions.ts @@ -324,36 +324,41 @@ export class NotebookFrameActions { } } - public run_selected_cells(v?: string[]): void { + public run_selected_cells(ids?: string[]): void { this.save_input_editor(); - if (v === undefined) { - v = this.store.get_selected_cell_ids_list(); + if (ids == null) { + ids = this.store.get_selected_cell_ids_list(); } // for whatever reason, any running of a cell deselects // in official jupyter this.unselect_all_cells(); - for (const id of v) { - const save = id === v[v.length - 1]; // save only last one. - this.run_cell(id, save); - } + this.runCells(ids); + } + + run_cell(id: string) { + this.runCells([id]); } // This is here since it depends on knowing the edit state // of markdown cells. - public run_cell(id: string, save: boolean = true): void { - const type = this.jupyter_actions.store.get_cell_type(id); - if (type === "markdown") { - if (this.store.get("md_edit_ids", Set()).contains(id)) { - this.set_md_cell_not_editing(id); + public runCells(ids: string[]): void { + const v: string[] = []; + for (const id of ids) { + const type = this.jupyter_actions.store.get_cell_type(id); + if (type === "markdown") { + if (this.store.get("md_edit_ids", Set()).contains(id)) { + this.set_md_cell_not_editing(id); + } + } else if (type === "code") { + v.push(id); } - return; + // running is a no-op for raw cells. } - if (type === "code") { - this.jupyter_actions.run_cell(id, save); + if (v.length > 0) { + this.jupyter_actions.runCells(v); } - // running is a no-op for raw cells. } /*** diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index c3fe57d307..8dcd87cdcc 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -216,33 +216,6 @@ export class JupyterActions extends JupyterActions0 { } }; - public run_cell( - id: string, - save: boolean = true, - noHalt: boolean = false, - ): void { - if (this.store.get("read_only")) return; - const cell = this.store.getIn(["cells", id]); - if (cell == null) { - // it is trivial to run a cell that does not exist -- nothing needs to be done. - return; - } - - const cell_type = cell.get("cell_type", "code"); - if (cell_type == "code") { - const code = this.get_cell_input(id).trim(); - if (!code) { - this.clear_cell(id, save); - return; - } - this.runCells([id], { noHalt }); - //this.run_code_cell(id, save, no_halt); - if (save) { - this.save_asap(); - } - } - } - private async api_call_formatter( str: string, config: FormatterConfig, @@ -1540,7 +1513,7 @@ export class JupyterActions extends JupyterActions0 { private jupyterClient?; private runQueue: any[] = []; private runningNow = false; - runCells = async (ids: string[], opts: { noHalt?: boolean } = {}) => { + async runCells(ids: string[], opts: { noHalt?: boolean } = {}) { if (this.store?.get("read_only")) { return; } @@ -1571,6 +1544,9 @@ export class JupyterActions extends JupyterActions0 { for (const id of ids) { const cell = this.store.getIn(["cells", id])?.toJS(); + if (cell?.cell_type != "code") { + continue; + } if (!cell?.input?.trim()) { // nothing to do continue; @@ -1591,7 +1567,7 @@ export class JupyterActions extends JupyterActions0 { } cells.push(cell); } - this.addPendingCells(ids); + this.addPendingCells(cells.map(({ id }) => id)); // ensures cells run in order: cells.sort(field_cmp("pos")); @@ -1625,7 +1601,7 @@ export class JupyterActions extends JupyterActions0 { handler?.done(); this.save_asap(); } catch (err) { - console.log("runCells", err); + console.warn("runCells", err); } finally { this.runningNow = false; if (this.runQueue.length > 0) { @@ -1633,5 +1609,5 @@ export class JupyterActions extends JupyterActions0 { this.runCells(ids, opts); } } - }; + } } diff --git a/src/packages/frontend/jupyter/cell-buttonbar.tsx b/src/packages/frontend/jupyter/cell-buttonbar.tsx index 3d6c91afd4..47afd9f011 100644 --- a/src/packages/frontend/jupyter/cell-buttonbar.tsx +++ b/src/packages/frontend/jupyter/cell-buttonbar.tsx @@ -103,7 +103,7 @@ export const CellButtonBar: React.FC = React.memo( tooltip: "Run this cell", label: "Run", icon: "step-forward", - onClick: () => actions?.run_cell(id), + onClick: () => actions?.runCells([id]), }; } } diff --git a/src/packages/frontend/jupyter/output-messages/ipywidget.tsx b/src/packages/frontend/jupyter/output-messages/ipywidget.tsx index b0c1a6f50b..d0815ffcad 100644 --- a/src/packages/frontend/jupyter/output-messages/ipywidget.tsx +++ b/src/packages/frontend/jupyter/output-messages/ipywidget.tsx @@ -146,7 +146,7 @@ ax.plot(x, y) - ); - case "clear": - return ( - - ); - } - } - function render_currently_selected(): React.JSX.Element | undefined { if (props.listing.length === 0) { return; @@ -235,7 +177,6 @@ export function ActionBar(props: Props) { )} - {render_select_entire_directory()} ); } diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index 41bfcd0313..e200d507d2 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -19,7 +19,6 @@ import { ShallowTypedMap } from "@cocalc/frontend/app-framework/ShallowTypedMap" import { A, ActivityDisplay, - ErrorDisplay, Loading, SettingBox, } from "@cocalc/frontend/components"; @@ -38,7 +37,6 @@ import { ProjectActions } from "@cocalc/frontend/project_store"; import { ProjectMap, ProjectStatus } from "@cocalc/frontend/todo-types"; import AskNewFilename from "../ask-filename"; import { useProjectContext } from "../context"; -import { AccessErrors } from "./access-errors"; import { ActionBar } from "./action-bar"; import { ActionBox } from "./action-box"; import { FileListing } from "./file-listing"; @@ -48,6 +46,7 @@ import { NewButton } from "./new-button"; import { PathNavigator } from "./path-navigator"; import { SearchBar } from "./search-bar"; import ExplorerTour from "./tour/tour"; +import ShowError from "@cocalc/frontend/components/error"; export type Configuration = ShallowTypedMap<{ main: MainConfiguration }>; @@ -79,7 +78,6 @@ interface ReduxProps { current_path: string; history_path: string; activity?: object; - page_number: number; file_action?: | "compress" | "delete" @@ -162,7 +160,6 @@ const Explorer0 = rclass( current_path: rtypes.string, history_path: rtypes.string, activity: rtypes.object, - page_number: rtypes.number.isRequired, file_action: rtypes.string, file_search: rtypes.string, show_hidden: rtypes.bool, @@ -186,7 +183,6 @@ const Explorer0 = rclass( }; static defaultProps = { - page_number: 0, file_search: "", new_name: "", redux, @@ -225,20 +221,6 @@ const Explorer0 = rclass( } }; - previous_page = () => { - if (this.props.page_number > 0) { - this.props.actions.setState({ - page_number: this.props.page_number - 1, - }); - } - }; - - next_page = () => { - this.props.actions.setState({ - page_number: this.props.page_number + 1, - }); - }; - create_file = (ext, switch_over) => { if (switch_over == undefined) { switch_over = true; @@ -265,7 +247,7 @@ const Explorer0 = rclass( current_path: this.props.current_path, switch_over, }); - this.props.actions.setState({ file_search: "", page_number: 0 }); + this.props.actions.setState({ file_search: "" }); }; create_folder = (switch_over = true): void => { @@ -274,151 +256,9 @@ const Explorer0 = rclass( current_path: this.props.current_path, switch_over, }); - this.props.actions.setState({ file_search: "", page_number: 0 }); + this.props.actions.setState({ file_search: "" }); }; - render_files_action_box() { - return ( - - - - ); - } - - render_library() { - return ( - - - - Library{" "} - - (help...) - - - } - close={() => this.props.actions.toggle_library(false)} - > - this.props.actions.toggle_library(false)} - /> - - - - ); - } - - render_files_actions(project_is_running) { - return ( - - ); - } - - render_new_file() { - return ( -
- -
- ); - } - - render_activity() { - return ( - this.props.actions.clear_all_activity()} - style={{ top: "100px" }} - /> - ); - } - - render_error() { - if (this.props.error) { - return ( - this.props.actions.setState({ error: "" })} - /> - ); - } - } - - render_access_error() { - return ; - } - - render_file_listing() { - return ( - - - - ); - } - file_listing_page_size() { return ( this.props.other_settings && @@ -426,135 +266,6 @@ const Explorer0 = rclass( ); } - render_control_row(): React.JSX.Element { - return ( -
-
-
- -
- -
-
- {!!this.props.compute_server_id && ( -
- -
- )} -
- {!IS_MOBILE && ( -
- {this.render_new_file()} -
- )} - {!IS_MOBILE && ( - - )} -
- -
-
- ); - } - - render_project_files_buttons(): React.JSX.Element { - return ( -
- -
- ); - } - - render_custom_software_reset() { - if (!this.props.show_custom_software_reset) { - return undefined; - } - // also don't show this box, if any files are selected - if (this.props.checked_files.size > 0) { - return undefined; - } - return ( - - ); - } - render() { let project_is_running: boolean, project_state: ProjectStatus | undefined; @@ -599,9 +310,109 @@ const Explorer0 = rclass( padding: "2px 2px 0 2px", }} > - {this.render_error()} - {this.render_activity()} - {this.render_control_row()} + this.props.actions.setState({ error })} + /> + this.props.actions.clear_all_activity()} + style={{ top: "100px" }} + /> +
+
+
+ +
+ +
+
+ {!!this.props.compute_server_id && ( +
+ +
+ )} +
+ {!IS_MOBILE && ( +
+
+ +
+
+ )} + {!IS_MOBILE && ( +
+ +
+ )} +
+ +
+
+ {this.props.ext_selection != null && ( )} @@ -613,20 +424,101 @@ const Explorer0 = rclass( minWidth: "20em", }} > - {this.render_files_actions(project_is_running)} + + +
+
- {this.render_project_files_buttons()} - {project_is_running - ? this.render_custom_software_reset() - : undefined} - - {this.props.show_library ? this.render_library() : undefined} + {project_is_running && + this.props.show_custom_software_reset && + this.props.checked_files.size == 0 && ( + + )} + + {this.props.show_library && ( + + + + Library{" "} + + (help...) + + + } + close={() => this.props.actions.toggle_library(false)} + > + this.props.actions.toggle_library(false)} + /> + + + + )} {this.props.checked_files.size > 0 && this.props.file_action != undefined ? ( - {this.render_files_action_box()} + + + + + ) : undefined} @@ -640,7 +532,39 @@ const Explorer0 = rclass( padding: "0 5px 5px 5px", }} > - {this.render_file_listing()} + + + ; - current_path: string; - file_search: string; - actions: ProjectActions; - file_creation_error?: string; - create_file: (ext?: string, switch_over?: boolean) => void; - create_folder: (switch_over?: boolean) => void; - listingRef; - }, - ref: React.LegacyRef | undefined, - ) => { - return ( -
- -
- ); - }, -); diff --git a/src/packages/frontend/project/listing/use-files.ts b/src/packages/frontend/project/listing/use-files.ts index fa7dd3f9d8..6e925e817d 100644 --- a/src/packages/frontend/project/listing/use-files.ts +++ b/src/packages/frontend/project/listing/use-files.ts @@ -170,7 +170,6 @@ async function cacheNeighbors({ v.push(parent); } } - const t = Date.now(); const f = async (path: string) => { await ensureCached({ cacheId, fs, path }); }; @@ -178,5 +177,4 @@ async function cacheNeighbors({ // grab up to MAX_SUBDIR_CACHE missing listings in parallel v = v.slice(0, MAX_SUBDIR_CACHE); await Promise.all(v.map(f)); - console.log(Date.now() - t, v); } diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index d13880bd2a..285e6e4f06 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -1500,7 +1500,6 @@ export class ProjectActions extends Actions { this.setState({ current_path: path, history_path, - page_number: 0, most_recent_file_click: undefined, }); }; @@ -1552,7 +1551,6 @@ export class ProjectActions extends Actions { set_file_search(search): void { this.setState({ file_search: search, - page_number: 0, file_action: undefined, most_recent_file_click: undefined, create_file_alert: false, diff --git a/src/packages/frontend/project_store.ts b/src/packages/frontend/project_store.ts index 6499c91853..388a9fb74a 100644 --- a/src/packages/frontend/project_store.ts +++ b/src/packages/frontend/project_store.ts @@ -93,7 +93,6 @@ export interface ProjectStoreState { // Project Files activity: any; // immutable, active_file_sort: TypedMap<{ column_name: string; is_descending: boolean }>; - page_number: number; file_action?: string; // undefineds is meaningfully none here file_search?: string; show_hidden?: boolean; @@ -310,7 +309,6 @@ export class ProjectStore extends Store { // Project Files activity: undefined, - page_number: 0, checked_files: immutable.Set(), show_library: false, file_listing_scroll_top: undefined, From 397dde1ae1d9b47fba55baeaea0307a609550570 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 29 Jul 2025 00:19:45 +0000 Subject: [PATCH 122/798] move explorer keyboard handling to the explorer itself - fixes bug where if the filter looses focus you can't navigate or select - better approach anyways (more direct) --- .../frontend/project/explorer/explorer.tsx | 38 +++++++++++++++---- .../frontend/project/explorer/search-bar.tsx | 32 ---------------- 2 files changed, 30 insertions(+), 40 deletions(-) diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index e200d507d2..b3976a60d2 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -47,6 +47,14 @@ import { PathNavigator } from "./path-navigator"; import { SearchBar } from "./search-bar"; import ExplorerTour from "./tour/tour"; import ShowError from "@cocalc/frontend/components/error"; +import { join } from "path"; + +const FLEX_ROW_STYLE = { + display: "flex", + flexFlow: "row wrap", + justifyContent: "space-between", + alignItems: "stretch", +} as const; export type Configuration = ShallowTypedMap<{ main: MainConfiguration }>; @@ -105,6 +113,7 @@ interface ReduxProps { show_custom_software_reset?: boolean; explorerTour?: boolean; compute_server_id: number; + selected_file_index?: number; } interface State { @@ -178,6 +187,7 @@ const Explorer0 = rclass( show_custom_software_reset: rtypes.bool, explorerTour: rtypes.bool, compute_server_id: rtypes.number, + selected_file_index: rtypes.number, }, }; }; @@ -212,6 +222,26 @@ const Explorer0 = rclass( handle_files_key_down = (e): void => { if (e.key === "Shift") { this.setState({ shift_is_down: true }); + } else if (e.key == "ArrowUp") { + this.props.actions.decrement_selected_file_index(); + } else if (e.key == "ArrowDown") { + this.props.actions.increment_selected_file_index(); + } else if (e.key == "Enter") { + const x = + this.listingRef.current?.[this.props.selected_file_index ?? 0]; + if (x != null) { + const { isdir, name } = x; + const path = join(this.props.current_path, name); + if (isdir) { + this.props.actions.open_directory(path); + } else { + this.props.actions.open_file({ path, foreground: !e.ctrlKey }); + } + if (!e.ctrlKey) { + this.props.actions.set_file_search(""); + this.props.actions.clear_selected_file_index(); + } + } } }; @@ -292,13 +322,6 @@ const Explorer0 = rclass( project_is_running = false; } - const FLEX_ROW_STYLE = { - display: "flex", - flexFlow: "row wrap", - justifyContent: "space-between", - alignItems: "stretch", - }; - // be careful with adding height:'100%'. it could cause flex to miscalculate. see #3904 return (
@@ -400,7 +423,6 @@ const Explorer0 = rclass( file_creation_error={this.props.file_creation_error} create_file={this.create_file} create_folder={this.create_folder} - listingRef={this.listingRef} />
)} diff --git a/src/packages/frontend/project/explorer/search-bar.tsx b/src/packages/frontend/project/explorer/search-bar.tsx index ebe746a307..7070669954 100644 --- a/src/packages/frontend/project/explorer/search-bar.tsx +++ b/src/packages/frontend/project/explorer/search-bar.tsx @@ -14,7 +14,6 @@ import { useProjectContext } from "../context"; import { TERM_MODE_CHAR } from "./file-listing"; import { TerminalModeDisplay } from "@cocalc/frontend/project/explorer/file-listing/terminal-mode-display"; import { useTypedRedux } from "@cocalc/frontend/app-framework"; -import { join } from "path"; const HelpStyle = { wordWrap: "break-word", @@ -50,7 +49,6 @@ interface Props { file_creation_error?: string; disabled?: boolean; ext_selection?: string; - listingRef; } // Commands such as CD throw a setState error. @@ -65,14 +63,11 @@ export const SearchBar = memo( file_creation_error, disabled = false, ext_selection, - listingRef, }: Props) => { const intl = useIntl(); const { project_id } = useProjectContext(); const numDisplayedFiles = useTypedRedux({ project_id }, "numDisplayedFiles") ?? 0; - const selected_file_index = - useTypedRedux({ project_id }, "selected_file_index") ?? 0; // edit → run → edit // TODO use "state" to show a progress spinner while a command is running @@ -241,19 +236,6 @@ export const SearchBar = memo( if (value.startsWith(TERM_MODE_CHAR)) { const command = value.slice(1, value.length); execute_command(command); - } else if (listingRef.current?.[selected_file_index] != null) { - const { isdir, name } = listingRef.current[selected_file_index]; - const path = join(current_path, name); - if (isdir) { - actions.open_directory(path); - } else { - actions.open_file({ path, foreground: !ctrl_down }); - } - if (!ctrl_down) { - actions.set_file_search(""); - actions.clear_selected_file_index(); - } - return; } else if (file_search.length > 0 && shift_down) { // only create a file, if shift is pressed as well to avoid creating // jupyter notebooks (default file-type) by accident. @@ -266,18 +248,6 @@ export const SearchBar = memo( } } - function on_up_press(): void { - if (selected_file_index > 0) { - actions.decrement_selected_file_index(); - } - } - - function on_down_press(): void { - if (selected_file_index < numDisplayedFiles - 1) { - actions.increment_selected_file_index(); - } - } - function on_change(search: string): void { actions.zero_selected_file_index(); actions.set_file_search(search); @@ -302,8 +272,6 @@ export const SearchBar = memo( value={file_search} on_change={on_change} on_submit={search_submit} - on_up={on_up_press} - on_down={on_down_press} on_clear={on_clear} disabled={disabled || !!ext_selection} /> From ecc14de58934aafcf01bccd4f43773a1d5e50365 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 29 Jul 2025 00:25:03 +0000 Subject: [PATCH 123/798] explorer: support [*]-up arrow to move to parent directory - noticed this is missing and obviously expected (and trivial) --- src/packages/frontend/project/explorer/explorer.tsx | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index b3976a60d2..58857e1f8e 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -47,7 +47,7 @@ import { PathNavigator } from "./path-navigator"; import { SearchBar } from "./search-bar"; import ExplorerTour from "./tour/tour"; import ShowError from "@cocalc/frontend/components/error"; -import { join } from "path"; +import { dirname, join } from "path"; const FLEX_ROW_STYLE = { display: "flex", @@ -223,7 +223,12 @@ const Explorer0 = rclass( if (e.key === "Shift") { this.setState({ shift_is_down: true }); } else if (e.key == "ArrowUp") { - this.props.actions.decrement_selected_file_index(); + if (e.shiftKey || e.ctrlKey || e.metaKey) { + const path = dirname(this.props.current_path); + this.props.actions.open_directory(path == "." ? "" : path); + } else { + this.props.actions.decrement_selected_file_index(); + } } else if (e.key == "ArrowDown") { this.props.actions.increment_selected_file_index(); } else if (e.key == "Enter") { From 145ce9a615094e6b94b8c42da1f91728c0568e2c Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 29 Jul 2025 01:45:12 +0000 Subject: [PATCH 124/798] rewrite file explorer to be a functional component --- .../frontend/account/other-settings.tsx | 19 - .../frontend/custom-software/reset-bar.tsx | 10 +- .../frontend/project/explorer/action-bar.tsx | 6 +- .../frontend/project/explorer/action-box.tsx | 113 +-- .../frontend/project/explorer/explorer.tsx | 872 +++++++----------- .../explorer/file-listing/file-checkbox.tsx | 42 +- .../explorer/file-listing/file-listing.tsx | 30 +- .../explorer/file-listing/listing-header.tsx | 8 +- .../project/explorer/file-listing/utils.ts | 6 +- .../project/explorer/misc-side-buttons.tsx | 69 +- .../frontend/project/explorer/new-button.tsx | 22 +- .../project/explorer/path-navigator.tsx | 13 +- .../project/explorer/path-segment-link.tsx | 22 +- .../frontend/project/explorer/tour/tour.tsx | 5 +- .../project/listing/filter-listing.ts | 5 + 15 files changed, 520 insertions(+), 722 deletions(-) diff --git a/src/packages/frontend/account/other-settings.tsx b/src/packages/frontend/account/other-settings.tsx index bdf6fcb0ca..dba71ce509 100644 --- a/src/packages/frontend/account/other-settings.tsx +++ b/src/packages/frontend/account/other-settings.tsx @@ -320,24 +320,6 @@ export function OtherSettings(props: Readonly): React.JSX.Element { ); } - function render_page_size(): Rendered { - return ( - - on_change("page_size", n)} - min={1} - max={10000} - number={props.other_settings.get("page_size")} - /> - - ); - } - function render_no_free_warnings(): Rendered { let extra; if (!props.is_stripe_customer) { @@ -714,7 +696,6 @@ export function OtherSettings(props: Readonly): React.JSX.Element { {render_vertical_fixed_bar_options()} {render_new_filenames()} {render_default_file_sort()} - {render_page_size()} {render_standby_timeout()}
diff --git a/src/packages/frontend/custom-software/reset-bar.tsx b/src/packages/frontend/custom-software/reset-bar.tsx index 2442a38bbe..445b947900 100644 --- a/src/packages/frontend/custom-software/reset-bar.tsx +++ b/src/packages/frontend/custom-software/reset-bar.tsx @@ -6,15 +6,15 @@ import { Button as AntdButton, Card } from "antd"; import React from "react"; import { FormattedMessage, useIntl } from "react-intl"; - import { A, Icon } from "@cocalc/frontend/components"; import { labels } from "@cocalc/frontend/i18n"; import { ProjectMap } from "@cocalc/frontend/todo-types"; import { COLORS, SITE_NAME } from "@cocalc/util/theme"; - import { Available as AvailableFeatures } from "../project_configuration"; import { ComputeImages } from "./init"; import { props2img, RESET_ICON } from "./util"; +import { useTypedRedux } from "@cocalc/frontend/app-framework"; +import { type ProjectActions } from "@cocalc/frontend/project_store"; const doc_snap = "https://doc.cocalc.com/project-files.html#snapshots"; const doc_tt = "https://doc.cocalc.com/time-travel.html"; @@ -36,13 +36,13 @@ interface Props { project_id: string; images: ComputeImages; project_map?: ProjectMap; - actions: any; + actions: ProjectActions; available_features?: AvailableFeatures; - site_name?: string; } export const CustomSoftwareReset: React.FC = (props: Props) => { - const { actions, site_name } = props; + const { actions } = props; + const site_name = useTypedRedux("customize", "site_name"); const intl = useIntl(); diff --git a/src/packages/frontend/project/explorer/action-bar.tsx b/src/packages/frontend/project/explorer/action-bar.tsx index 7c7407e713..fa82fc17df 100644 --- a/src/packages/frontend/project/explorer/action-bar.tsx +++ b/src/packages/frontend/project/explorer/action-bar.tsx @@ -12,10 +12,10 @@ import { Button, ButtonToolbar } from "@cocalc/frontend/antd-bootstrap"; import { Gap, Icon } from "@cocalc/frontend/components"; import { useStudentProjectFunctionality } from "@cocalc/frontend/course"; import { CustomSoftwareInfo } from "@cocalc/frontend/custom-software/info-bar"; -import { ComputeImages } from "@cocalc/frontend/custom-software/init"; +import { type ComputeImages } from "@cocalc/frontend/custom-software/init"; import { IS_MOBILE } from "@cocalc/frontend/feature"; import { labels } from "@cocalc/frontend/i18n"; -import { file_actions, ProjectActions } from "@cocalc/frontend/project_store"; +import { file_actions, type ProjectActions } from "@cocalc/frontend/project_store"; import * as misc from "@cocalc/util/misc"; import { COLORS } from "@cocalc/util/theme"; import { useProjectContext } from "../context"; @@ -31,7 +31,7 @@ interface Props { checked_files: immutable.Set; listing: { name: string; isdir: boolean }[]; current_path?: string; - project_map?: immutable.Map; + project_map?; images?: ComputeImages; actions: ProjectActions; available_features?; diff --git a/src/packages/frontend/project/explorer/action-box.tsx b/src/packages/frontend/project/explorer/action-box.tsx index d52c5e9e18..b5a5df0f19 100644 --- a/src/packages/frontend/project/explorer/action-box.tsx +++ b/src/packages/frontend/project/explorer/action-box.tsx @@ -9,7 +9,6 @@ import { Button as AntdButton, Radio, Space } from "antd"; import * as immutable from "immutable"; import { useState } from "react"; import { useIntl } from "react-intl"; - import { Alert, Button, @@ -28,7 +27,6 @@ import { SelectProject } from "@cocalc/frontend/projects/select-project"; import ConfigureShare from "@cocalc/frontend/share/config"; import * as misc from "@cocalc/util/misc"; import { COLORS } from "@cocalc/util/theme"; -import { useProjectContext } from "../context"; import DirectorySelector from "../directory-selector"; import { in_snapshot_path } from "../utils"; import CreateArchive from "./create-archive"; @@ -56,13 +54,17 @@ interface ReactProps { file_map: object; actions: ProjectActions; displayed_listing?: object; - //new_name?: string; - name: string; } -export function ActionBox(props: ReactProps) { +export function ActionBox({ + checked_files, + file_action, + current_path, + project_id, + file_map, + actions, +}: ReactProps) { const intl = useIntl(); - const { project_id } = useProjectContext(); const runQuota = useRunQuota(project_id, null); const get_user_type: () => string = useRedux("account", "get_user_type"); const compute_server_id = useTypedRedux({ project_id }, "compute_server_id"); @@ -74,7 +76,6 @@ export function ActionBox(props: ReactProps) { const [copy_from_compute_server_to, set_copy_from_compute_server_to] = useState<"compute-server" | "project">("compute-server"); const [move_destination, set_move_destination] = useState(""); - //const [new_name, set_new_name] = useState(props.new_name ?? ""); const [show_different_project, set_show_different_project] = useState(false); const [overwrite_newer, set_overwrite_newer] = useState(); @@ -84,7 +85,7 @@ export function ActionBox(props: ReactProps) { ); function cancel_action(): void { - props.actions.set_file_action(); + actions.set_file_action(); } function action_key(e): void { @@ -93,7 +94,7 @@ export function ActionBox(props: ReactProps) { cancel_action(); break; case 13: - switch (props.file_action) { + switch (file_action) { case "move": submit_action_move(); break; @@ -107,7 +108,7 @@ export function ActionBox(props: ReactProps) { function render_selected_files_list(): React.JSX.Element { return ( 
-        {props.checked_files.toArray().map((name) => (
+        {checked_files.toArray().map((name) => (
           
{misc.path_split(name).tail}

         ))}
@@ -115,17 +116,17 @@ export function ActionBox(props: ReactProps) { } function delete_click(): void { - const paths = props.checked_files.toArray(); + const paths = checked_files.toArray(); for (const path of paths) { - props.actions.close_tab(path); + actions.close_tab(path); } - props.actions.delete_files({ paths }); - props.actions.set_file_action(); - props.actions.set_all_files_unchecked(); + actions.delete_files({ paths }); + actions.set_file_action(); + actions.set_all_files_unchecked(); } function render_delete_warning(): React.JSX.Element | undefined { - if (props.current_path === ".trash") { + if (current_path === ".trash") { return ( @@ -140,7 +141,7 @@ export function ActionBox(props: ReactProps) { } function render_delete(): React.JSX.Element | undefined { - const { size } = props.checked_files; + const { size } = checked_files; return (
@@ -161,7 +162,7 @@ export function ActionBox(props: ReactProps) { href="" onClick={(e) => { e.preventDefault(); - props.actions.open_directory(".snapshots"); + actions.open_directory(".snapshots"); }} > ~/.snapshots @@ -178,7 +179,7 @@ export function ActionBox(props: ReactProps) { Delete {size} {misc.plural(size, "Item")} @@ -190,16 +191,16 @@ export function ActionBox(props: ReactProps) { } function move_click(): void { - props.actions.move_files({ - src: props.checked_files.toArray(), + actions.move_files({ + src: checked_files.toArray(), dest: move_destination, }); - props.actions.set_file_action(); - props.actions.set_all_files_unchecked(); + actions.set_file_action(); + actions.set_all_files_unchecked(); } function valid_move_input(): boolean { - const src_path = misc.path_split(props.checked_files.first()).head; + const src_path = misc.path_split(checked_files.first()).head; let dest = move_destination.trim(); if (dest === src_path) { return false; @@ -210,11 +211,11 @@ export function ActionBox(props: ReactProps) { if (dest.charAt(dest.length - 1) === "/") { dest = dest.slice(0, dest.length - 1); } - return dest !== props.current_path; + return dest !== current_path; } function render_move(): React.JSX.Element { - const { size } = props.checked_files; + const { size } = checked_files; return (
@@ -243,9 +244,9 @@ export function ActionBox(props: ReactProps) { onSelect={(move_destination: string) => set_move_destination(move_destination) } - project_id={props.project_id} - startingPath={props.current_path} - isExcluded={(path) => props.checked_files.has(path)} + project_id={project_id} + startingPath={current_path} + isExcluded={(path) => checked_files.has(path)} style={{ width: "100%" }} bodyStyle={{ maxHeight: "250px" }} /> @@ -267,7 +268,7 @@ export function ActionBox(props: ReactProps) {

Target Project

set_copy_destination_project_id(copy_destination_project_id) @@ -279,8 +280,10 @@ export function ActionBox(props: ReactProps) { } } - function render_copy_different_project_options(): React.JSX.Element | undefined { - if (props.project_id !== copy_destination_project_id) { + function render_copy_different_project_options(): + | React.JSX.Element + | undefined { + if (project_id !== copy_destination_project_id) { return (
@@ -430,7 +433,7 @@ export function ActionBox(props: ReactProps) { } function render_copy(): React.JSX.Element { - const { size } = props.checked_files; + const { size } = checked_files; const signed_in = get_user_type() === "signed_in"; if (!signed_in) { return ( @@ -498,7 +501,7 @@ export function ActionBox(props: ReactProps) {
set_dest_compute_server_id(dest_compute_server_id) @@ -513,7 +516,7 @@ export function ActionBox(props: ReactProps) { set_copy_destination_directory(value) } key="copy_destination_directory" - startingPath={props.current_path} + startingPath={current_path} project_id={copy_destination_project_id} style={{ width: "100%" }} bodyStyle={{ maxHeight: "250px" }} @@ -540,18 +543,18 @@ export function ActionBox(props: ReactProps) { function render_share(): React.JSX.Element { // currently only works for a single selected file - const path: string = props.checked_files.first() ?? ""; + const path: string = checked_files.first() ?? ""; if (!path) { return <>; } - const public_data = props.file_map[misc.path_split(path).tail]; + const public_data = file_map[misc.path_split(path).tail]; if (public_data == undefined) { // directory listing not loaded yet... (will get re-rendered when loaded) return ; } return ( props.actions.set_public_path(path, opts)} + set_public_path={(opts) => actions.set_public_path(path, opts)} has_network_access={!!runQuota.network} /> ); } - function render_action_box(action: FileAction): React.JSX.Element | undefined { + function render_action_box( + action: FileAction, + ): React.JSX.Element | undefined { switch (action) { case "compress": return ; @@ -590,12 +595,12 @@ export function ActionBox(props: ReactProps) { } } - const action = props.file_action; + const action = file_action; const action_button = file_actions[action || "undefined"]; if (action_button == undefined) { return
Undefined action
; } - if (props.file_map == undefined) { + if (file_map == undefined) { return ; } else { return ( diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index 58857e1f8e..50402c84dc 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -3,40 +3,26 @@ * License: MS-RSL – see LICENSE.md for details */ -import * as immutable from "immutable"; import * as _ from "lodash"; -import React from "react"; import { UsersViewing } from "@cocalc/frontend/account/avatar/users-viewing"; import { Col, Row } from "@cocalc/frontend/antd-bootstrap"; -import { - project_redux_name, - rclass, - redux, - rtypes, - TypedMap, -} from "@cocalc/frontend/app-framework"; -import { ShallowTypedMap } from "@cocalc/frontend/app-framework/ShallowTypedMap"; +import { type CSSProperties, useEffect, useRef, useState } from "react"; import { A, ActivityDisplay, + ErrorDisplay, Loading, SettingBox, } from "@cocalc/frontend/components"; import { ComputeServerDocStatus } from "@cocalc/frontend/compute/doc-status"; import SelectComputeServerForFileExplorer from "@cocalc/frontend/compute/select-server-for-explorer"; -import { ComputeImages } from "@cocalc/frontend/custom-software/init"; import { CustomSoftwareReset } from "@cocalc/frontend/custom-software/reset-bar"; import { IS_MOBILE } from "@cocalc/frontend/feature"; import { FileUploadWrapper } from "@cocalc/frontend/file-upload"; import { Library } from "@cocalc/frontend/library"; -import { - Available, - MainConfiguration, -} from "@cocalc/frontend/project_configuration"; -import { ProjectActions } from "@cocalc/frontend/project_store"; -import { ProjectMap, ProjectStatus } from "@cocalc/frontend/todo-types"; +import { ProjectStatus } from "@cocalc/frontend/todo-types"; import AskNewFilename from "../ask-filename"; -import { useProjectContext } from "../context"; +import { useProjectContext } from "@cocalc/frontend/project/context"; import { ActionBar } from "./action-bar"; import { ActionBox } from "./action-box"; import { FileListing } from "./file-listing"; @@ -46,8 +32,8 @@ import { NewButton } from "./new-button"; import { PathNavigator } from "./path-navigator"; import { SearchBar } from "./search-bar"; import ExplorerTour from "./tour/tour"; -import ShowError from "@cocalc/frontend/components/error"; import { dirname, join } from "path"; +import { redux, useTypedRedux } from "@cocalc/frontend/app-framework"; const FLEX_ROW_STYLE = { display: "flex", @@ -56,9 +42,7 @@ const FLEX_ROW_STYLE = { alignItems: "stretch", } as const; -export type Configuration = ShallowTypedMap<{ main: MainConfiguration }>; - -const error_style: React.CSSProperties = { +const ERROR_STYLE: CSSProperties = { marginRight: "1ex", whiteSpace: "pre-line", position: "absolute", @@ -67,543 +51,393 @@ const error_style: React.CSSProperties = { boxShadow: "5px 5px 5px grey", } as const; -interface ReactProps { - project_id: string; - actions: ProjectActions; - name: string; -} - -interface ReduxProps { - project_map?: ProjectMap; - get_my_group: (project_id: string) => "admin" | "public"; - get_total_project_quotas: (project_id: string) => { member_host: boolean }; - other_settings?: immutable.Map; - is_logged_in?: boolean; - kucalc?: string; - site_name?: string; - images: ComputeImages; - active_file_sort: TypedMap<{ column_name: string; is_descending: boolean }>; - current_path: string; - history_path: string; - activity?: object; - file_action?: - | "compress" - | "delete" - | "rename" - | "duplicate" - | "move" - | "copy" - | "share" - | "download" - | "upload"; - file_search: string; - show_hidden?: boolean; - show_masked?: boolean; - error?: string; - checked_files: immutable.Set; - file_creation_error?: string; - ext_selection?: string; - new_name?: string; - library?: object; - show_library?: boolean; - public_paths?: immutable.List; // used only to trigger table init - configuration?: Configuration; - available_features?: Available; - file_listing_scroll_top?: number; - show_custom_software_reset?: boolean; - explorerTour?: boolean; - compute_server_id: number; - selected_file_index?: number; -} - -interface State { - shift_is_down: boolean; -} - export function Explorer() { - const { project_id } = useProjectContext(); - return ( - + const { actions, project_id } = useProjectContext(); + const newFileRef = useRef(null); + const searchAndTerminalBar = useRef(null); + const fileListingRef = useRef(null); + const currentDirectoryRef = useRef(null); + const miscButtonsRef = useRef(null); + const listingRef = useRef(null); + + const activity = useTypedRedux({ project_id }, "activity")?.toJS(); + const available_features = useTypedRedux( + { project_id }, + "available_features", + )?.toJS(); + const checked_files = useTypedRedux({ project_id }, "checked_files"); + const compute_server_id = useTypedRedux({ project_id }, "compute_server_id"); + const configuration = useTypedRedux({ project_id }, "configuration"); + const current_path = useTypedRedux({ project_id }, "current_path"); + const error = useTypedRedux({ project_id }, "error"); + const ext_selection = useTypedRedux({ project_id }, "ext_selection"); + const file_action = useTypedRedux({ project_id }, "file_action"); + const file_creation_error = useTypedRedux( + { project_id }, + "file_creation_error", ); -} - -// TODO: change/rewrite Explorer to not have any rtypes.objects and -// add a shouldComponentUpdate!! -const Explorer0 = rclass( - class Explorer extends React.Component { - newFileRef = React.createRef(); - searchAndTerminalBar = React.createRef(); - fileListingRef = React.createRef(); - currentDirectoryRef = React.createRef(); - miscButtonsRef = React.createRef(); - listingRef = React.createRef(); - - static reduxProps = ({ name }) => { - return { - projects: { - project_map: rtypes.immutable.Map, - get_my_group: rtypes.func.isRequired, - get_total_project_quotas: rtypes.func.isRequired, - }, - - account: { - other_settings: rtypes.immutable.Map, - is_logged_in: rtypes.bool, - }, - - customize: { - kucalc: rtypes.string, - site_name: rtypes.string, - }, - - compute_images: { - images: rtypes.immutable.Map, - }, - - [name]: { - active_file_sort: rtypes.immutable.Map, - current_path: rtypes.string, - history_path: rtypes.string, - activity: rtypes.object, - file_action: rtypes.string, - file_search: rtypes.string, - show_hidden: rtypes.bool, - show_masked: rtypes.bool, - error: rtypes.string, - checked_files: rtypes.immutable, - file_creation_error: rtypes.string, - ext_selection: rtypes.string, - new_name: rtypes.string, - library: rtypes.object, - show_library: rtypes.bool, - public_paths: rtypes.immutable, // used only to trigger table init - configuration: rtypes.immutable, - available_features: rtypes.object, - file_listing_scroll_top: rtypes.number, - show_custom_software_reset: rtypes.bool, - explorerTour: rtypes.bool, - compute_server_id: rtypes.number, - selected_file_index: rtypes.number, - }, - }; - }; - - static defaultProps = { - file_search: "", - new_name: "", - redux, - }; - - constructor(props) { - super(props); - this.state = { - shift_is_down: false, - }; - } + const file_search = useTypedRedux({ project_id }, "file_search"); + const selected_file_index = useTypedRedux( + { project_id }, + "selected_file_index", + ); + const show_custom_software_reset = useTypedRedux( + { project_id }, + "show_custom_software_reset", + ); + const show_library = useTypedRedux({ project_id }, "show_library"); + const [shiftIsDown, setShiftIsDown] = useState(false); - componentDidMount() { - // Update AFTER react draws everything - // Should probably be moved elsewhere - // Prevents cascading changes which impact responsiveness - // https://github.com/sagemathinc/cocalc/pull/3705#discussion_r268263750 - $(window).on("keydown", this.handle_files_key_down); - $(window).on("keyup", this.handle_files_key_up); - } + const project_map = useTypedRedux("projects", "project_map"); - componentWillUnmount() { - $(window).off("keydown", this.handle_files_key_down); - $(window).off("keyup", this.handle_files_key_up); - } + const images = useTypedRedux("compute_images", "images"); - handle_files_key_down = (e): void => { - if (e.key === "Shift") { - this.setState({ shift_is_down: true }); - } else if (e.key == "ArrowUp") { - if (e.shiftKey || e.ctrlKey || e.metaKey) { - const path = dirname(this.props.current_path); - this.props.actions.open_directory(path == "." ? "" : path); - } else { - this.props.actions.decrement_selected_file_index(); - } - } else if (e.key == "ArrowDown") { - this.props.actions.increment_selected_file_index(); - } else if (e.key == "Enter") { - const x = - this.listingRef.current?.[this.props.selected_file_index ?? 0]; - if (x != null) { - const { isdir, name } = x; - const path = join(this.props.current_path, name); - if (isdir) { - this.props.actions.open_directory(path); - } else { - this.props.actions.open_file({ path, foreground: !e.ctrlKey }); - } - if (!e.ctrlKey) { - this.props.actions.set_file_search(""); - this.props.actions.clear_selected_file_index(); - } - } - } - }; + if (actions == null || project_map == null) { + return ; + } - handle_files_key_up = (e): void => { - if (e.key === "Shift") { - this.setState({ shift_is_down: false }); - } + useEffect(() => { + $(window).on("keydown", handle_files_key_down); + $(window).on("keyup", handle_files_key_up); + return () => { + $(window).off("keydown", handle_files_key_down); + $(window).off("keyup", handle_files_key_up); }; - - create_file = (ext, switch_over) => { - if (switch_over == undefined) { - switch_over = true; + }, []); + + const handle_files_key_down = (e): void => { + if (e.key === "Shift") { + setShiftIsDown(true); + } else if (e.key == "ArrowUp") { + if (e.shiftKey || e.ctrlKey || e.metaKey) { + const path = dirname(current_path); + actions.open_directory(path == "." ? "" : path); + } else { + actions.decrement_selected_file_index(); } - const { file_search } = this.props; - if ( - ext == undefined && - file_search.lastIndexOf(".") <= file_search.lastIndexOf("/") - ) { - let disabled_ext; - if (this.props.configuration != undefined) { - ({ disabled_ext } = this.props.configuration.get("main", { - disabled_ext: [], - })); + } else if (e.key == "ArrowDown") { + actions.increment_selected_file_index(); + } else if (e.key == "Enter") { + const x = listingRef.current?.[selected_file_index ?? 0]; + if (x != null) { + const { isdir, name } = x; + const path = join(current_path, name); + if (isdir) { + actions.open_directory(path); } else { - disabled_ext = []; + actions.open_file({ path, foreground: !e.ctrlKey }); + } + if (!e.ctrlKey) { + actions.set_file_search(""); + actions.clear_selected_file_index(); } - ext = default_ext(disabled_ext); } - - this.props.actions.create_file({ - name: file_search, - ext, - current_path: this.props.current_path, - switch_over, - }); - this.props.actions.setState({ file_search: "" }); - }; - - create_folder = (switch_over = true): void => { - this.props.actions.create_folder({ - name: this.props.file_search, - current_path: this.props.current_path, - switch_over, - }); - this.props.actions.setState({ file_search: "" }); - }; - - file_listing_page_size() { - return ( - this.props.other_settings && - this.props.other_settings.get("page_size", 50) - ); } + }; - render() { - let project_is_running: boolean, project_state: ProjectStatus | undefined; - - if (this.props.checked_files == undefined) { - // hasn't loaded/initialized at all - return ; - } - - const my_group = this.props.get_my_group(this.props.project_id); + const handle_files_key_up = (e): void => { + if (e.key === "Shift") { + setShiftIsDown(false); + } + }; - // regardless of consequences, for admins a project is always running - // see https://github.com/sagemathinc/cocalc/issues/3863 - if (my_group === "admin") { - project_state = new ProjectStatus({ state: "running" }); - project_is_running = true; - // next, we check if this is a common user (not public) - } else if (my_group !== "public") { - project_state = this.props.project_map?.getIn([ - this.props.project_id, - "state", - ]) as any; - project_is_running = project_state?.get("state") == "running"; - } else { - project_is_running = false; - } + const create_file = (ext, switch_over) => { + if (switch_over == undefined) { + switch_over = true; + } + if ( + ext == undefined && + file_search != null && + file_search.lastIndexOf(".") <= file_search.lastIndexOf("/") + ) { + const disabled_ext = // @ts-ignore + configuration?.getIn(["main", "disabled_ext"])?.toJS() ?? []; + ext = default_ext(disabled_ext); + } - // be careful with adding height:'100%'. it could cause flex to miscalculate. see #3904 - return ( -
+ actions.create_file({ + name: file_search ?? "", + ext, + current_path: current_path, + switch_over, + }); + actions.setState({ file_search: "" }); + }; + + const create_folder = (switch_over = true): void => { + actions.create_folder({ + name: file_search ?? "", + current_path: current_path, + switch_over, + }); + actions.setState({ file_search: "" }); + }; + + let project_is_running: boolean, project_state: ProjectStatus | undefined; + + if (checked_files == undefined) { + // hasn't loaded/initialized at all + return ; + } + + const my_group = redux.getStore("projects").get_my_group(project_id); + + // regardless of consequences, for admins a project is always running + // see https://github.com/sagemathinc/cocalc/issues/3863 + if (my_group === "admin") { + project_state = new ProjectStatus({ state: "running" }); + project_is_running = true; + // next, we check if this is a common user (not public) + } else if (my_group !== "public") { + project_state = project_map?.getIn([project_id, "state"]) as any; + project_is_running = project_state?.get("state") == "running"; + } else { + project_is_running = false; + } + + // be careful with adding height:'100%'. it could cause flex to miscalculate. see #3904 + return ( +
+
+ {error && ( + actions.setState({ error: "" })} + /> + )} + actions.clear_all_activity()} + style={{ top: "100px" }} + /> +
- this.props.actions.setState({ error })} - /> - this.props.actions.clear_all_activity()} - style={{ top: "100px" }} - /> -
-
-
- -
- -
-
- {!!this.props.compute_server_id && ( -
- -
- )} -
- {!IS_MOBILE && ( -
-
- -
-
- )} - {!IS_MOBILE && ( -
- -
- )} +
+
- +
- - {this.props.ext_selection != null && ( - - )} -
+ {!!compute_server_id && (
-
-
- + {!IS_MOBILE && ( +
+
+
- - {project_is_running && - this.props.show_custom_software_reset && - this.props.checked_files.size == 0 && ( - - )} - - {this.props.show_library && ( - - - - Library{" "} - - (help...) - - - } - close={() => this.props.actions.toggle_library(false)} - > - this.props.actions.toggle_library(false)} - /> - - - - )} - - {this.props.checked_files.size > 0 && - this.props.file_action != undefined ? ( - - - - - - ) : undefined} + )} + {!IS_MOBILE && ( +
+ +
+ )} +
+
+
+ {ext_selection != null && } +
- - - + +
+
+
-
- ); - } - }, -); + + {project_is_running && + show_custom_software_reset && + checked_files.size == 0 && + images != null && ( + + )} + + {show_library && ( + + + + Library{" "} + + (help...) + + + } + close={() => actions.toggle_library(false)} + > + actions.toggle_library(false)} + /> + + + + )} + + {checked_files.size > 0 && file_action != undefined ? ( + + + + + + ) : undefined} +
+ +
+ + + +
+ +
+ ); +} diff --git a/src/packages/frontend/project/explorer/file-listing/file-checkbox.tsx b/src/packages/frontend/project/explorer/file-listing/file-checkbox.tsx index 79481771c9..723fcc5b3d 100644 --- a/src/packages/frontend/project/explorer/file-listing/file-checkbox.tsx +++ b/src/packages/frontend/project/explorer/file-listing/file-checkbox.tsx @@ -17,26 +17,26 @@ interface Props { style?: React.CSSProperties; } -export const FileCheckbox: React.FC = React.memo((props: Props) => { - const { name, checked, actions, current_path, style } = props; - - function handle_click(e) { - e.stopPropagation(); // so we don't open the file - const full_name = path_to_file(current_path, name); - if (e.shiftKey) { - actions.set_selected_file_range(full_name, !checked); - } else { - actions.set_file_checked(full_name, !checked); +export const FileCheckbox: React.FC = React.memo( + ({ name, checked, actions, current_path, style }: Props) => { + function handle_click(e) { + e.stopPropagation(); // so we don't open the file + const full_name = path_to_file(current_path, name); + if (e.shiftKey) { + actions.set_selected_file_range(full_name, !checked); + } else { + actions.set_file_checked(full_name, !checked); + } + actions.set_most_recent_file_click(full_name); } - actions.set_most_recent_file_click(full_name); - } - return ( - - - - ); -}); + return ( + + + + ); + }, +); diff --git a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx index a46c23d765..c76c733b7d 100644 --- a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx +++ b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx @@ -13,10 +13,10 @@ import { useEffect, useRef } from "react"; import { FormattedMessage } from "react-intl"; import { Virtuoso, VirtuosoHandle } from "react-virtuoso"; import { - AppRedux, Rendered, TypedMap, useTypedRedux, + redux, } from "@cocalc/frontend/app-framework"; import useVirtuosoScrollHook from "@cocalc/frontend/components/virtuoso-scroll-hook"; import { ProjectActions } from "@cocalc/frontend/project_actions"; @@ -36,7 +36,6 @@ import filterListing from "@cocalc/frontend/project/listing/filter-listing"; interface Props { // TODO: everything but actions/redux should be immutable JS data, and use shouldComponentUpdate actions: ProjectActions; - redux: AppRedux; name: string; active_file_sort: TypedMap<{ column_name: string; is_descending: boolean }>; @@ -46,16 +45,9 @@ interface Props { current_path: string; project_id: string; shift_is_down: boolean; - sort_by: (heading: string) => void; - library?: object; - other_settings?: immutable.Map; - last_scroll_top?: number; configuration_main?: MainConfiguration; isRunning?: boolean; // true if this project is running - show_hidden?: boolean; - show_masked?: boolean; - stale?: boolean; listingRef; @@ -82,21 +74,26 @@ function sortDesc(active_file_sort?): { } export function FileListing(props) { + const { project_id } = props; + const active_file_sort = useTypedRedux({ project_id }, "active_file_sort"); const path = props.current_path; - const fs = useFs({ project_id: props.project_id }); + const fs = useFs({ project_id }); let { listing, error } = useListing({ fs, path, ...sortDesc(props.active_file_sort), - cacheId: { project_id: props.project_id }, + cacheId: { project_id }, }); + const showHidden = useTypedRedux({ project_id }, "show_hidden"); + const showMasked = useTypedRedux({ project_id }, "show_masked"); props.listingRef.current = listing = error ? null : filterListing({ listing, search: props.file_search, - showHidden: props.show_hidden, + showHidden, + showMasked, }); useEffect(() => { @@ -109,12 +106,11 @@ export function FileListing(props) { if (listing == null) { return ; } - return ; + return ; } function FileListing0({ actions, - redux, name, active_file_sort, listing, @@ -122,7 +118,6 @@ function FileListing0({ current_path, project_id, shift_is_down, - sort_by, configuration_main, file_search = "", stale, @@ -276,7 +271,10 @@ function FileListing0({ flexDirection: "column", }} > - + {listing.length > 0 ? render_rows() : render_no_files()}
diff --git a/src/packages/frontend/project/explorer/file-listing/listing-header.tsx b/src/packages/frontend/project/explorer/file-listing/listing-header.tsx index 7d40cd5ee5..978153bf02 100644 --- a/src/packages/frontend/project/explorer/file-listing/listing-header.tsx +++ b/src/packages/frontend/project/explorer/file-listing/listing-header.tsx @@ -24,11 +24,7 @@ const row_style: React.CSSProperties = { const inner_icon_style = { marginRight: "10px" }; -// TODO: Something should uniformly describe how sorted table headers work. -// 5/8/2017 We have 3 right now, Course students, assignments panel and this one. -export const ListingHeader: React.FC = (props: Props) => { - const { active_file_sort, sort_by } = props; - +export function ListingHeader({ active_file_sort, sort_by }: Props) { function render_sort_link( column_name: string, display_name: string, @@ -81,4 +77,4 @@ export const ListingHeader: React.FC = (props: Props) => { ); -}; +} diff --git a/src/packages/frontend/project/explorer/file-listing/utils.ts b/src/packages/frontend/project/explorer/file-listing/utils.ts index 9745e51556..3fd6015077 100644 --- a/src/packages/frontend/project/explorer/file-listing/utils.ts +++ b/src/packages/frontend/project/explorer/file-listing/utils.ts @@ -48,9 +48,7 @@ export const EXTs: ReadonlyArray = Object.freeze([ "sage-chat", ]); -export function default_ext( - disabled_ext: { includes: (s: string) => boolean } | undefined -): Extension { +export function default_ext(disabled_ext: string[] | undefined): Extension { if (disabled_ext != null) { for (const ext of EXTs) { if (disabled_ext.includes(ext)) continue; @@ -79,7 +77,7 @@ export function full_path_text(file_search: string, disabled_ext: string[]) { export function generate_click_for( file_action_name: string, full_path: string, - project_actions: ProjectActions + project_actions: ProjectActions, ) { return (e) => { e.preventDefault(); diff --git a/src/packages/frontend/project/explorer/misc-side-buttons.tsx b/src/packages/frontend/project/explorer/misc-side-buttons.tsx index 73772b9272..b4444c0c18 100644 --- a/src/packages/frontend/project/explorer/misc-side-buttons.tsx +++ b/src/packages/frontend/project/explorer/misc-side-buttons.tsx @@ -5,7 +5,6 @@ import { Space } from "antd"; import { join } from "path"; -import React from "react"; import { defineMessage, useIntl } from "react-intl"; import { Button, ButtonToolbar } from "@cocalc/frontend/antd-bootstrap"; import { Icon, Tip, VisibleLG } from "@cocalc/frontend/components"; @@ -13,10 +12,11 @@ import LinkRetry from "@cocalc/frontend/components/link-retry"; import { useStudentProjectFunctionality } from "@cocalc/frontend/course"; import { labels } from "@cocalc/frontend/i18n"; import { serverURL, SPEC } from "@cocalc/frontend/project/named-server-panel"; -import { Available } from "@cocalc/frontend/project_configuration"; -import { ProjectActions } from "@cocalc/frontend/project_store"; import track from "@cocalc/frontend/user-tracking"; import { KUCALC_COCALC_COM } from "@cocalc/util/db-schema/site-defaults"; +import { useProjectContext } from "@cocalc/frontend/project/context"; +import { useTypedRedux } from "@cocalc/frontend/app-framework"; +import { type JSX, type MouseEvent } from "react"; const SHOW_SERVER_LAUNCHERS = false; @@ -27,53 +27,42 @@ const OPEN_MSG = defineMessage({ defaultMessage: `Opens the current directory in a {name} server instance, running inside this project.`, }); -interface Props { - actions: ProjectActions; - available_features?: Available; - current_path: string; - kucalc?: string; - project_id: string; - show_hidden?: boolean; - show_masked?: boolean; -} - -export const MiscSideButtons: React.FC = (props) => { - const { - actions, - available_features, - current_path, - kucalc, - project_id, - show_hidden, - show_masked, - } = props; - +export function MiscSideButtons() { + const { actions, project_id } = useProjectContext(); + const show_hidden = useTypedRedux({ project_id }, "show_hidden"); + const show_masked = useTypedRedux({ project_id }, "show_masked"); + const current_path = useTypedRedux({ project_id }, "current_path"); + const available_features = useTypedRedux( + { project_id }, + "available_features", + )?.toJS(); + const kucalc = useTypedRedux("customize", "kucalc"); const intl = useIntl(); const student_project_functionality = useStudentProjectFunctionality(project_id); - const handle_hidden_toggle = (e: React.MouseEvent): void => { + const handle_hidden_toggle = (e: MouseEvent): void => { e.preventDefault(); - return actions.setState({ + return actions?.setState({ show_hidden: !show_hidden, }); }; - const handle_masked_toggle = (e: React.MouseEvent): void => { + const handle_masked_toggle = (e: MouseEvent): void => { e.preventDefault(); - actions.setState({ + actions?.setState({ show_masked: !show_masked, }); }; - const handle_backup = (e: React.MouseEvent): void => { + const handle_backup = (e: MouseEvent): void => { e.preventDefault(); - actions.open_directory(".snapshots"); + actions?.open_directory(".snapshots"); track("snapshots", { action: "open", where: "explorer" }); }; - function render_hidden_toggle(): React.JSX.Element { + function render_hidden_toggle(): JSX.Element { const icon = show_hidden ? "eye" : "eye-slash"; return (
); -}; +} diff --git a/src/packages/frontend/project/explorer/new-button.tsx b/src/packages/frontend/project/explorer/new-button.tsx index 437821485f..a9e37355c7 100644 --- a/src/packages/frontend/project/explorer/new-button.tsx +++ b/src/packages/frontend/project/explorer/new-button.tsx @@ -10,7 +10,6 @@ import { DropdownMenu, Icon } from "@cocalc/frontend/components"; import { labels } from "@cocalc/frontend/i18n"; import { ProjectActions } from "@cocalc/frontend/project_store"; import { COLORS } from "@cocalc/util/theme"; -import { Configuration } from "./explorer"; import { EXTs as ALL_FILE_BUTTON_TYPES } from "./file-listing/utils"; const { file_options } = require("@cocalc/frontend/editor"); @@ -21,21 +20,18 @@ interface Props { actions: ProjectActions; create_folder: (switch_over?: boolean) => void; create_file: (ext?: string, switch_over?: boolean) => void; - configuration?: Configuration; + configuration?; disabled: boolean; } -export const NewButton: React.FC = (props: Props) => { - const { - file_search = "", - /*current_path,*/ - actions, - create_folder, - create_file, - configuration, - disabled, - } = props; - +export const NewButton: React.FC = ({ + file_search = "", + actions, + create_folder, + create_file, + configuration, + disabled, +}: Props) => { const intl = useIntl(); function new_file_button_types() { diff --git a/src/packages/frontend/project/explorer/path-navigator.tsx b/src/packages/frontend/project/explorer/path-navigator.tsx index 52cdd0eb7f..835e082e44 100644 --- a/src/packages/frontend/project/explorer/path-navigator.tsx +++ b/src/packages/frontend/project/explorer/path-navigator.tsx @@ -23,13 +23,12 @@ interface Props { // This path consists of several PathSegmentLinks export const PathNavigator: React.FC = React.memo( - (props: Readonly) => { - const { - project_id, - style, - className = "cc-path-navigator", - mode = "files", - } = props; + ({ + project_id, + style, + className = "cc-path-navigator", + mode = "files", + }: Readonly) => { const current_path = useTypedRedux({ project_id }, "current_path"); const history_path = useTypedRedux({ project_id }, "history_path"); const actions = useActions({ project_id }); diff --git a/src/packages/frontend/project/explorer/path-segment-link.tsx b/src/packages/frontend/project/explorer/path-segment-link.tsx index de1fbb2cea..63c4d56dd5 100644 --- a/src/packages/frontend/project/explorer/path-segment-link.tsx +++ b/src/packages/frontend/project/explorer/path-segment-link.tsx @@ -27,18 +27,16 @@ export interface PathSegmentItem { } // One segment of the directory links at the top of the files listing. -export function createPathSegmentLink(props: Readonly): PathSegmentItem { - const { - path = "", - display, - on_click, - full_name, - history, - active = false, - key, - style, - } = props; - +export function createPathSegmentLink({ + path = "", + display, + on_click, + full_name, + history, + active = false, + key, + style, +}: Readonly): PathSegmentItem { function render_content(): React.JSX.Element | string | undefined { if (full_name && full_name !== display) { return ( diff --git a/src/packages/frontend/project/explorer/tour/tour.tsx b/src/packages/frontend/project/explorer/tour/tour.tsx index 8a8a0e41c1..6b7d4bd412 100644 --- a/src/packages/frontend/project/explorer/tour/tour.tsx +++ b/src/packages/frontend/project/explorer/tour/tour.tsx @@ -1,14 +1,12 @@ import type { TourProps } from "antd"; import { Checkbox, Tour } from "antd"; - -import { redux } from "@cocalc/frontend/app-framework"; +import { redux, useTypedRedux } from "@cocalc/frontend/app-framework"; import { Paragraph, Text } from "@cocalc/frontend/components"; import { A } from "@cocalc/frontend/components/A"; import { Icon } from "@cocalc/frontend/components/icon"; import actionsImage from "./actions.png"; export default function ExplorerTour({ - open, project_id, newFileRef, searchAndTerminalBar, @@ -16,6 +14,7 @@ export default function ExplorerTour({ currentDirectoryRef, miscButtonsRef, }) { + const open = useTypedRedux({ project_id }, "explorerTour"); const steps: TourProps["steps"] = [ { title: ( diff --git a/src/packages/frontend/project/listing/filter-listing.ts b/src/packages/frontend/project/listing/filter-listing.ts index 53e296da7f..a5b03edb69 100644 --- a/src/packages/frontend/project/listing/filter-listing.ts +++ b/src/packages/frontend/project/listing/filter-listing.ts @@ -4,11 +4,16 @@ export default function filterListing({ listing, search, showHidden, + showMasked, }: { listing?: DirectoryListingEntry[] | null; search?: string; showHidden?: boolean; + showMasked?: boolean; }): DirectoryListingEntry[] | null { + if (!showMasked) { + console.log("TODO: show masked"); + } if (listing == null) { return null; } From 489eaf1419cbee4b160f081af1de0c96c85e8a43 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 29 Jul 2025 02:54:15 +0000 Subject: [PATCH 125/798] fix bugs in new file listings --- .../frontend/project/explorer/explorer.tsx | 84 +++++++++---------- .../explorer/file-listing/file-listing.tsx | 6 +- 2 files changed, 44 insertions(+), 46 deletions(-) diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index 50402c84dc..d81087b553 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -77,10 +77,6 @@ export function Explorer() { "file_creation_error", ); const file_search = useTypedRedux({ project_id }, "file_search"); - const selected_file_index = useTypedRedux( - { project_id }, - "selected_file_index", - ); const show_custom_software_reset = useTypedRedux( { project_id }, "show_custom_software_reset", @@ -97,49 +93,51 @@ export function Explorer() { } useEffect(() => { + const handle_files_key_down = (e): void => { + if (e.key === "Shift") { + setShiftIsDown(true); + } else if (e.key == "ArrowUp") { + if (e.shiftKey || e.ctrlKey || e.metaKey) { + const path = dirname(current_path); + actions.open_directory(path == "." ? "" : path); + } else { + actions.decrement_selected_file_index(); + } + } else if (e.key == "ArrowDown") { + actions.increment_selected_file_index(); + } else if (e.key == "Enter") { + const n = + redux.getProjectStore(project_id).get("selected_file_index") ?? 0; + const x = listingRef.current?.[n]; + if (x != null) { + const { isdir, name } = x; + const path = join(current_path, name); + if (isdir) { + actions.open_directory(path); + } else { + actions.open_file({ path, foreground: !e.ctrlKey }); + } + if (!e.ctrlKey) { + actions.set_file_search(""); + actions.clear_selected_file_index(); + } + } + } + }; + + const handle_files_key_up = (e): void => { + if (e.key === "Shift") { + setShiftIsDown(false); + } + }; + $(window).on("keydown", handle_files_key_down); $(window).on("keyup", handle_files_key_up); return () => { $(window).off("keydown", handle_files_key_down); $(window).off("keyup", handle_files_key_up); }; - }, []); - - const handle_files_key_down = (e): void => { - if (e.key === "Shift") { - setShiftIsDown(true); - } else if (e.key == "ArrowUp") { - if (e.shiftKey || e.ctrlKey || e.metaKey) { - const path = dirname(current_path); - actions.open_directory(path == "." ? "" : path); - } else { - actions.decrement_selected_file_index(); - } - } else if (e.key == "ArrowDown") { - actions.increment_selected_file_index(); - } else if (e.key == "Enter") { - const x = listingRef.current?.[selected_file_index ?? 0]; - if (x != null) { - const { isdir, name } = x; - const path = join(current_path, name); - if (isdir) { - actions.open_directory(path); - } else { - actions.open_file({ path, foreground: !e.ctrlKey }); - } - if (!e.ctrlKey) { - actions.set_file_search(""); - actions.clear_selected_file_index(); - } - } - } - }; - - const handle_files_key_up = (e): void => { - if (e.key === "Shift") { - setShiftIsDown(false); - } - }; + }, [project_id, current_path]); const create_file = (ext, switch_over) => { if (switch_over == undefined) { @@ -151,7 +149,7 @@ export function Explorer() { file_search.lastIndexOf(".") <= file_search.lastIndexOf("/") ) { const disabled_ext = // @ts-ignore - configuration?.getIn(["main", "disabled_ext"])?.toJS() ?? []; + configuration?.getIn(["main", "disabled_ext"])?.toJS?.() ?? []; ext = default_ext(disabled_ext); } @@ -425,7 +423,7 @@ export function Explorer() { create_file={create_file} create_folder={create_folder} project_id={project_id} - shift_is_down={shiftIsDown} + shiftIsDown={shiftIsDown} configuration_main={configuration?.get("main")} /> diff --git a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx index c76c733b7d..422fb0ace4 100644 --- a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx +++ b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx @@ -44,7 +44,7 @@ interface Props { checked_files: immutable.Set; current_path: string; project_id: string; - shift_is_down: boolean; + shiftIsDown: boolean; configuration_main?: MainConfiguration; isRunning?: boolean; // true if this project is running @@ -117,7 +117,7 @@ function FileListing0({ checked_files, current_path, project_id, - shift_is_down, + shiftIsDown, configuration_main, file_search = "", stale, @@ -161,7 +161,7 @@ function FileListing0({ key={index} current_path={current_path} actions={actions} - no_select={shift_is_down} + no_select={shiftIsDown} link_target={link_target} computeServerId={computeServerId} /> From adf444f577a214250399511527b9ab9e205998fe Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 29 Jul 2025 05:14:59 +0000 Subject: [PATCH 126/798] files: select a range of files --- .../frontend/project/directory-selector.tsx | 6 +- .../frontend/project/explorer/action-box.tsx | 5 +- .../frontend/project/explorer/download.tsx | 10 +- .../explorer/file-listing/file-checkbox.tsx | 50 +++--- .../explorer/file-listing/file-listing.tsx | 4 +- .../explorer/file-listing/file-row.tsx | 2 + .../frontend/project/listing/use-files.ts | 10 ++ src/packages/frontend/project_actions.ts | 162 ++++++++++++------ src/packages/frontend/project_store.ts | 2 - 9 files changed, 159 insertions(+), 92 deletions(-) diff --git a/src/packages/frontend/project/directory-selector.tsx b/src/packages/frontend/project/directory-selector.tsx index bc10ca9fdb..bb1a474595 100644 --- a/src/packages/frontend/project/directory-selector.tsx +++ b/src/packages/frontend/project/directory-selector.tsx @@ -26,7 +26,9 @@ import { redux, useTypedRedux } from "@cocalc/frontend/app-framework"; import { useFrameContext } from "@cocalc/frontend/frame-editors/frame-tree/frame-context"; import ShowError from "@cocalc/frontend/components/error"; import useFs from "@cocalc/frontend/project/listing/use-fs"; -import useFiles from "@cocalc/frontend/project/listing/use-files"; +import useFiles, { + getCacheId, +} from "@cocalc/frontend/project/listing/use-files"; const NEW_FOLDER = "New Folder"; @@ -378,7 +380,7 @@ function Subdirs(props) { const { files, error, refresh } = useFiles({ fs, path, - cacheId: { project_id }, + cacheId: getCacheId({ project_id, compute_server_id: computeServerId }), }); if (error) { return ; diff --git a/src/packages/frontend/project/explorer/action-box.tsx b/src/packages/frontend/project/explorer/action-box.tsx index b5a5df0f19..f39af2a2b9 100644 --- a/src/packages/frontend/project/explorer/action-box.tsx +++ b/src/packages/frontend/project/explorer/action-box.tsx @@ -46,14 +46,13 @@ export const PRE_STYLE = { type FileAction = undefined | keyof typeof file_actions; -interface ReactProps { +interface Props { checked_files: immutable.Set; file_action: FileAction; current_path: string; project_id: string; file_map: object; actions: ProjectActions; - displayed_listing?: object; } export function ActionBox({ @@ -63,7 +62,7 @@ export function ActionBox({ project_id, file_map, actions, -}: ReactProps) { +}: Props) { const intl = useIntl(); const runQuota = useRunQuota(project_id, null); const get_user_type: () => string = useRedux("account", "get_user_type"); diff --git a/src/packages/frontend/project/explorer/download.tsx b/src/packages/frontend/project/explorer/download.tsx index ddaa75c793..17f7740a76 100644 --- a/src/packages/frontend/project/explorer/download.tsx +++ b/src/packages/frontend/project/explorer/download.tsx @@ -3,7 +3,7 @@ import { useEffect, useRef, useState } from "react"; import { useIntl } from "react-intl"; import { default_filename } from "@cocalc/frontend/account"; -import { redux, useRedux } from "@cocalc/frontend/app-framework"; +import { useRedux } from "@cocalc/frontend/app-framework"; import ShowError from "@cocalc/frontend/components/error"; import { Icon } from "@cocalc/frontend/components/icon"; import { labels } from "@cocalc/frontend/i18n"; @@ -12,7 +12,7 @@ import { path_split, path_to_file, plural } from "@cocalc/util/misc"; import { PRE_STYLE } from "./action-box"; import CheckedFiles from "./checked-files"; -export default function Download({}) { +export default function Download() { const intl = useIntl(); const inputRef = useRef(null); const { actions } = useProjectContext(); @@ -33,6 +33,9 @@ export default function Download({}) { ); useEffect(() => { + if (actions == null) { + return; + } if (checked_files == null) { return; } @@ -41,8 +44,7 @@ export default function Download({}) { return; } const file = checked_files.first(); - const isdir = redux.getProjectStore(project_id).get("displayed_listing") - ?.file_map?.[path_split(file).tail]?.isdir; + const isdir = !!actions.isDirViaCache(file); setArchiveMode(!!isdir); if (!isdir) { const store = actions?.get_store(); diff --git a/src/packages/frontend/project/explorer/file-listing/file-checkbox.tsx b/src/packages/frontend/project/explorer/file-listing/file-checkbox.tsx index 723fcc5b3d..9dd9c62301 100644 --- a/src/packages/frontend/project/explorer/file-listing/file-checkbox.tsx +++ b/src/packages/frontend/project/explorer/file-listing/file-checkbox.tsx @@ -3,8 +3,6 @@ * License: MS-RSL – see LICENSE.md for details */ -import React from "react"; - import { ProjectActions } from "@cocalc/frontend/project_actions"; import { Icon } from "@cocalc/frontend/components"; import { path_to_file } from "@cocalc/util/misc"; @@ -15,28 +13,34 @@ interface Props { actions: ProjectActions; current_path: string; style?: React.CSSProperties; + listing; } -export const FileCheckbox: React.FC = React.memo( - ({ name, checked, actions, current_path, style }: Props) => { - function handle_click(e) { - e.stopPropagation(); // so we don't open the file - const full_name = path_to_file(current_path, name); - if (e.shiftKey) { - actions.set_selected_file_range(full_name, !checked); - } else { - actions.set_file_checked(full_name, !checked); - } - actions.set_most_recent_file_click(full_name); +export function FileCheckbox({ + name, + checked, + actions, + current_path, + style, + listing, +}: Props) { + function handle_click(e) { + e.stopPropagation(); // so we don't open the file + const full_name = path_to_file(current_path, name); + if (e.shiftKey) { + actions.set_selected_file_range(full_name, !checked, listing); + } else { + actions.set_file_checked(full_name, !checked); } + actions.set_most_recent_file_click(full_name); + } - return ( - - - - ); - }, -); + return ( + + + + ); +} diff --git a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx index 422fb0ace4..86b89dd6a0 100644 --- a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx +++ b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx @@ -34,7 +34,6 @@ import useListing, { import filterListing from "@cocalc/frontend/project/listing/filter-listing"; interface Props { - // TODO: everything but actions/redux should be immutable JS data, and use shouldComponentUpdate actions: ProjectActions; name: string; @@ -82,7 +81,7 @@ export function FileListing(props) { fs, path, ...sortDesc(props.active_file_sort), - cacheId: { project_id }, + cacheId: props.actions.getCacheId(), }); const showHidden = useTypedRedux({ project_id }, "show_hidden"); const showMasked = useTypedRedux({ project_id }, "show_masked"); @@ -164,6 +163,7 @@ function FileListing0({ no_select={shiftIsDown} link_target={link_target} computeServerId={computeServerId} + listing={listing} /> ); } diff --git a/src/packages/frontend/project/explorer/file-listing/file-row.tsx b/src/packages/frontend/project/explorer/file-listing/file-row.tsx index 869d99b776..f5217d7bd4 100644 --- a/src/packages/frontend/project/explorer/file-listing/file-row.tsx +++ b/src/packages/frontend/project/explorer/file-listing/file-row.tsx @@ -55,6 +55,7 @@ interface Props { // if given, include a little 'server' tag in this color, and tooltip etc using id. // Also important for download and preview links! computeServerId?: number; + listing; } export const FileRow: React.FC = React.memo((props) => { @@ -356,6 +357,7 @@ export const FileRow: React.FC = React.memo((props) => { current_path={props.current_path} actions={props.actions} style={{ verticalAlign: "sub", color: "#888" }} + listing={props.listing} /> )} diff --git a/src/packages/frontend/project/listing/use-files.ts b/src/packages/frontend/project/listing/use-files.ts index 6e925e817d..7075c98b4c 100644 --- a/src/packages/frontend/project/listing/use-files.ts +++ b/src/packages/frontend/project/listing/use-files.ts @@ -178,3 +178,13 @@ async function cacheNeighbors({ v = v.slice(0, MAX_SUBDIR_CACHE); await Promise.all(v.map(f)); } + +export function getCacheId({ + project_id, + compute_server_id = 0, +}: { + project_id: string; + compute_server_id?: number; +}) { + return { project_id, compute_server_id }; +} diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index 285e6e4f06..296678b9cb 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -109,9 +109,11 @@ import { client_db } from "@cocalc/util/schema"; import { get_editor } from "./editors/react-wrapper"; import { type FilesystemClient } from "@cocalc/conat/files/fs"; import { + getCacheId, getFiles, type Files, } from "@cocalc/frontend/project/listing/use-files"; +import { map as awaitMap } from "awaiting"; const { defaults, required } = misc; @@ -1618,7 +1620,7 @@ export class ProjectActions extends Actions { } // Set the selected state of all files between the most_recent_file_click and the given file - set_selected_file_range(file: string, checked: boolean): void { + set_selected_file_range(file: string, checked: boolean, listing): void { let range; const store = this.get_store(); if (store == undefined) { @@ -1631,9 +1633,9 @@ export class ProjectActions extends Actions { } else { // get the range of files const current_path = store.get("current_path"); - const names = store - .get("displayed_listing") - .listing.map((a) => misc.path_to_file(current_path, a.name)); + const names = listing.map(({ name }) => + misc.path_to_file(current_path, name), + ); range = misc.get_array_range(names, most_recent, file); } @@ -1898,22 +1900,29 @@ export class ProjectActions extends Actions { } }; - // DANGER: ASSUMES PATH IS IN THE DISPLAYED LISTING - private _convert_to_displayed_path(path): string { - if (path.slice(-1) === "/") { - return path; - } else { - const store = this.get_store(); - const file_name = misc.path_split(path).tail; - if (store !== undefined && store.get("displayed_listing")) { - const file_data = store.get("displayed_listing").file_map[file_name]; - if (file_data !== undefined && file_data.isdir) { - return path + "/"; - } + private appendSlashToDirectoryPaths = async ( + paths: string[], + compute_server_id?: number, + ): Promise => { + const f = async (path: string) => { + if (path.endsWith("/")) { + return path; } - return path; - } - } + const isdir = this.isDirViaCache(path, compute_server_id); + if (isdir === false) { + return path; + } + if (isdir === true) { + return path + "/"; + } + if (await this.isdir(path, compute_server_id)) { + return path + "/"; + } else { + return path; + } + }; + return await Promise.all(paths.map(f)); + }; // this is called in "projects.cjsx" (more then once) // in turn, it is calling init methods just once, though @@ -2234,12 +2243,15 @@ export class ProjectActions extends Actions { dest_compute_server_id: this.get_store()?.get("compute_server_id") ?? 0, }); // true for duplicating files - const with_slashes = opts.src.map(this._convert_to_displayed_path); + const withSlashes = await this.appendSlashToDirectoryPaths( + opts.src, + opts.src_compute_server_id, + ); this.log({ event: "file_action", action: "copied", - files: with_slashes.slice(0, 3), + files: withSlashes.slice(0, 3), count: opts.src.length > 3 ? opts.src.length : undefined, dest: opts.dest + (opts.only_contents ? "" : "/"), ...(opts.src_compute_server_id != opts.dest_compute_server_id @@ -2253,7 +2265,7 @@ export class ProjectActions extends Actions { }); if (opts.only_contents) { - opts.src = with_slashes; + opts.src = withSlashes; } // If files start with a -, make them interpretable by rsync (see https://github.com/sagemathinc/cocalc/issues/516) @@ -2341,17 +2353,23 @@ export class ProjectActions extends Actions { }); }; - copy_paths_between_projects(opts) { - opts = defaults(opts, { - public: false, - src_project_id: required, // id of source project - src: required, // list of relative paths of directories or files in the source project - target_project_id: required, // id of target project - target_path: undefined, // defaults to src_path - overwrite_newer: false, // overwrite newer versions of file at destination (destructive) - delete_missing: false, // delete files in dest that are missing from source (destructive) - backup: false, // make ~ backup files instead of overwriting changed files - }); + copy_paths_between_projects = async (opts: { + public: boolean; + // id of source project + src_project_id: string; + // list of relative paths of directories or files in the source project + src: string[]; + // id of target project + target_project_id: string; + // defaults to src_path + target_path?: string; + // overwrite newer versions of file at destination (destructive) + overwrite_newer?: boolean; + // delete files in dest that are missing from source (destructive) + delete_missing?: boolean; + // make ~ backup files instead of overwriting changed files + backup?: boolean; + }) => { const id = misc.uuid(); this.set_activity({ id, @@ -2361,8 +2379,7 @@ export class ProjectActions extends Actions { )} to a project`, }); const { src } = opts; - delete opts.src; - const with_slashes = src.map(this._convert_to_displayed_path); + const withSlashes = await this.appendSlashToDirectoryPaths(src); let dest: string | undefined = undefined; if (opts.target_path != null) { dest = opts.target_path; @@ -2374,13 +2391,13 @@ export class ProjectActions extends Actions { event: "file_action", action: "copied", dest, - files: with_slashes.slice(0, 3), + files: withSlashes.slice(0, 3), count: src.length > 3 ? src.length : undefined, project: opts.target_project_id, }); - const f = async (src_path, cb) => { - const opts0 = misc.copy(opts); - delete opts0.cb; + const f = async (src_path) => { + const opts0: any = misc.copy(opts); + delete opts0.src; opts0.src_path = src_path; // we do this for consistent semantics with file copy opts0.target_path = misc.path_to_file( @@ -2388,15 +2405,11 @@ export class ProjectActions extends Actions { misc.path_split(src_path).tail, ); opts0.timeout = 90 * 1000; - try { - await webapp_client.project_client.copy_path_between_projects(opts0); - cb(); - } catch (err) { - cb(err); - } + await webapp_client.project_client.copy_path_between_projects(opts0); }; - async.mapLimit(src, 3, f, this._finish_exec(id, opts.cb)); - } + await awaitMap(src, 5, f); + this._finish_exec(id); + }; public async rename_file(opts: { src: string; @@ -2456,19 +2469,56 @@ export class ProjectActions extends Actions { if (path == null) { return null; } - // todo: compute_server_id here and in place that does useListing! - return getFiles({ cacheId: { project_id: this.project_id }, path }); + return this.getFilesCache(path); + }; + + private getCacheId = (compute_server_id?: number) => { + return getCacheId({ + project_id: this.project_id, + compute_server_id: + compute_server_id ?? this.get_store()?.get("compute_server_id") ?? 0, + }); + }; + + private getFilesCache = ( + path: string, + compute_server_id?: number, + ): Files | null => { + return getFiles({ + cacheId: this.getCacheId(compute_server_id), + path, + }); + }; + + // using listings cache, attempt to tell if path is a directory; + // undefined if no data about path in the cache. + isDirViaCache = ( + path: string, + compute_server_id?: number, + ): boolean | undefined => { + if (!path) { + return true; + } + const { head: dir, tail: base } = misc.path_split(path); + const files = this.getFilesCache(dir, compute_server_id); + const data = files?.[base]; + if (data == null) { + return undefined; + } else { + return !!data.isdir; + } }; // return true if exists and is a directory - isdir = async (path: string): Promise => { + // error if doesn't exist or can't find out. + // Use isDirViaCache for more of a fast hint. + isdir = async ( + path: string, + compute_server_id?: number, + ): Promise => { if (path == "") return true; // easy special case - try { - const stats = await this.fs().stat(path); - return stats.isDirectory(); - } catch (_) { - return false; - } + const stats = await this.fs(compute_server_id).stat(path); + return stats.isDirectory(); }; public async move_files(opts: { diff --git a/src/packages/frontend/project_store.ts b/src/packages/frontend/project_store.ts index 388a9fb74a..78fb2e0aea 100644 --- a/src/packages/frontend/project_store.ts +++ b/src/packages/frontend/project_store.ts @@ -75,7 +75,6 @@ export interface ProjectStoreState { show_upload: boolean; create_file_alert: boolean; - displayed_listing?: any; // computed(object), configuration?: ProjectConfiguration; configuration_loading: boolean; // for UI feedback available_features?: TypedMap; @@ -292,7 +291,6 @@ export class ProjectStore extends Store { just_closed_files: immutable.List([]), show_upload: false, create_file_alert: false, - displayed_listing: undefined, // computed(object), show_masked: true, configuration: undefined, configuration_loading: false, // for UI feedback From 57934c7d0c9616359d0b25f720a6d63fca838e49 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 29 Jul 2025 16:33:29 +0000 Subject: [PATCH 127/798] refactor explorer listings so that action bar works again --- .../frontend/project/explorer/action-bar.tsx | 3 +- .../frontend/project/explorer/explorer.tsx | 119 +++++++++++++----- .../explorer/file-listing/file-listing.tsx | 87 ++----------- .../explorer/file-listing/file-row.tsx | 7 +- src/packages/frontend/project_actions.ts | 2 +- src/packages/util/types/directory-listing.ts | 7 +- 6 files changed, 111 insertions(+), 114 deletions(-) diff --git a/src/packages/frontend/project/explorer/action-bar.tsx b/src/packages/frontend/project/explorer/action-bar.tsx index fa82fc17df..f427ff1864 100644 --- a/src/packages/frontend/project/explorer/action-bar.tsx +++ b/src/packages/frontend/project/explorer/action-bar.tsx @@ -19,6 +19,7 @@ import { file_actions, type ProjectActions } from "@cocalc/frontend/project_stor import * as misc from "@cocalc/util/misc"; import { COLORS } from "@cocalc/util/theme"; import { useProjectContext } from "../context"; +import { DirectoryListingEntry } from "@cocalc/util/types"; const ROW_INFO_STYLE = { color: COLORS.GRAY, @@ -29,7 +30,7 @@ const ROW_INFO_STYLE = { interface Props { project_id?: string; checked_files: immutable.Set; - listing: { name: string; isdir: boolean }[]; + listing: DirectoryListingEntry[]; current_path?: string; project_map?; images?: ComputeImages; diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index d81087b553..9a0d7bf19b 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -34,6 +34,13 @@ import { SearchBar } from "./search-bar"; import ExplorerTour from "./tour/tour"; import { dirname, join } from "path"; import { redux, useTypedRedux } from "@cocalc/frontend/app-framework"; +import useFs from "@cocalc/frontend/project/listing/use-fs"; +import useListing, { + type SortField, +} from "@cocalc/frontend/project/listing/use-listing"; +import filterListing from "@cocalc/frontend/project/listing/filter-listing"; +import ShowError from "@cocalc/frontend/components/error"; +import { MainConfiguration } from "@cocalc/frontend/project_configuration"; const FLEX_ROW_STYLE = { display: "flex", @@ -51,14 +58,34 @@ const ERROR_STYLE: CSSProperties = { boxShadow: "5px 5px 5px grey", } as const; +function sortDesc(active_file_sort?): { + sortField: SortField; + sortDirection: "asc" | "desc"; +} { + const { column_name, is_descending } = active_file_sort?.toJS() ?? { + column_name: "name", + is_descending: false, + }; + if (column_name == "time") { + return { + sortField: "mtime", + sortDirection: is_descending ? "asc" : "desc", + }; + } + return { + sortField: column_name, + sortDirection: is_descending ? "desc" : "asc", + }; +} + export function Explorer() { const { actions, project_id } = useProjectContext(); + const newFileRef = useRef(null); const searchAndTerminalBar = useRef(null); const fileListingRef = useRef(null); const currentDirectoryRef = useRef(null); const miscButtonsRef = useRef(null); - const listingRef = useRef(null); const activity = useTypedRedux({ project_id }, "activity")?.toJS(); const available_features = useTypedRedux( @@ -76,7 +103,7 @@ export function Explorer() { { project_id }, "file_creation_error", ); - const file_search = useTypedRedux({ project_id }, "file_search"); + const file_search = useTypedRedux({ project_id }, "file_search") ?? ""; const show_custom_software_reset = useTypedRedux( { project_id }, "show_custom_software_reset", @@ -88,6 +115,34 @@ export function Explorer() { const images = useTypedRedux("compute_images", "images"); + const active_file_sort = useTypedRedux({ project_id }, "active_file_sort"); + const fs = useFs({ project_id }); + let { listing, error: listingError } = useListing({ + fs, + path: current_path, + ...sortDesc(active_file_sort), + cacheId: actions?.getCacheId(compute_server_id), + }); + const showHidden = useTypedRedux({ project_id }, "show_hidden"); + const showMasked = useTypedRedux({ project_id }, "show_masked"); + + listing = listingError + ? null + : filterListing({ + listing, + search: file_search, + showHidden, + showMasked, + }); + + useEffect(() => { + actions?.setState({ numDisplayedFiles: listing?.length ?? 0 }); + }, [listing?.length]); + + if (listingError) { + return ; + } + if (actions == null || project_map == null) { return ; } @@ -108,7 +163,7 @@ export function Explorer() { } else if (e.key == "Enter") { const n = redux.getProjectStore(project_id).get("selected_file_index") ?? 0; - const x = listingRef.current?.[n]; + const x = listing?.[n]; if (x != null) { const { isdir, name } = x; const path = join(current_path, name); @@ -118,7 +173,7 @@ export function Explorer() { actions.open_file({ path, foreground: !e.ctrlKey }); } if (!e.ctrlKey) { - actions.set_file_search(""); + setTimeout(() => actions.set_file_search(""), 10); actions.clear_selected_file_index(); } } @@ -314,18 +369,20 @@ export function Explorer() { minWidth: "20em", }} > - + {listing != null && ( + + )}
- + {listing == null ? ( + + ) : ( + + )}
; - listing: any[]; + listing: DirectoryListingEntry[]; file_search: string; checked_files: immutable.Set; current_path: string; @@ -46,72 +39,11 @@ interface Props { shiftIsDown: boolean; configuration_main?: MainConfiguration; isRunning?: boolean; // true if this project is running - stale?: boolean; - - listingRef; -} - -function sortDesc(active_file_sort?): { - sortField: SortField; - sortDirection: "asc" | "desc"; -} { - const { column_name, is_descending } = active_file_sort?.toJS() ?? { - column_name: "name", - is_descending: false, - }; - if (column_name == "time") { - return { - sortField: "mtime", - sortDirection: is_descending ? "asc" : "desc", - }; - } - return { - sortField: column_name, - sortDirection: is_descending ? "desc" : "asc", - }; } -export function FileListing(props) { - const { project_id } = props; - const active_file_sort = useTypedRedux({ project_id }, "active_file_sort"); - const path = props.current_path; - const fs = useFs({ project_id }); - let { listing, error } = useListing({ - fs, - path, - ...sortDesc(props.active_file_sort), - cacheId: props.actions.getCacheId(), - }); - const showHidden = useTypedRedux({ project_id }, "show_hidden"); - const showMasked = useTypedRedux({ project_id }, "show_masked"); - - props.listingRef.current = listing = error - ? null - : filterListing({ - listing, - search: props.file_search, - showHidden, - showMasked, - }); - - useEffect(() => { - props.actions.setState({ numDisplayedFiles: listing?.length ?? 0 }); - }, [listing?.length]); - - if (error) { - return ; - } - if (listing == null) { - return ; - } - return ; -} - -function FileListing0({ +export function FileListing({ actions, - name, - active_file_sort, listing, checked_files, current_path, @@ -120,11 +52,12 @@ function FileListing0({ configuration_main, file_search = "", stale, - // show_masked, }: Props) { + const active_file_sort = useTypedRedux({ project_id }, "active_file_sort"); const computeServerId = useTypedRedux({ project_id }, "compute_server_id"); const selected_file_index = useTypedRedux({ project_id }, "selected_file_index") ?? 0; + const name = actions.name; function render_row( name, @@ -132,8 +65,6 @@ function FileListing0({ time, mask, isdir, - display_name, - public_data, issymlink, index: number, link_target?: string, // if given, is a known symlink to this file @@ -145,7 +76,6 @@ function FileListing0({ ; + } + function render_rows(): Rendered { return ( = [ interface Props { isdir: boolean; name: string; - display_name: string; // if given, will display this, and will show true filename in popover + display_name?: string; // if given, will display this, and will show true filename in popover size: number; // sometimes is NOT known! time: number; issymlink: boolean; @@ -46,7 +46,6 @@ interface Props { selected: boolean; color: string; mask: boolean; - public_data: object; is_public: boolean; current_path: string; actions: ProjectActions; @@ -211,7 +210,9 @@ export const FileRow: React.FC = React.memo((props) => { explicit: true, }); if (foreground) { - props.actions.set_file_search(""); + // delay slightly since it looks weird to see the full listing right when you click on a file + const actions = props.actions; + setTimeout(() => actions.set_file_search(""), 10); } } } diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index 296678b9cb..316cc52280 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -2472,7 +2472,7 @@ export class ProjectActions extends Actions { return this.getFilesCache(path); }; - private getCacheId = (compute_server_id?: number) => { + getCacheId = (compute_server_id?: number) => { return getCacheId({ project_id: this.project_id, compute_server_id: diff --git a/src/packages/util/types/directory-listing.ts b/src/packages/util/types/directory-listing.ts index 4b9f58da66..3352a99268 100644 --- a/src/packages/util/types/directory-listing.ts +++ b/src/packages/util/types/directory-listing.ts @@ -2,8 +2,11 @@ export interface DirectoryListingEntry { name: string; isdir?: boolean; issymlink?: boolean; - link_target?: string; // set if issymlink is true and we're able to determine the target of the link - size?: number; // bytes for file, number of entries for directory (*including* . and ..). + // set if issymlink is true and we're able to determine the target of the link + link_target?: string; + // bytes for file, number of entries for directory (*including* . and ..). + size?: number; mtime?: number; error?: string; + mask?: boolean; } From df6f9db0a739451c6e1bee2c76475278fb324559 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 29 Jul 2025 19:12:32 +0000 Subject: [PATCH 128/798] fix crash in socket reconnect after close --- src/packages/conat/socket/base.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/packages/conat/socket/base.ts b/src/packages/conat/socket/base.ts index 24d57ac148..3f98df515f 100644 --- a/src/packages/conat/socket/base.ts +++ b/src/packages/conat/socket/base.ts @@ -116,14 +116,16 @@ export abstract class ConatSocketBase extends EventEmitter { } if (this.reconnection) { setTimeout(() => { - this.connect(); + if (this.state != "closed") { + this.connect(); + } }, RECONNECT_DELAY); } }; connect = async () => { - if (this.state != "disconnected") { - // already connected + if (this.state != "disconnected" || !this.client) { + // already connected or closed return; } this.setState("connecting"); From ecc2f494e29926dba757ca484a9e5d5d1b5a7ded Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 29 Jul 2025 19:29:36 +0000 Subject: [PATCH 129/798] action bar -- show even if project is not running --- .../frontend/project/explorer/action-bar.tsx | 105 ++++++++++-------- 1 file changed, 57 insertions(+), 48 deletions(-) diff --git a/src/packages/frontend/project/explorer/action-bar.tsx b/src/packages/frontend/project/explorer/action-bar.tsx index f427ff1864..98d967f393 100644 --- a/src/packages/frontend/project/explorer/action-bar.tsx +++ b/src/packages/frontend/project/explorer/action-bar.tsx @@ -15,7 +15,10 @@ import { CustomSoftwareInfo } from "@cocalc/frontend/custom-software/info-bar"; import { type ComputeImages } from "@cocalc/frontend/custom-software/init"; import { IS_MOBILE } from "@cocalc/frontend/feature"; import { labels } from "@cocalc/frontend/i18n"; -import { file_actions, type ProjectActions } from "@cocalc/frontend/project_store"; +import { + file_actions, + type ProjectActions, +} from "@cocalc/frontend/project_store"; import * as misc from "@cocalc/util/misc"; import { COLORS } from "@cocalc/util/theme"; import { useProjectContext } from "../context"; @@ -40,22 +43,33 @@ interface Props { project_is_running?: boolean; } -export function ActionBar(props: Props) { +export function ActionBar({ + project_id, + checked_files, + listing, + current_path, + project_map, + images, + actions, + available_features, + show_custom_software_reset, + project_is_running, +}: Props) { const intl = useIntl(); const [showLabels, setShowLabels] = useState(true); const { mainWidthPx } = useProjectContext(); const buttonRef = useRef(null); - const widthThld = useRef(0); + const tableHeaderWidth = useRef(0); const student_project_functionality = useStudentProjectFunctionality( - props.actions.project_id, + actions.project_id, ); if (student_project_functionality.disableActions) { return
; } useEffect(() => { - const btnbar = buttonRef.current; - if (btnbar == null) return; + const buttonBar = buttonRef.current; + if (buttonBar == null) return; const resizeObserver = new ResizeObserver( throttle( (entries) => { @@ -65,8 +79,8 @@ export function ActionBar(props: Props) { // (e.g. german buttons were cutoff all the time), but could need more tweaking if (showLabels && width > mainWidthPx + 100) { setShowLabels(false); - widthThld.current = width; - } else if (!showLabels && width < widthThld.current - 1) { + tableHeaderWidth.current = width; + } else if (!showLabels && width < tableHeaderWidth.current - 1) { setShowLabels(true); } } @@ -75,22 +89,20 @@ export function ActionBar(props: Props) { { leading: false, trailing: true }, ), ); - resizeObserver.observe(btnbar); + resizeObserver.observe(buttonBar); return () => { resizeObserver.disconnect(); }; }, [mainWidthPx, buttonRef.current]); function clear_selection(): void { - props.actions.set_all_files_unchecked(); + actions.set_all_files_unchecked(); } function check_all_click_handler(): void { - if (props.checked_files.size === 0) { - props.actions.set_file_list_checked( - props.listing.map((file) => - misc.path_to_file(props.current_path ?? "", file.name), - ), + if (checked_files.size === 0) { + actions.set_file_list_checked( + listing.map((file) => misc.path_to_file(current_path ?? "", file.name)), ); } else { clear_selection(); @@ -98,11 +110,11 @@ export function ActionBar(props: Props) { } function render_check_all_button(): React.JSX.Element | undefined { - if (props.listing.length === 0) { + if (listing.length === 0) { return; } - const checked = props.checked_files.size > 0; + const checked = checked_files.size > 0; const button_text = intl.formatMessage( { id: "project.explorer.action-bar.check_all.button", @@ -114,10 +126,10 @@ export function ActionBar(props: Props) { ); let button_icon; - if (props.checked_files.size === 0) { + if (checked_files.size === 0) { button_icon = "square-o"; } else { - if (props.checked_files.size >= props.listing.length) { + if (checked_files.size >= listing.length) { button_icon = "check-square-o"; } else { button_icon = "minus-square-o"; @@ -136,11 +148,11 @@ export function ActionBar(props: Props) { } function render_currently_selected(): React.JSX.Element | undefined { - if (props.listing.length === 0) { + if (listing.length === 0) { return; } - const checked = props.checked_files.size; - const total = props.listing.length; + const checked = checked_files.size; + const total = listing.length; const style = ROW_INFO_STYLE; if (checked === 0) { @@ -186,12 +198,12 @@ export function ActionBar(props: Props) { function render_action_button(name: string): React.JSX.Element { const disabled = isDisabledSnapshots(name) && - (props.current_path != null - ? props.current_path.startsWith(".snapshots") + (current_path != null + ? current_path.startsWith(".snapshots") : undefined); const obj = file_actions[name]; const handle_click = (_e: React.MouseEvent) => { - props.actions.set_file_action(name); + actions.set_file_action(name); }; return ( @@ -213,16 +225,13 @@ export function ActionBar(props: Props) { | "copy" | "share" )[]; - if (!props.project_is_running) { + if (checked_files.size === 0) { return; - } - if (props.checked_files.size === 0) { - return; - } else if (props.checked_files.size === 1) { + } else if (checked_files.size === 1) { let isdir; - const item = props.checked_files.first(); - for (const file of props.listing) { - if (misc.path_to_file(props.current_path ?? "", file.name) === item) { + const item = checked_files.first(); + for (const file of listing) { + if (misc.path_to_file(current_path ?? "", file.name) === item) { ({ isdir } = file); } } @@ -246,25 +255,25 @@ export function ActionBar(props: Props) { } function render_button_area(): React.JSX.Element | undefined { - if (props.checked_files.size === 0) { + if (checked_files.size === 0) { if ( - props.project_id == null || - props.images == null || - props.project_map == null || - props.available_features == null + project_id == null || + images == null || + project_map == null || + available_features == null ) { return; } return ( ); @@ -272,7 +281,7 @@ export function ActionBar(props: Props) { return render_action_buttons(); } } - if (props.checked_files.size === 0 && IS_MOBILE) { + if (checked_files.size === 0 && IS_MOBILE) { return null; } return ( @@ -280,13 +289,13 @@ export function ActionBar(props: Props) {
- {props.project_is_running ? render_check_all_button() : undefined} + {render_check_all_button()} {render_button_area()}
- {props.project_is_running ? render_currently_selected() : undefined} + {render_currently_selected()}
); From b297394beea6f4ac7b0905f859c7502cd2e4044d Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 29 Jul 2025 21:08:48 +0000 Subject: [PATCH 130/798] ts --- .../frontend/project/explorer/explorer.tsx | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index 9a0d7bf19b..0289ce991f 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -139,16 +139,11 @@ export function Explorer() { actions?.setState({ numDisplayedFiles: listing?.length ?? 0 }); }, [listing?.length]); - if (listingError) { - return ; - } - - if (actions == null || project_map == null) { - return ; - } - useEffect(() => { const handle_files_key_down = (e): void => { + if (actions == null) { + return; + } if (e.key === "Shift") { setShiftIsDown(true); } else if (e.key == "ArrowUp") { @@ -194,6 +189,14 @@ export function Explorer() { }; }, [project_id, current_path]); + if (listingError) { + return ; + } + + if (actions == null || project_map == null) { + return ; + } + const create_file = (ext, switch_over) => { if (switch_over == undefined) { switch_over = true; @@ -471,7 +474,9 @@ export function Explorer() { className="smc-vfill" > {listing == null ? ( - +
+ +
) : ( Date: Wed, 30 Jul 2025 01:07:38 +0000 Subject: [PATCH 131/798] typo --- src/packages/conat/core/cluster.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/packages/conat/core/cluster.ts b/src/packages/conat/core/cluster.ts index 863f2cd943..e4e8c8efbe 100644 --- a/src/packages/conat/core/cluster.ts +++ b/src/packages/conat/core/cluster.ts @@ -272,8 +272,8 @@ export async function trimClusterStreams( minAge: number, ): Promise<{ seqsInterest: number[]; seqsSticky: number[] }> { const { interest, sticky } = streams; - // First deal with interst - // we iterate over the interest stream checking for subjects + // First deal with interest. + // We iterate over the interest stream checking for subjects // with no current interest at all; in such cases it is safe // to purge them entirely from the stream. const seqs: number[] = []; From 47699d88080a4448ce449cfd538300551a2e8709 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 30 Jul 2025 03:11:12 +0000 Subject: [PATCH 132/798] sync: fix hash_of_saved_version --- src/packages/sync/editor/generic/sync-doc.ts | 31 ++++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 2295693231..a69ae7a801 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -1274,7 +1274,7 @@ export class SyncDoc extends EventEmitter { try { stats = await this.fs.stat(this.path); } catch (err) { - this.lastDiskValue = undefined; // nonexistent or don't know + this.valueOnDisk = undefined; // nonexistent or don't know if (err.code == "ENOENT") { // path does not exist -- nothing further to do return false; @@ -2243,7 +2243,7 @@ export class SyncDoc extends EventEmitter { let contents; try { contents = await this.fs.readFile(this.path, "utf8"); - this.lastDiskValue = contents; + this.valueOnDisk = contents; dbg("file exists"); size = contents.length; this.from_str(contents); @@ -2286,14 +2286,15 @@ export class SyncDoc extends EventEmitter { return this.hasUnsavedChanges(); }; - // Returns hash of last version saved to disk (as far as we know). + // Returns hash of last version that we saved to disk or undefined + // if we haven't saved yet. + // NOTE: this does not take into account saving by another client + // anymore; it used to, but that made things much more complicated. hash_of_saved_version = (): number | undefined => { - if (!this.isReady()) { + if (!this.isReady() || this.valueOnDisk == null) { return; } - return this.syncstring_table_get_one().getIn(["save", "hash"]) as - | number - | undefined; + return hash_string(this.valueOnDisk); }; /* Return hash of the live version of the document, @@ -2359,9 +2360,13 @@ export class SyncDoc extends EventEmitter { return true; }; - private lastDiskValue: string | undefined = undefined; + // valueOnDisk = value of the file on disk, if known. If there's an + // event indicating what was on disk may have changed, this + // this.valueOnDisk is deleted until the new version is loaded. + private valueOnDisk: string | undefined = undefined; + private hasUnsavedChanges = (): boolean => { - return this.lastDiskValue != this.to_str(); + return this.valueOnDisk != this.to_str(); }; writeFile = async () => { @@ -2389,7 +2394,7 @@ export class SyncDoc extends EventEmitter { await this.fs.writeFile(this.path, value); const lastChanged = this.last_changed(); await this.fs.utimes(this.path, lastChanged / 1000, lastChanged / 1000); - this.lastDiskValue = value; + this.valueOnDisk = value; }; /* Initiates a save of file to disk, then waits for the @@ -2695,10 +2700,10 @@ export class SyncDoc extends EventEmitter { this.emit("watching"); for await (const { eventType, ignore } of this.fileWatcher) { if (this.isClosed()) return; - // we don't know what's on disk anymore, - this.lastDiskValue = undefined; - //console.log("got change", eventType); if (!ignore) { + // we don't know what's on disk anymore, + this.valueOnDisk = undefined; + // and we should find out! this.readFileDebounced(); } if (eventType == "rename") { From 8752002e0c9b0f34d70fc7a031a08e325dec4b6d Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 30 Jul 2025 03:28:42 +0000 Subject: [PATCH 133/798] sync: missing save-to-disk event --- src/packages/sync/editor/generic/sync-doc.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index a69ae7a801..abc9515e91 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -2395,6 +2395,7 @@ export class SyncDoc extends EventEmitter { const lastChanged = this.last_changed(); await this.fs.utimes(this.path, lastChanged / 1000, lastChanged / 1000); this.valueOnDisk = value; + this.emit("save-to-disk"); }; /* Initiates a save of file to disk, then waits for the From b8eb20c7354290b51984e458c604f4f414f77078 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 30 Jul 2025 04:08:14 +0000 Subject: [PATCH 134/798] sync: reimplement read-write versus read-only handling --- src/packages/conat/files/fs.ts | 2 +- .../frame-editors/frame-tree/save-button.tsx | 4 +- .../frame-editors/frame-tree/title-bar.tsx | 4 +- src/packages/sync/editor/generic/sync-doc.ts | 90 ++++++++++++++----- 4 files changed, 74 insertions(+), 26 deletions(-) diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index c7ce7280f8..ea7f567f10 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -140,7 +140,7 @@ interface IStats { birthtime: Date; } -class Stats { +export class Stats { dev: number; ino: number; mode: number; diff --git a/src/packages/frontend/frame-editors/frame-tree/save-button.tsx b/src/packages/frontend/frame-editors/frame-tree/save-button.tsx index 4874a04c2b..0a9585fe4d 100644 --- a/src/packages/frontend/frame-editors/frame-tree/save-button.tsx +++ b/src/packages/frontend/frame-editors/frame-tree/save-button.tsx @@ -47,7 +47,7 @@ export function SaveButton({ const intl = useIntl(); const label = useMemo(() => { - if (!no_labels) { + if (!no_labels || read_only) { return intl.formatMessage(labels.frame_editors_title_bar_save_label, { type: is_public ? "is_public" : read_only ? "read_only" : "save", }); @@ -67,7 +67,7 @@ export function SaveButton({ ); function renderLabel() { - if (!no_labels && label) { + if (label) { return {` ${label}`}; } } diff --git a/src/packages/frontend/frame-editors/frame-tree/title-bar.tsx b/src/packages/frontend/frame-editors/frame-tree/title-bar.tsx index 67bb8337b8..eed5a2bdce 100644 --- a/src/packages/frontend/frame-editors/frame-tree/title-bar.tsx +++ b/src/packages/frontend/frame-editors/frame-tree/title-bar.tsx @@ -717,8 +717,8 @@ export function FrameTitleBar(props: FrameTitleBarProps) { label === APPLICATION_MENU ? manageCommands.applicationMenuTitle() : isIntlMessage(label) - ? intl.formatMessage(label) - : label + ? intl.formatMessage(label) + : label } items={v} /> diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index abc9515e91..2fe3749193 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -40,6 +40,8 @@ import { decodeUUIDtoNum, } from "@cocalc/util/compute/manager"; +const STAT_DEBOUNCE = 10000; + import { DEFAULT_SNAPSHOT_INTERVAL } from "@cocalc/util/db-schema/syncstring-schema"; type XPatch = any; @@ -83,7 +85,7 @@ import { isTestClient, patch_cmp } from "./util"; import mergeDeep from "@cocalc/util/immutable-deep-merge"; import { JUPYTER_SYNCDB_EXTENSIONS } from "@cocalc/util/jupyter/names"; import { LegacyHistory } from "./legacy"; -import { type Filesystem } from "@cocalc/conat/files/fs"; +import { type Filesystem, type Stats } from "@cocalc/conat/files/fs"; import { getLogger } from "@cocalc/conat/client"; const DEBUG = false; @@ -1267,12 +1269,10 @@ export class SyncDoc extends EventEmitter { }; private loadFromDiskIfNewer = async (): Promise => { - // [ ] TODO: readonly handling... - if (this.fs == null) throw Error("bug"); const dbg = this.dbg("loadFromDiskIfNewer"); let stats; try { - stats = await this.fs.stat(this.path); + stats = await this.stat(); } catch (err) { this.valueOnDisk = undefined; // nonexistent or don't know if (err.code == "ENOENT") { @@ -2236,7 +2236,6 @@ export class SyncDoc extends EventEmitter { }; readFile = reuseInFlight(async (): Promise => { - if (this.fs == null) throw Error("bug"); const dbg = this.dbg("readFile"); let size: number; @@ -2264,12 +2263,47 @@ export class SyncDoc extends EventEmitter { }); is_read_only = (): boolean => { - // [ ] TODO - return this.syncstring_table_get_one().get("read_only"); + if (this.stats) { + return isReadOnlyForOwner(this.stats); + } else { + return false; + } + }; + + private stats?: Stats; + stat = async (): Promise => { + const prevStats = this.stats; + this.stats = (await this.fs.stat(this.path)) as Stats; + if (prevStats?.mode != this.stats.mode) { + this.emit("metadata-change"); + } + return this.stats; }; + debouncedStat = debounce( + async () => { + try { + await this.stat(); + } catch {} + }, + STAT_DEBOUNCE, + { leading: true, trailing: true }, + ); + wait_until_read_only_known = async (): Promise => { - // [ ] TODO + await until(async () => { + if (this.isClosed()) { + return true; + } + if (this.stats != null) { + return true; + } + try { + await this.stat(); + return true; + } catch {} + return false; + }); }; /* Returns true if the current live version of this document has @@ -2376,9 +2410,14 @@ export class SyncDoc extends EventEmitter { return; } dbg(); - if (this.fs == null) { - throw Error("bug"); + if (this.is_read_only()) { + await this.stat(); + if (this.is_read_only()) { + // it is definitely still read only. + return; + } } + const value = this.to_str(); // include {ignore:true} with events for this long, // so no clients waste resources loading in response to us saving @@ -2391,7 +2430,17 @@ export class SyncDoc extends EventEmitter { } if (this.isClosed()) return; this.last_save_to_disk_time = new Date(); - await this.fs.writeFile(this.path, value); + try { + await this.fs.writeFile(this.path, value); + } catch (err) { + if (err.code == "EACCES") { + try { + // update read only knowledge -- that may have caused save error. + await this.stat(); + } catch {} + } + throw err; + } const lastChanged = this.last_changed(); await this.fs.utimes(this.path, lastChanged / 1000, lastChanged / 1000); this.valueOnDisk = value; @@ -2674,6 +2723,7 @@ export class SyncDoc extends EventEmitter { try { this.emit("handle-file-change"); await this.readFile(); + await this.stat(); } catch {} }, WATCH_DEBOUNCE, @@ -2685,9 +2735,6 @@ export class SyncDoc extends EventEmitter { private fileWatcher?: any; private initFileWatcher = async () => { - if (this.fs == null) { - throw Error("this.fs must be defined"); - } // use this.fs interface to watch path for changes -- we try once: try { this.fileWatcher = await this.fs.watch(this.path, { unique: true }); @@ -2706,6 +2753,8 @@ export class SyncDoc extends EventEmitter { this.valueOnDisk = undefined; // and we should find out! this.readFileDebounced(); + } else { + this.debouncedStat(); } if (eventType == "rename") { break; @@ -2742,11 +2791,8 @@ export class SyncDoc extends EventEmitter { // false if it definitely does not, and throws exception otherwise, // e.g., network error. private fileExists = async (): Promise => { - if (this.fs == null) { - throw Error("bug -- fs must be defined"); - } try { - await this.fs.stat(this.path); + await this.stat(); return true; } catch (err) { if (err.code == "ENOENT") { @@ -2759,9 +2805,6 @@ export class SyncDoc extends EventEmitter { private closeIfFileDeleted = async () => { if (this.isClosed()) return; - if (this.fs == null) { - throw Error("bug -- fs must be defined"); - } const start = Date.now(); const threshold = this.deletedThreshold ?? DELETED_THRESHOLD; while (true) { @@ -2808,3 +2851,8 @@ function isCompletePatchStream(dstream) { } return false; } + +function isReadOnlyForOwner(stats): boolean { + // 0o200 is owner write permission + return (stats.mode & 0o200) === 0; +} From 3520607562d98b2d6ec2fc137bc0e9bb5b39e58c Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 30 Jul 2025 04:19:34 +0000 Subject: [PATCH 135/798] copy between projects -- hide activity indicator when done --- src/packages/frontend/project_actions.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index 316cc52280..683b63c1ec 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -2407,8 +2407,8 @@ export class ProjectActions extends Actions { opts0.timeout = 90 * 1000; await webapp_client.project_client.copy_path_between_projects(opts0); }; - await awaitMap(src, 5, f); - this._finish_exec(id); + await awaitMap(withSlashes, 5, f); + this.set_activity({ id, stop: "" }); }; public async rename_file(opts: { From 78ed4917d3319f327f2c2e0d91e3c92d201e0759 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 30 Jul 2025 18:18:39 +0000 Subject: [PATCH 136/798] state issue with listing --- src/packages/frontend/project/explorer/explorer.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index 0289ce991f..50fc1b46da 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -140,6 +140,9 @@ export function Explorer() { }, [listing?.length]); useEffect(() => { + if (listing == null) { + return; + } const handle_files_key_down = (e): void => { if (actions == null) { return; @@ -187,7 +190,7 @@ export function Explorer() { $(window).off("keydown", handle_files_key_down); $(window).off("keyup", handle_files_key_up); }; - }, [project_id, current_path]); + }, [project_id, current_path, listing]); if (listingError) { return ; From a12ea7c32dbfb266ce126ae52fa17ff89b79a1e8 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 30 Jul 2025 19:00:34 +0000 Subject: [PATCH 137/798] focus explorer filter on directory change --- .../frontend/components/search-input.tsx | 80 +++++++++++-------- .../frontend/project/explorer/explorer.tsx | 4 + .../frontend/project/explorer/search-bar.tsx | 1 + 3 files changed, 50 insertions(+), 35 deletions(-) diff --git a/src/packages/frontend/components/search-input.tsx b/src/packages/frontend/components/search-input.tsx index e3839ca547..446c788d96 100644 --- a/src/packages/frontend/components/search-input.tsx +++ b/src/packages/frontend/components/search-input.tsx @@ -10,13 +10,7 @@ */ import { Input, InputRef } from "antd"; - -import { - React, - useEffect, - useRef, - useState, -} from "@cocalc/frontend/app-framework"; +import { CSSProperties, useEffect, useRef, useState } from "react"; interface Props { size?; @@ -31,21 +25,37 @@ interface Props { on_down?: () => void; on_up?: () => void; on_escape?: (value: string) => void; - style?: React.CSSProperties; - input_class?: string; + style?: CSSProperties; autoFocus?: boolean; autoSelect?: boolean; placeholder?: string; - focus?: number; // if this changes, focus the search box. + focus?; // if this changes, focus the search box. status?: "warning" | "error"; } -export const SearchInput: React.FC = React.memo((props) => { - const [value, setValue] = useState( - props.value ?? props.default_value ?? "", - ); +export function SearchInput({ + size, + default_value, + value: value0, + on_change, + on_clear, + on_submit, + buttonAfter, + disabled, + clear_on_submit, + on_down, + on_up, + on_escape, + style, + autoFocus, + autoSelect, + placeholder, + focus, + status, +}: Props) { + const [value, setValue] = useState(value0 ?? default_value ?? ""); // if value changes, we update as well! - useEffect(() => setValue(props.value ?? ""), [props.value]); + useEffect(() => setValue(value ?? ""), [value]); const [ctrl_down, set_ctrl_down] = useState(false); const [shift_down, set_shift_down] = useState(false); @@ -53,7 +63,7 @@ export const SearchInput: React.FC = React.memo((props) => { const input_ref = useRef(null); useEffect(() => { - if (props.autoSelect && input_ref.current) { + if (autoSelect && input_ref.current) { try { input_ref.current?.select(); } catch (_) {} @@ -71,20 +81,20 @@ export const SearchInput: React.FC = React.memo((props) => { function clear_value(): void { setValue(""); - props.on_change?.("", get_opts()); - props.on_clear?.(); + on_change?.("", get_opts()); + on_clear?.(); } function submit(e?): void { if (e != null) { e.preventDefault(); } - if (typeof props.on_submit === "function") { - props.on_submit(value, get_opts()); + if (typeof on_submit === "function") { + on_submit(value, get_opts()); } - if (props.clear_on_submit) { + if (clear_on_submit) { clear_value(); - props.on_change?.(value, get_opts()); + on_change?.(value, get_opts()); } } @@ -94,10 +104,10 @@ export const SearchInput: React.FC = React.memo((props) => { escape(); break; case 40: - props.on_down?.(); + on_down?.(); break; case 38: - props.on_up?.(); + on_up?.(); break; case 17: set_ctrl_down(true); @@ -120,35 +130,35 @@ export const SearchInput: React.FC = React.memo((props) => { } function escape(): void { - if (typeof props.on_escape === "function") { - props.on_escape(value); + if (typeof on_escape === "function") { + on_escape(value); } clear_value(); } return ( { e.preventDefault(); const value = e.target?.value ?? ""; setValue(value); - props.on_change?.(value, get_opts()); + on_change?.(value, get_opts()); if (!value) clear_value(); }} onKeyDown={key_down} onKeyUp={key_up} - disabled={props.disabled} - enterButton={props.buttonAfter} - status={props.status} + disabled={disabled} + enterButton={buttonAfter} + status={status} /> ); -}); +} diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index 50fc1b46da..e682bfa1f4 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -159,6 +159,10 @@ export function Explorer() { } else if (e.key == "ArrowDown") { actions.increment_selected_file_index(); } else if (e.key == "Enter") { + if (file_search.startsWith("/")) { + // running a terminal command + return; + } const n = redux.getProjectStore(project_id).get("selected_file_index") ?? 0; const x = listing?.[n]; diff --git a/src/packages/frontend/project/explorer/search-bar.tsx b/src/packages/frontend/project/explorer/search-bar.tsx index 7070669954..535e426536 100644 --- a/src/packages/frontend/project/explorer/search-bar.tsx +++ b/src/packages/frontend/project/explorer/search-bar.tsx @@ -274,6 +274,7 @@ export const SearchBar = memo( on_submit={search_submit} on_clear={on_clear} disabled={disabled || !!ext_selection} + focus={current_path} /> {render_file_creation_error()} {render_help_info()} From 850ce5d4e3fb81fbf9683042ce4a565946f93293 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 30 Jul 2025 19:37:07 +0000 Subject: [PATCH 138/798] masked files -- make that work again --- .../project/explorer/compute-file-masks.ts | 20 +++++++++---------- .../frontend/project/explorer/explorer.tsx | 1 + .../frontend/project/listing/use-listing.ts | 10 +++++++++- .../frontend/project/page/flyouts/files.tsx | 4 ++-- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/packages/frontend/project/explorer/compute-file-masks.ts b/src/packages/frontend/project/explorer/compute-file-masks.ts index 6b8e4d95d0..f7162818f5 100644 --- a/src/packages/frontend/project/explorer/compute-file-masks.ts +++ b/src/packages/frontend/project/explorer/compute-file-masks.ts @@ -54,7 +54,7 @@ const MASKED_FILE_EXTENSIONS = { * the general outcome of this function is to set for some file entry objects * in "listing" the attribute .mask=true */ -export function compute_file_masks(listing: DirectoryListing): void { +export function computeFileMasks(listing: DirectoryListing): void { // map filename to file for easier lookup const filename_map: { [name: string]: DirectoryListingEntry } = dict( listing.map((item) => [item.name, item]), @@ -75,29 +75,29 @@ export function compute_file_masks(listing: DirectoryListing): void { for (let mask_ext of MASKED_FILE_EXTENSIONS[ext] ?? []) { // check each possible compiled extension - let bn; // derived basename + let derivedBasename; // some uppercase-strings have special meaning if (startswith(mask_ext, "NODOT")) { - bn = basename.slice(0, -1); // exclude the trailing dot + derivedBasename = basename.slice(0, -1); // exclude the trailing dot mask_ext = mask_ext.slice("NODOT".length); } else if (mask_ext.indexOf("FILENAME") >= 0) { - bn = mask_ext.replace("FILENAME", filename); + derivedBasename = mask_ext.replace("FILENAME", filename); mask_ext = ""; } else if (mask_ext.indexOf("BASENAME") >= 0) { - bn = mask_ext.replace("BASENAME", basename.slice(0, -1)); + derivedBasename = mask_ext.replace("BASENAME", basename.slice(0, -1)); mask_ext = ""; } else if (mask_ext.indexOf("BASEDASHNAME") >= 0) { // BASEDASHNAME is like BASENAME, but replaces spaces by dashes // https://github.com/sagemathinc/cocalc/issues/3229 const fragment = basename.slice(0, -1).replace(/ /g, "-"); - bn = mask_ext.replace("BASEDASHNAME", fragment); + derivedBasename = mask_ext.replace("BASEDASHNAME", fragment); mask_ext = ""; } else { - bn = basename; + derivedBasename = basename; } - const mask_fn = `${bn}${mask_ext}`; - if (filename_map[mask_fn] != null) { - filename_map[mask_fn].mask = true; + const maskFilename = `${derivedBasename}${mask_ext}`; + if (filename_map[maskFilename] != null) { + filename_map[maskFilename].mask = true; } } } diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index e682bfa1f4..edb7cf3528 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -122,6 +122,7 @@ export function Explorer() { path: current_path, ...sortDesc(active_file_sort), cacheId: actions?.getCacheId(compute_server_id), + mask: true, }); const showHidden = useTypedRedux({ project_id }, "show_hidden"); const showMasked = useTypedRedux({ project_id }, "show_masked"); diff --git a/src/packages/frontend/project/listing/use-listing.ts b/src/packages/frontend/project/listing/use-listing.ts index da5a1870ac..a4c481c52f 100644 --- a/src/packages/frontend/project/listing/use-listing.ts +++ b/src/packages/frontend/project/listing/use-listing.ts @@ -12,6 +12,7 @@ import { type FilesystemClient } from "@cocalc/conat/files/fs"; import { type ConatError } from "@cocalc/conat/core/client"; import type { JSONValue } from "@cocalc/util/types"; import { getFiles, type Files } from "./use-files"; +import { computeFileMasks } from "@cocalc/frontend/project/explorer/compute-file-masks"; export type SortField = "name" | "mtime" | "size" | "type"; export type SortDirection = "asc" | "desc"; @@ -39,6 +40,7 @@ export default function useListing({ sortDirection = "asc", throttleUpdate, cacheId, + mask, }: { // fs = undefined is supported and just waits until you provide a fs that is defined fs?: FilesystemClient | null; @@ -47,6 +49,7 @@ export default function useListing({ sortDirection?: SortDirection; throttleUpdate?: number; cacheId?: JSONValue; + mask?: boolean; }): { listing: null | DirectoryListingEntry[]; error: null | ConatError; @@ -60,7 +63,7 @@ export default function useListing({ }); const listing = useMemo(() => { - return filesToListing({ files, sortField, sortDirection }); + return filesToListing({ files, sortField, sortDirection, mask }); }, [sortField, sortDirection, files]); return { listing, error, refresh }; @@ -70,10 +73,12 @@ function filesToListing({ files, sortField = "name", sortDirection = "asc", + mask, }: { files?: Files | null; sortField?: SortField; sortDirection?: SortDirection; + mask?: boolean; }): null | DirectoryListingEntry[] { if (files == null) { return null; @@ -100,5 +105,8 @@ function filesToListing({ } else { console.warn(`invalid sort direction: '${sortDirection}'`); } + if (mask) { + computeFileMasks(v); + } return v; } diff --git a/src/packages/frontend/project/page/flyouts/files.tsx b/src/packages/frontend/project/page/flyouts/files.tsx index 4ed6d1f2a7..55e6a5d2cf 100644 --- a/src/packages/frontend/project/page/flyouts/files.tsx +++ b/src/packages/frontend/project/page/flyouts/files.tsx @@ -26,7 +26,7 @@ import { file_options } from "@cocalc/frontend/editor-tmp"; import { FileUploadWrapper } from "@cocalc/frontend/file-upload"; import { should_open_in_foreground } from "@cocalc/frontend/lib/should-open-in-foreground"; import { useProjectContext } from "@cocalc/frontend/project/context"; -import { compute_file_masks } from "@cocalc/frontend/project/explorer/compute-file-masks"; +import { computeFileMasks } from "@cocalc/frontend/project/explorer/compute-file-masks"; import { DirectoryListing, DirectoryListingEntry, @@ -164,7 +164,7 @@ export function FilesFlyout({ const files = directoryListing; if (files == null) return EMPTY_LISTING; let activeFile: DirectoryListingEntry | null = null; - compute_file_masks(files); + computeFileMasks(files); const searchWords = file_search.trim().toLowerCase(); const processedFiles: DirectoryListingEntry[] = files From 631502c0bc60e96e9ccbc8623eaddac2d2e72d50 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 30 Jul 2025 19:47:31 +0000 Subject: [PATCH 139/798] fix little bug I just introduced in SearchInput --- src/packages/frontend/components/search-input.tsx | 4 +++- src/packages/frontend/project/explorer/search-bar.tsx | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/packages/frontend/components/search-input.tsx b/src/packages/frontend/components/search-input.tsx index 446c788d96..88d017315a 100644 --- a/src/packages/frontend/components/search-input.tsx +++ b/src/packages/frontend/components/search-input.tsx @@ -55,7 +55,9 @@ export function SearchInput({ }: Props) { const [value, setValue] = useState(value0 ?? default_value ?? ""); // if value changes, we update as well! - useEffect(() => setValue(value ?? ""), [value]); + useEffect(() => { + setValue(value0 ?? ""); + }, [value0]); const [ctrl_down, set_ctrl_down] = useState(false); const [shift_down, set_shift_down] = useState(false); diff --git a/src/packages/frontend/project/explorer/search-bar.tsx b/src/packages/frontend/project/explorer/search-bar.tsx index 535e426536..0ede8f79d6 100644 --- a/src/packages/frontend/project/explorer/search-bar.tsx +++ b/src/packages/frontend/project/explorer/search-bar.tsx @@ -81,6 +81,10 @@ export const SearchBar = memo( undefined, ); + useEffect(() => { + actions.set_file_search(""); + }, [current_path]); + useEffect(() => { if (cmd == null) return; const { input, id } = cmd; From ecdf99252a1ff3dceb4c945d4091832412cee882 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 30 Jul 2025 20:02:43 +0000 Subject: [PATCH 140/798] get rid of file mask toggle button for full page file explorer since it's advanced/complicated (leave it in for flyout panels) in for --- .../project/explorer/compute-file-masks.ts | 19 +++++++------ .../frontend/project/explorer/explorer.tsx | 5 ++-- .../project/explorer/misc-side-buttons.tsx | 28 ------------------- .../project/listing/filter-listing.ts | 5 ---- .../frontend/project/listing/use-listing.ts | 1 + src/packages/util/db-schema/accounts.ts | 7 ++++- 6 files changed, 19 insertions(+), 46 deletions(-) diff --git a/src/packages/frontend/project/explorer/compute-file-masks.ts b/src/packages/frontend/project/explorer/compute-file-masks.ts index f7162818f5..50e6d94b67 100644 --- a/src/packages/frontend/project/explorer/compute-file-masks.ts +++ b/src/packages/frontend/project/explorer/compute-file-masks.ts @@ -4,10 +4,10 @@ */ import { derive_rmd_output_filename } from "@cocalc/frontend/frame-editors/rmd-editor/utils"; -import { dict, filename_extension, startswith } from "@cocalc/util/misc"; +import { dict, filename_extension } from "@cocalc/util/misc"; import { DirectoryListing, DirectoryListingEntry } from "./types"; -const MASKED_FILENAMES = ["__pycache__"] as const; +const MASKED_FILENAMES = new Set(["__pycache__"]); const MASKED_FILE_EXTENSIONS = { py: ["pyc"], @@ -60,12 +60,13 @@ export function computeFileMasks(listing: DirectoryListing): void { listing.map((item) => [item.name, item]), ); for (const file of listing) { - // mask certain known directories - if (MASKED_FILENAMES.indexOf(file.name as any) >= 0) { + // mask certain known paths + if (MASKED_FILENAMES.has(file.name as any)) { filename_map[file.name].mask = true; } - // note: never skip already masked files, because of rnw/rtex->tex + // NOTE: never skip already masked files, because of rnw/rtex->tex + const ext = filename_extension(file.name).toLowerCase(); // some extensions like Rmd modify the basename during compilation @@ -77,16 +78,16 @@ export function computeFileMasks(listing: DirectoryListing): void { // check each possible compiled extension let derivedBasename; // some uppercase-strings have special meaning - if (startswith(mask_ext, "NODOT")) { + if (mask_ext.startsWith("NODOT")) { derivedBasename = basename.slice(0, -1); // exclude the trailing dot mask_ext = mask_ext.slice("NODOT".length); - } else if (mask_ext.indexOf("FILENAME") >= 0) { + } else if (mask_ext.includes("FILENAME")) { derivedBasename = mask_ext.replace("FILENAME", filename); mask_ext = ""; - } else if (mask_ext.indexOf("BASENAME") >= 0) { + } else if (mask_ext.includes("BASENAME")) { derivedBasename = mask_ext.replace("BASENAME", basename.slice(0, -1)); mask_ext = ""; - } else if (mask_ext.indexOf("BASEDASHNAME") >= 0) { + } else if (mask_ext.includes("BASEDASHNAME")) { // BASEDASHNAME is like BASENAME, but replaces spaces by dashes // https://github.com/sagemathinc/cocalc/issues/3229 const fragment = basename.slice(0, -1).replace(/ /g, "-"); diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index edb7cf3528..5e1f523a6a 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -114,6 +114,7 @@ export function Explorer() { const project_map = useTypedRedux("projects", "project_map"); const images = useTypedRedux("compute_images", "images"); + const mask = useTypedRedux("account", "other_settings")?.get("mask_files"); const active_file_sort = useTypedRedux({ project_id }, "active_file_sort"); const fs = useFs({ project_id }); @@ -122,10 +123,9 @@ export function Explorer() { path: current_path, ...sortDesc(active_file_sort), cacheId: actions?.getCacheId(compute_server_id), - mask: true, + mask, }); const showHidden = useTypedRedux({ project_id }, "show_hidden"); - const showMasked = useTypedRedux({ project_id }, "show_masked"); listing = listingError ? null @@ -133,7 +133,6 @@ export function Explorer() { listing, search: file_search, showHidden, - showMasked, }); useEffect(() => { diff --git a/src/packages/frontend/project/explorer/misc-side-buttons.tsx b/src/packages/frontend/project/explorer/misc-side-buttons.tsx index b4444c0c18..5d9fd3d090 100644 --- a/src/packages/frontend/project/explorer/misc-side-buttons.tsx +++ b/src/packages/frontend/project/explorer/misc-side-buttons.tsx @@ -30,7 +30,6 @@ const OPEN_MSG = defineMessage({ export function MiscSideButtons() { const { actions, project_id } = useProjectContext(); const show_hidden = useTypedRedux({ project_id }, "show_hidden"); - const show_masked = useTypedRedux({ project_id }, "show_masked"); const current_path = useTypedRedux({ project_id }, "current_path"); const available_features = useTypedRedux( { project_id }, @@ -49,13 +48,6 @@ export function MiscSideButtons() { }); }; - const handle_masked_toggle = (e: MouseEvent): void => { - e.preventDefault(); - actions?.setState({ - show_masked: !show_masked, - }); - }; - const handle_backup = (e: MouseEvent): void => { e.preventDefault(); actions?.open_directory(".snapshots"); @@ -78,25 +70,6 @@ export function MiscSideButtons() { ); } - function render_masked_toggle(): JSX.Element { - return ( - - ); - } - function render_backup(): JSX.Element | undefined { // NOTE -- snapshots aren't available except in "kucalc" version // -- they are complicated nontrivial thing that isn't usually setup... @@ -211,7 +184,6 @@ export function MiscSideButtons() {
{render_hidden_toggle()} - {render_masked_toggle()} {render_backup()} diff --git a/src/packages/frontend/project/listing/filter-listing.ts b/src/packages/frontend/project/listing/filter-listing.ts index a5b03edb69..53e296da7f 100644 --- a/src/packages/frontend/project/listing/filter-listing.ts +++ b/src/packages/frontend/project/listing/filter-listing.ts @@ -4,16 +4,11 @@ export default function filterListing({ listing, search, showHidden, - showMasked, }: { listing?: DirectoryListingEntry[] | null; search?: string; showHidden?: boolean; - showMasked?: boolean; }): DirectoryListingEntry[] | null { - if (!showMasked) { - console.log("TODO: show masked"); - } if (listing == null) { return null; } diff --git a/src/packages/frontend/project/listing/use-listing.ts b/src/packages/frontend/project/listing/use-listing.ts index a4c481c52f..764d602108 100644 --- a/src/packages/frontend/project/listing/use-listing.ts +++ b/src/packages/frontend/project/listing/use-listing.ts @@ -106,6 +106,7 @@ function filesToListing({ console.warn(`invalid sort direction: '${sortDirection}'`); } if (mask) { + // note -- this masking is as much time as everything above computeFileMasks(v); } return v; diff --git a/src/packages/util/db-schema/accounts.ts b/src/packages/util/db-schema/accounts.ts index fc7774d8d5..27d8ad9a2d 100644 --- a/src/packages/util/db-schema/accounts.ts +++ b/src/packages/util/db-schema/accounts.ts @@ -482,7 +482,12 @@ Table({ other_settings: { katex: true, confirm_close: false, - mask_files: true, + // mask_files -- note that there is a performance cost to this, e.g., 5ms if you have 10K files in + // a directory (basically it doubles the processing costs). + // It's also confusing and can be subtly wrong. Finally, it's almost never necessary due to us changing the defaults + // for running latex to put all the temp files in /tmp -- in general we should always put temp files in tmp anyways + // with all build processes. So mask_files is off by default if not explicitly selected. + mask_files: false, page_size: 500, standby_timeout_m: 5, default_file_sort: "name", From 0e420a55496c5866ad4323d6d275fed698e4e94b Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 30 Jul 2025 21:56:47 +0000 Subject: [PATCH 141/798] isdir --> isDir; isopen --> isOpen, etc. -- consistent naming. Also much more use of {...} : Props so we can always identify unused props quickly. --- src/packages/backend/get-listing.ts | 18 +- src/packages/conat/files/listing.ts | 20 +- .../database/postgres-server-queries.coffee | 2 +- .../file-server/btrfs/subvolume-bup.ts | 13 +- src/packages/file-server/btrfs/subvolume.ts | 4 +- src/packages/file-server/btrfs/subvolumes.ts | 4 +- .../file-server/btrfs/test/subvolume.test.ts | 6 +- src/packages/file-server/btrfs/util.ts | 2 +- src/packages/frontend/client/project.ts | 2 +- .../frontend/course/assignments/actions.ts | 2 +- .../course/export/export-assignment.ts | 2 +- src/packages/frontend/cspell.json | 1 + .../terminal-editor/commands-guide.tsx | 2 +- .../frontend/project/directory-selector.tsx | 2 +- .../frontend/project/explorer/action-bar.tsx | 6 +- .../frontend/project/explorer/action-box.tsx | 34 +--- .../frontend/project/explorer/download.tsx | 6 +- .../frontend/project/explorer/explorer.tsx | 4 +- .../explorer/file-listing/file-listing.tsx | 50 +---- .../explorer/file-listing/file-row.tsx | 172 ++++++++---------- .../frontend/project/explorer/types.ts | 32 +--- .../frontend/project/listing/use-files.ts | 2 +- .../frontend/project/listing/use-listing.ts | 2 +- .../project/page/flyouts/active-group.tsx | 10 +- .../frontend/project/page/flyouts/active.tsx | 14 +- .../project/page/flyouts/file-list-item.tsx | 56 +++--- .../project/page/flyouts/files-bottom.tsx | 10 +- .../project/page/flyouts/files-controls.tsx | 23 ++- .../frontend/project/page/flyouts/files.tsx | 24 +-- .../frontend/project/page/flyouts/log.tsx | 4 +- src/packages/frontend/project_actions.ts | 20 +- src/packages/frontend/share/config.tsx | 105 +++++------ src/packages/util/types/directory-listing.ts | 17 +- 33 files changed, 305 insertions(+), 366 deletions(-) diff --git a/src/packages/backend/get-listing.ts b/src/packages/backend/get-listing.ts index 02bcdbc835..90c66c3c8d 100644 --- a/src/packages/backend/get-listing.ts +++ b/src/packages/backend/get-listing.ts @@ -3,10 +3,12 @@ * License: MS-RSL – see LICENSE.md for details */ +// DEPRECATABLE? + /* This is used by backends to serve directory listings to clients: -{files:[..., {size:?,name:?,mtime:?,isdir:?}]} +{files:[..., {size:?,name:?,mtime:?,isDir:?}]} where mtime is integer SECONDS since epoch, size is in bytes, and isdir is only there if true. @@ -46,7 +48,7 @@ const getListing = reuseInFlight( if (!hidden && file.name[0] === ".") { continue; } - let entry: DirectoryListingEntry; + let entry: Partial; try { // I don't actually know if file.name can fail to be JSON-able with node.js -- is there // even a string in Node.js that cannot be dumped to JSON? With python @@ -61,14 +63,14 @@ const getListing = reuseInFlight( try { let stats: Stats; if (file.isSymbolicLink()) { - // Optimization: don't explicitly set issymlink if it is false - entry.issymlink = true; + // Optimization: don't explicitly set isSymLink if it is false + entry.isSymLink = true; } - if (entry.issymlink) { + if (entry.isSymLink) { // at least right now we only use this symlink stuff to display // information to the user in a listing, and nothing else. try { - entry.link_target = await readlink(dir + "/" + entry.name); + entry.linkTarget = await readlink(dir + "/" + entry.name); } catch (err) { // If we don't know the link target for some reason; just ignore this. } @@ -81,7 +83,7 @@ const getListing = reuseInFlight( } entry.mtime = stats.mtime.valueOf() / 1000; if (stats.isDirectory()) { - entry.isdir = true; + entry.isDir = true; const v = await readdir(dir + "/" + entry.name); if (hidden) { entry.size = v.length; @@ -100,7 +102,7 @@ const getListing = reuseInFlight( } catch (err) { entry.error = `${entry.error ? entry.error : ""}${err}`; } - files.push(entry); + files.push(entry as DirectoryListingEntry); } return files; }, diff --git a/src/packages/conat/files/listing.ts b/src/packages/conat/files/listing.ts index 912363c0ab..e1836510d3 100644 --- a/src/packages/conat/files/listing.ts +++ b/src/packages/conat/files/listing.ts @@ -27,11 +27,11 @@ interface FileData { // last modification time as time since epoch in **milliseconds** (as is usual for javascript) mtime: number; size: number; - // isdir = mainly for backward compat: - isdir?: boolean; + // isDir = mainly for backward compat: + isDir?: boolean; // issymlink = mainly for backward compat: - issymlink?: boolean; - link_target?: string; + isSymLink?: boolean; + linkTarget?: string; // see typeDescription above. type?: FileTypeLabel; } @@ -114,13 +114,13 @@ export class Listing extends EventEmitter { }; if (stats.isSymbolicLink()) { // resolve target. - data.link_target = await this.opts.fs.readlink( + data.linkTarget = await this.opts.fs.readlink( join(this.opts.path, filename), ); - data.issymlink = true; + data.isSymLink = true; } if (stats.isDirectory()) { - data.isdir = true; + data.isDir = true; } this.files[filename] = data; } catch (err) { @@ -167,13 +167,13 @@ async function getListing( const size = parseInt(v[2]); files[name] = { mtime, size, type: v[3] as FileTypeLabel }; if (v[3] == "l") { - files[name].issymlink = true; + files[name].isSymLink = true; } if (v[3] == "d") { - files[name].isdir = true; + files[name].isDir = true; } if (v[4]) { - files[name].link_target = v[4]; + files[name].linkTarget = v[4]; } } catch {} } diff --git a/src/packages/database/postgres-server-queries.coffee b/src/packages/database/postgres-server-queries.coffee index 423fedbc84..e3f0eccb42 100644 --- a/src/packages/database/postgres-server-queries.coffee +++ b/src/packages/database/postgres-server-queries.coffee @@ -1517,7 +1517,7 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext opts = defaults opts, project_id : required path : required - listing : required # files in path [{name:..., isdir:boolean, ....}, ...] + listing : required # files in path [{name:..., isDir:boolean, ....}, ...] cb : required # Get all public paths for the given project_id, then check if path is "in" one according # to the definition in misc. diff --git a/src/packages/file-server/btrfs/subvolume-bup.ts b/src/packages/file-server/btrfs/subvolume-bup.ts index b4d64c9b5a..3237e32dba 100644 --- a/src/packages/file-server/btrfs/subvolume-bup.ts +++ b/src/packages/file-server/btrfs/subvolume-bup.ts @@ -110,6 +110,7 @@ export class SubvolumeBup { }); }; + // [ ] TODO: remove this ls and instead rely only on the fs sandbox code. ls = async (path: string = ""): Promise => { if (!path) { const { stdout } = await sudo({ @@ -125,10 +126,10 @@ export class SubvolumeBup { } const mtime = parseBupTime(name).valueOf() / 1000; newest = Math.max(mtime, newest); - v.push({ name, isdir: true, mtime }); + v.push({ name, isDir: true, mtime }); } if (v.length > 0) { - v.push({ name: "latest", isdir: true, mtime: newest }); + v.push({ name: "latest", isDir: true, mtime: newest }); } return v; } @@ -153,20 +154,20 @@ export class SubvolumeBup { // [-rw-------","6b851643360e435eb87ef9a6ab64a8b1/6b851643360e435eb87ef9a6ab64a8b1","5","2025-07-15","06:12","a.txt"] const w = x.split(/\s+/); if (w.length >= 6) { - let isdir, name; + let isDir, name; if (w[5].endsWith("@") || w[5].endsWith("=") || w[5].endsWith("|")) { w[5] = w[5].slice(0, -1); } if (w[5].endsWith("/")) { - isdir = true; + isDir = true; name = w[5].slice(0, -1); } else { name = w[5]; - isdir = false; + isDir = false; } const size = parseInt(w[2]); const mtime = new Date(w[3] + "T" + w[4]).valueOf() / 1000; - v.push({ name, size, mtime, isdir }); + v.push({ name, size, mtime, isDir }); } } return v; diff --git a/src/packages/file-server/btrfs/subvolume.ts b/src/packages/file-server/btrfs/subvolume.ts index f10e9f6182..c29d393994 100644 --- a/src/packages/file-server/btrfs/subvolume.ts +++ b/src/packages/file-server/btrfs/subvolume.ts @@ -4,7 +4,7 @@ A subvolume import { type Filesystem, DEFAULT_SUBVOLUME_SIZE } from "./filesystem"; import refCache from "@cocalc/util/refcache"; -import { isdir, sudo } from "./util"; +import { isDir, sudo } from "./util"; import { join } from "path"; import { SubvolumeBup } from "./subvolume-bup"; import { SubvolumeSnapshots } from "./subvolume-snapshots"; @@ -94,7 +94,7 @@ export class Subvolume { if (target.endsWith("/")) { targetPath += "/"; } - if (!srcPath.endsWith("/") && (await isdir(srcPath))) { + if (!srcPath.endsWith("/") && (await isDir(srcPath))) { srcPath += "/"; if (!targetPath.endsWith("/")) { targetPath += "/"; diff --git a/src/packages/file-server/btrfs/subvolumes.ts b/src/packages/file-server/btrfs/subvolumes.ts index 0f8da1468f..e1a6c418be 100644 --- a/src/packages/file-server/btrfs/subvolumes.ts +++ b/src/packages/file-server/btrfs/subvolumes.ts @@ -4,7 +4,7 @@ import getLogger from "@cocalc/backend/logger"; import { SNAPSHOTS } from "./subvolume-snapshots"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import { join, normalize } from "path"; -import { btrfs, isdir } from "./util"; +import { btrfs, isDir } from "./util"; import { chmod, rename, rm } from "node:fs/promises"; import { executeCode } from "@cocalc/backend/execute-code"; @@ -99,7 +99,7 @@ export class Subvolumes { if (!targetPath.startsWith(this.filesystem.opts.mount)) { throw Error("suspicious target"); } - if (!srcPath.endsWith("/") && (await isdir(srcPath))) { + if (!srcPath.endsWith("/") && (await isDir(srcPath))) { srcPath += "/"; if (!targetPath.endsWith("/")) { targetPath += "/"; diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index 1736f17c4f..fe9286935e 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -248,7 +248,7 @@ describe("test bup backups", () => { it("confirm a.txt is in our backup", async () => { const x = await vol.bup.ls("latest"); expect(x).toEqual([ - { name: "a.txt", size: 5, mtime: x[0].mtime, isdir: false }, + { name: "a.txt", size: 5, mtime: x[0].mtime, isDir: false }, ]); }); @@ -270,8 +270,8 @@ describe("test bup backups", () => { await vol.bup.save(); const x = await vol.bup.ls("latest"); expect(x).toEqual([ - { name: "a.txt", size: 5, mtime: x[0].mtime, isdir: false }, - { name: "mydir", size: 0, mtime: x[1].mtime, isdir: true }, + { name: "a.txt", size: 5, mtime: x[0].mtime, isDir: false }, + { name: "mydir", size: 0, mtime: x[1].mtime, isDir: true }, ]); expect(Math.abs((x[0].mtime ?? 0) * 1000 - Date.now())).toBeLessThan( 5 * 60_000, diff --git a/src/packages/file-server/btrfs/util.ts b/src/packages/file-server/btrfs/util.ts index b6408f8440..ee71b79eac 100644 --- a/src/packages/file-server/btrfs/util.ts +++ b/src/packages/file-server/btrfs/util.ts @@ -44,7 +44,7 @@ export async function btrfs( return await sudo({ ...opts, command: "btrfs" }); } -export async function isdir(path: string) { +export async function isDir(path: string) { return (await stat(path)).isDirectory(); } diff --git a/src/packages/frontend/client/project.ts b/src/packages/frontend/client/project.ts index 6d621d4d84..8938d29aa4 100644 --- a/src/packages/frontend/client/project.ts +++ b/src/packages/frontend/client/project.ts @@ -496,7 +496,7 @@ export class ProjectClient { return (await this.api(opts.project_id)).realpath(opts.path); }; - isdir = async ({ + isDir = async ({ project_id, path, }: { diff --git a/src/packages/frontend/course/assignments/actions.ts b/src/packages/frontend/course/assignments/actions.ts index 2be16c86fe..f53ba0d227 100644 --- a/src/packages/frontend/course/assignments/actions.ts +++ b/src/packages/frontend/course/assignments/actions.ts @@ -1598,7 +1598,7 @@ ${details} let has_student_subdir: boolean = false; for (const entry of listing) { - if (entry.isdir && entry.name == STUDENT_SUBDIR) { + if (entry.isDir && entry.name == STUDENT_SUBDIR) { has_student_subdir = true; break; } diff --git a/src/packages/frontend/course/export/export-assignment.ts b/src/packages/frontend/course/export/export-assignment.ts index 0ab46b502e..35e14ea09b 100644 --- a/src/packages/frontend/course/export/export-assignment.ts +++ b/src/packages/frontend/course/export/export-assignment.ts @@ -82,7 +82,7 @@ async function export_one_directory( let x: any; const timeout = 60; // 60 seconds for (x of listing) { - if (x.isdir) continue; // we ignore subdirectories... + if (x.isDir) continue; // we ignore subdirectories... const { name } = x; if (startswith(name, "STUDENT")) continue; if (startswith(name, ".")) continue; diff --git a/src/packages/frontend/cspell.json b/src/packages/frontend/cspell.json index d742413365..2a9d09d1bf 100644 --- a/src/packages/frontend/cspell.json +++ b/src/packages/frontend/cspell.json @@ -50,6 +50,7 @@ "immutablejs", "ipynb", "isdir", + "isDir", "kernelspec", "LLM", "LLMs", diff --git a/src/packages/frontend/frame-editors/terminal-editor/commands-guide.tsx b/src/packages/frontend/frame-editors/terminal-editor/commands-guide.tsx index ba18b638f5..ad338c2faf 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/commands-guide.tsx +++ b/src/packages/frontend/frame-editors/terminal-editor/commands-guide.tsx @@ -102,7 +102,7 @@ export function CommandsGuide({ actions, local_view_state }: Props) { if (!hidden && name.startsWith(".")) { continue; } - if (files[name].isdir) { + if (files[name].isDir) { dirnames.push(name); } else { filenames.push(name); diff --git a/src/packages/frontend/project/directory-selector.tsx b/src/packages/frontend/project/directory-selector.tsx index bb1a474595..1463cbbbe3 100644 --- a/src/packages/frontend/project/directory-selector.tsx +++ b/src/packages/frontend/project/directory-selector.tsx @@ -394,7 +394,7 @@ function Subdirs(props) { const paths: string[] = []; const newPaths: string[] = []; for (const name in files) { - if (!files[name].isdir) continue; + if (!files[name].isDir) continue; if (name.startsWith(".") && !showHidden) continue; if (name.startsWith(NEW_FOLDER)) { newPaths.push(name); diff --git a/src/packages/frontend/project/explorer/action-bar.tsx b/src/packages/frontend/project/explorer/action-bar.tsx index 98d967f393..80977d97dd 100644 --- a/src/packages/frontend/project/explorer/action-bar.tsx +++ b/src/packages/frontend/project/explorer/action-bar.tsx @@ -228,15 +228,15 @@ export function ActionBar({ if (checked_files.size === 0) { return; } else if (checked_files.size === 1) { - let isdir; + let isDir; const item = checked_files.first(); for (const file of listing) { if (misc.path_to_file(current_path ?? "", file.name) === item) { - ({ isdir } = file); + ({ isDir } = file); } } - if (isdir) { + if (isDir) { // one directory selected action_buttons = [...ACTION_BUTTONS_DIR]; } else { diff --git a/src/packages/frontend/project/explorer/action-box.tsx b/src/packages/frontend/project/explorer/action-box.tsx index f39af2a2b9..1e05857215 100644 --- a/src/packages/frontend/project/explorer/action-box.tsx +++ b/src/packages/frontend/project/explorer/action-box.tsx @@ -104,7 +104,7 @@ export function ActionBox({ } } - function render_selected_files_list(): React.JSX.Element { + function render_selected_files_list() { return ( 
         {checked_files.toArray().map((name) => (
@@ -124,7 +124,7 @@ export function ActionBox({
     actions.set_all_files_unchecked();
   }
 
-  function render_delete_warning(): React.JSX.Element | undefined {
+  function render_delete_warning() {
     if (current_path === ".trash") {
       return (
         
@@ -139,7 +139,7 @@ export function ActionBox({
     }
   }
 
-  function render_delete(): React.JSX.Element | undefined {
+  function render_delete() {
     const { size } = checked_files;
     return (
       

@@ -213,7 +213,7 @@ export function ActionBox({
     return dest !== current_path;
   }
 
-  function render_move(): React.JSX.Element {
+  function render_move() {
     const { size } = checked_files;
     return (
       

@@ -261,7 +261,7 @@ export function ActionBox({
     }
   }
 
-  function render_different_project_dialog(): React.JSX.Element | undefined {
+  function render_different_project_dialog() {
     if (show_different_project) {
       return (
         
@@ -279,9 +279,7 @@ export function ActionBox({
     }
   }
 
-  function render_copy_different_project_options():
-    | React.JSX.Element
-    | undefined {
+  function render_copy_different_project_options() {
     if (project_id !== copy_destination_project_id) {
       return (
         

@@ -431,7 +429,7 @@ export function ActionBox({
     );
   }
 
-  function render_copy(): React.JSX.Element {
+  function render_copy() {
     const { size } = checked_files;
     const signed_in = get_user_type() === "signed_in";
     if (!signed_in) {
@@ -540,27 +538,17 @@ export function ActionBox({
     }
   }
 
-  function render_share(): React.JSX.Element {
+  function render_share() {
     // currently only works for a single selected file
     const path: string = checked_files.first() ?? "";
     if (!path) {
-      return <>;
-    }
-    const public_data = file_map[misc.path_split(path).tail];
-    if (public_data == undefined) {
-      // directory listing not loaded yet... (will get re-rendered when loaded)
-      return ;
+      return null;
     }
     return (
        actions.set_public_path(path, opts)}
@@ -569,9 +557,7 @@ export function ActionBox({
     );
   }
 
-  function render_action_box(
-    action: FileAction,
-  ): React.JSX.Element | undefined {
+  function render_action_box(action: FileAction) {
     switch (action) {
       case "compress":
         return ;
diff --git a/src/packages/frontend/project/explorer/download.tsx b/src/packages/frontend/project/explorer/download.tsx
index 17f7740a76..0bf5c11a42 100644
--- a/src/packages/frontend/project/explorer/download.tsx
+++ b/src/packages/frontend/project/explorer/download.tsx
@@ -44,9 +44,9 @@ export default function Download() {
       return;
     }
     const file = checked_files.first();
-    const isdir = !!actions.isDirViaCache(file);
-    setArchiveMode(!!isdir);
-    if (!isdir) {
+    const isDir = !!actions.isDirViaCache(file);
+    setArchiveMode(!!isDir);
+    if (!isDir) {
       const store = actions?.get_store();
       setUrl(store?.fileURL(file) ?? "");
     }
diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx
index 5e1f523a6a..3ec833a4aa 100644
--- a/src/packages/frontend/project/explorer/explorer.tsx
+++ b/src/packages/frontend/project/explorer/explorer.tsx
@@ -167,9 +167,9 @@ export function Explorer() {
           redux.getProjectStore(project_id).get("selected_file_index") ?? 0;
         const x = listing?.[n];
         if (x != null) {
-          const { isdir, name } = x;
+          const { isDir, name } = x;
           const path = join(current_path, name);
-          if (isdir) {
+          if (isDir) {
             actions.open_directory(path);
           } else {
             actions.open_file({ path, foreground: !e.ctrlKey });
diff --git a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx
index 52f7b6f275..4663119389 100644
--- a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx
+++ b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx
@@ -12,12 +12,7 @@ import * as immutable from "immutable";
 import { useEffect, useRef } from "react";
 import { FormattedMessage } from "react-intl";
 import { Virtuoso, VirtuosoHandle } from "react-virtuoso";
-import {
-  Rendered,
-  TypedMap,
-  useTypedRedux,
-  redux,
-} from "@cocalc/frontend/app-framework";
+import { TypedMap, useTypedRedux, redux } from "@cocalc/frontend/app-framework";
 import useVirtuosoScrollHook from "@cocalc/frontend/components/virtuoso-scroll-hook";
 import { ProjectActions } from "@cocalc/frontend/project_actions";
 import { MainConfiguration } from "@cocalc/frontend/project_configuration";
@@ -26,7 +21,7 @@ import { FileRow } from "./file-row";
 import { ListingHeader } from "./listing-header";
 import NoFiles from "./no-files";
 import { TERM_MODE_CHAR } from "./utils";
-import { DirectoryListingEntry } from "@cocalc/util/types";
+import { type DirectoryListingEntry } from "@cocalc/frontend/project/explorer/types";
 
 interface Props {
   actions: ProjectActions;
@@ -59,38 +54,22 @@ export function FileListing({
     useTypedRedux({ project_id }, "selected_file_index") ?? 0;
   const name = actions.name;
 
-  function render_row(
-    name,
-    size,
-    time,
-    mask,
-    isdir,
-    issymlink,
-    index: number,
-    link_target?: string, // if given, is a known symlink to this file
-  ): Rendered {
+  function renderRow(index, file) {
     const checked = checked_files.has(misc.path_to_file(current_path, name));
     const color = misc.rowBackground({ index, checked });
 
     return (
       
@@ -122,28 +101,19 @@ export function FileListing({
     return ;
   }
 
-  function render_rows(): Rendered {
+  function renderRows() {
     return (
        {
-          const a = listing[index];
-          if (a == null) {
+          const file = listing[index];
+          if (file == null) {
             // shouldn't happen
             return 
;
           }
-          return render_row(
-            a.name,
-            a.size,
-            a.mtime,
-            a.mask,
-            a.isdir,
-            a.issymlink,
-            index,
-            a.link_target,
-          );
+          return renderRow(index, file);
         }}
         {...virtuosoScroll}
       />
@@ -206,7 +176,7 @@ export function FileListing({
           active_file_sort={active_file_sort}
           sort_by={actions.set_sorted_file_column}
         />
-        {listing.length > 0 ? render_rows() : render_no_files()}
+        {listing.length > 0 ? renderRows() : render_no_files()}
       

     
   );
diff --git a/src/packages/frontend/project/explorer/file-listing/file-row.tsx b/src/packages/frontend/project/explorer/file-listing/file-row.tsx
index 92308958be..881f449b16 100644
--- a/src/packages/frontend/project/explorer/file-listing/file-row.tsx
+++ b/src/packages/frontend/project/explorer/file-listing/file-row.tsx
@@ -25,6 +25,7 @@ import { url_href } from "@cocalc/frontend/project/utils";
 import { FileCheckbox } from "./file-checkbox";
 import { PublicButton } from "./public-button";
 import { generate_click_for } from "./utils";
+import { type DirectoryListing } from "@cocalc/frontend/project/explorer/types";
 
 export const VIEWABLE_FILE_EXT: Readonly = [
   "md",
@@ -36,41 +37,59 @@ export const VIEWABLE_FILE_EXT: Readonly = [
 ] as const;
 
 interface Props {
-  isdir: boolean;
+  isDir: boolean;
   name: string;
-  display_name?: string; // if given, will display this, and will show true filename in popover
-  size: number; // sometimes is NOT known!
-  time: number;
-  issymlink: boolean;
+  // if given, will display this, and will show true filename in popover
+  display_name?: string;
+  size: number;
+  mtime: number;
+  isSymLink: boolean;
   checked: boolean;
   selected: boolean;
   color: string;
   mask: boolean;
-  is_public: boolean;
+  isPublic: boolean;
   current_path: string;
   actions: ProjectActions;
   no_select: boolean;
-  link_target?: string;
+  linkTarget?: string;
   // if given, include a little 'server' tag in this color, and tooltip etc using id.
   // Also important for download and preview links!
   computeServerId?: number;
-  listing;
+  listing: DirectoryListing;
 }
 
-export const FileRow: React.FC = React.memo((props) => {
+export function FileRow({
+  isDir,
+  name,
+  display_name,
+  size,
+  mtime,
+  checked,
+  selected,
+  color,
+  mask,
+  isPublic,
+  current_path,
+  actions,
+  no_select,
+  linkTarget,
+  computeServerId,
+  listing,
+}: Props) {
   const student_project_functionality = useStudentProjectFunctionality(
-    props.actions.project_id,
+    actions.project_id,
   );
   const [selection_at_last_mouse_down, set_selection_at_last_mouse_down] =
     useState(undefined);
 
   function render_icon() {
     const style: React.CSSProperties = {
-      color: props.mask ? "#bbbbbb" : COLORS.FILE_ICON,
+      color: mask ? "#bbbbbb" : COLORS.FILE_ICON,
       verticalAlign: "sub",
     } as const;
     let body: React.JSX.Element;
-    if (props.isdir) {
+    if (isDir) {
       body = (
         <>
            = React.memo((props) => {
       );
     } else {
       // get the file_associations[ext] just like it is defined in the editor
-      let name: IconName;
-      const info = file_options(props.name);
-      if (info != null) {
-        name = info.icon;
-      } else {
-        name = "file";
-      }
+      const info = file_options(name);
+      const iconName: IconName = info?.icon ?? "file";
 
-      body = ;
+      body = ;
     }
 
     return {body};
   }
 
-  function render_link_target() {
-    if (props.link_target == null || props.link_target == props.name) return;
-    return (
-      <>
-        {" "}
-        {" "}
-        {props.link_target}{" "}
-      
-    );
-  }
-
   function render_name_link(styles, name, ext) {
     return (
       
         {misc.trunc_middle(name, 50)}
-        
+        
           {ext === "" ? "" : `.${ext}`}
         
-        {render_link_target()}
+        {linkTarget != null && linkTarget != name && (
+          <>
+            {" "}
+            {" "}
+            {linkTarget}{" "}
+          
+        )}
       
     );
   }
 
   function render_name() {
-    let name = props.display_name ?? props.name;
+    let name0 = display_name ?? name;
     let ext: string;
-    if (props.isdir) {
+    if (isDir) {
       ext = "";
     } else {
-      const name_and_ext = misc.separate_file_extension(name);
-      ({ name, ext } = name_and_ext);
+      const name_and_ext = misc.separate_file_extension(name0);
+      ({ name: name0, ext } = name_and_ext);
     }
 
     const show_tip =
-      (props.display_name != undefined && props.name !== props.display_name) ||
-      name.length > 50;
+      (display_name != undefined && name0 !== display_name) ||
+      name0.length > 50;
 
     const styles = {
       whiteSpace: "pre-wrap",
       wordWrap: "break-word",
       overflowWrap: "break-word",
       verticalAlign: "middle",
-      color: props.mask ? "#bbbbbb" : COLORS.TAB,
+      color: mask ? "#bbbbbb" : COLORS.TAB,
     };
 
     if (show_tip) {
       return (
         
-          {render_name_link(styles, name, ext)}
+          {render_name_link(styles, name0, ext)}
         
       );
     } else {
-      return render_name_link(styles, name, ext);
+      return render_name_link(styles, name0, ext);
     }
   }
 
   const generate_on_share_click = memoizeOne((full_path: string) => {
-    return generate_click_for("share", full_path, props.actions);
+    return generate_click_for("share", full_path, actions);
   });
 
   function render_public_file_info() {
-    if (props.is_public) {
+    if (isPublic) {
       return ;
     }
   }
 
   function full_path() {
-    return misc.path_to_file(props.current_path, props.name);
+    return misc.path_to_file(current_path, name);
   }
 
   function handle_mouse_down() {
@@ -193,25 +202,24 @@ export const FileRow: React.FC = React.memo((props) => {
       // the click to do the selection triggering opening of the file.
       return;
     }
-    if (props.isdir) {
-      props.actions.open_directory(full_path());
-      props.actions.set_file_search("");
+    if (isDir) {
+      actions.open_directory(full_path());
+      actions.set_file_search("");
     } else {
       const foreground = should_open_in_foreground(e);
       const path = full_path();
       track("open-file", {
-        project_id: props.actions.project_id,
+        project_id: actions.project_id,
         path,
         how: "click-on-listing",
       });
-      props.actions.open_file({
+      actions.open_file({
         path,
         foreground,
         explicit: true,
       });
       if (foreground) {
         // delay slightly since it looks weird to see the full listing right when you click on a file
-        const actions = props.actions;
         setTimeout(() => actions.set_file_search(""), 10);
       }
     }
@@ -220,7 +228,7 @@ export const FileRow: React.FC = React.memo((props) => {
   function handle_download_click(e) {
     e.preventDefault();
     e.stopPropagation();
-    props.actions.download_file({
+    actions.download_file({
       path: full_path(),
       log: true,
     });
@@ -236,7 +244,7 @@ export const FileRow: React.FC = React.memo((props) => {
     try {
       return (
         
       );
@@ -295,7 +303,8 @@ export const FileRow: React.FC = React.memo((props) => {
 
   function render_download_button(url) {
     if (student_project_functionality.disableActions) return;
-    const size = misc.human_readable_size(props.size);
+    if (isDir) return;
+    const displaySize = misc.human_readable_size(size);
     // TODO: This really should not be in the size column...
     return (
        = React.memo((props) => {
         }
         content={
           <>
-            Download this {size} file
+            Download this {displaySize} file
             

             to your computer.
           
@@ -320,7 +329,7 @@ export const FileRow: React.FC = React.memo((props) => {
           onClick={handle_download_click}
           style={{ color: COLORS.GRAY, padding: 0 }}
         >
-          {size}
+          {displaySize}
           
         
       
@@ -330,35 +339,31 @@ export const FileRow: React.FC = React.memo((props) => {
   const row_styles: CSS = {
     cursor: "pointer",
     borderRadius: "4px",
-    backgroundColor: props.color,
+    backgroundColor: color,
     borderStyle: "solid",
-    borderColor: props.selected ? "#08c" : "transparent",
+    borderColor: selected ? "#08c" : "transparent",
     margin: "1px 1px 1px 1px",
   } as const;
 
   // See https://github.com/sagemathinc/cocalc/issues/1020
   // support right-click → copy url for the download button
-  const url = url_href(
-    props.actions.project_id,
-    full_path(),
-    props.computeServerId,
-  );
+  const url = url_href(actions.project_id, full_path(), computeServerId);
 
   return (
     
       
         {!student_project_functionality.disableActions && (
           
         )}
       
@@ -386,34 +391,13 @@ export const FileRow: React.FC = React.memo((props) => {
           
         
         {render_timestamp()}
-        {props.isdir ? (
-          <>
-            
-          
-        ) : (
+        {!isDir && (
           
             {render_download_button(url)}
-            {render_view_button(url, props.name)}
+            {render_view_button(url, name)}
           
         )}
       
     
   );
-});
-
-const directory_size_style: React.CSSProperties = {
-  color: COLORS.GRAY,
-  marginRight: "3em",
-} as const;
-
-function DirectorySize({ size }) {
-  if (size == undefined) {
-    return null;
-  }
-
-  return (
-    
-      {size} {misc.plural(size, "item")}
-    
-  );
 }
diff --git a/src/packages/frontend/project/explorer/types.ts b/src/packages/frontend/project/explorer/types.ts
index 9b85a77ad0..62b0f1b64b 100644
--- a/src/packages/frontend/project/explorer/types.ts
+++ b/src/packages/frontend/project/explorer/types.ts
@@ -3,30 +3,18 @@
  *  License: MS-RSL – see LICENSE.md for details
  */
 
-// NOTE(hsy): I don't know if these two types are the same, maybe they should be merged.
+import type { DirectoryListingEntry as DirectoryListingEntry0 } from "@cocalc/util/types";
 
-export interface ListingItem {
-  name: string;
-  isdir: boolean;
-  isopen?: boolean;
-  mtime?: number;
-  size?: number; // bytes
-}
-
-// NOTE: there is also @cocalc/util/types/directory-listing::DirectoryListingEntry
-// but ATM the relation ship to this one is unclear. Don't mix them up!
-// This type here is used in the frontend, e.g. in Explorer and Flyout Files.
-export interface DirectoryListingEntry {
-  display_name?: string; // unclear, if this even exists
-  name: string;
-  size?: number;
-  mtime?: number;
-  isdir?: boolean;
+// fill in extra info used in the frontend, mainly for the UI
+export interface DirectoryListingEntry extends DirectoryListingEntry0 {
+  // whether or not mask this file in the UI
   mask?: boolean;
-  isopen?: boolean; // opened in an editor
-  isactive?: boolean; // opeend in the currently active editor
-  is_public?: boolean; // a shared file
-  public?: any; // some data about the shared file (TODO type?)
+  // a publicly shared file
+  isPublic?: boolean;
+
+  // used in flyout panels
+  isOpen?: boolean;
+  isActive?: boolean;
 }
 
 export type DirectoryListing = DirectoryListingEntry[];
diff --git a/src/packages/frontend/project/listing/use-files.ts b/src/packages/frontend/project/listing/use-files.ts
index 7075c98b4c..4d681b5d0c 100644
--- a/src/packages/frontend/project/listing/use-files.ts
+++ b/src/packages/frontend/project/listing/use-files.ts
@@ -152,7 +152,7 @@ async function cacheNeighbors({
 }) {
   let v: string[] = [];
   for (const dir in files) {
-    if (!dir.startsWith(".") && files[dir].isdir) {
+    if (!dir.startsWith(".") && files[dir].isDir) {
       const full = join(path, dir);
       const k = key(cacheId, full);
       if (!cache.has(k) && !failed.has(k)) {
diff --git a/src/packages/frontend/project/listing/use-listing.ts b/src/packages/frontend/project/listing/use-listing.ts
index 764d602108..e49cdfafcd 100644
--- a/src/packages/frontend/project/listing/use-listing.ts
+++ b/src/packages/frontend/project/listing/use-listing.ts
@@ -5,7 +5,7 @@ TESTS: See packages/test/project/listing/
 */
 
 import { useMemo } from "react";
-import { DirectoryListingEntry } from "@cocalc/util/types";
+import { type DirectoryListingEntry } from "@cocalc/frontend/project/explorer/types";
 import { field_cmp } from "@cocalc/util/misc";
 import useFiles from "./use-files";
 import { type FilesystemClient } from "@cocalc/conat/files/fs";
diff --git a/src/packages/frontend/project/page/flyouts/active-group.tsx b/src/packages/frontend/project/page/flyouts/active-group.tsx
index e284164888..763e512e97 100644
--- a/src/packages/frontend/project/page/flyouts/active-group.tsx
+++ b/src/packages/frontend/project/page/flyouts/active-group.tsx
@@ -75,7 +75,7 @@ export function Group({
     const fileType = file_options(`foo.${group}`);
     return {
       iconName:
-        group === "" ? UNKNOWN_FILE_TYPE_ICON : fileType?.icon ?? "file",
+        group === "" ? UNKNOWN_FILE_TYPE_ICON : (fileType?.icon ?? "file"),
       display: (group === "" ? "No extension" : fileType?.name) || group,
     };
   }
@@ -83,7 +83,7 @@ export function Group({
   switch (mode) {
     case "folder":
       const isHome = group === "";
-      const isopen = openFilesGrouped[group].some((path) =>
+      const isOpen = openFilesGrouped[group].some((path) =>
         openFiles.includes(path),
       );
       return (
@@ -93,9 +93,9 @@ export function Group({
           mode="active"
           item={{
             name: group,
-            isdir: true,
-            isopen,
-            isactive: current_path === group && activeTab === "files",
+            isDir: true,
+            isOpen,
+            isActive: current_path === group && activeTab === "files",
           }}
           multiline={false}
           displayedNameOverride={displayed}
diff --git a/src/packages/frontend/project/page/flyouts/active.tsx b/src/packages/frontend/project/page/flyouts/active.tsx
index d8b81fa76c..8436d1cde9 100644
--- a/src/packages/frontend/project/page/flyouts/active.tsx
+++ b/src/packages/frontend/project/page/flyouts/active.tsx
@@ -194,7 +194,7 @@ export function ActiveFlyout(props: Readonly): React.JSX.Element {
     filteredFiles.forEach((path) => {
       const { head, tail } = path_split(path);
       const group =
-        mode === "folder" ? head : filename_extension_notilde(tail) ?? "";
+        mode === "folder" ? head : (filename_extension_notilde(tail) ?? "");
       if (grouped[group] == null) grouped[group] = [];
       grouped[group].push(path);
     });
@@ -258,7 +258,7 @@ export function ActiveFlyout(props: Readonly): React.JSX.Element {
     group?: string,
     isLast?: boolean,
   ): React.JSX.Element {
-    const isactive: boolean = activePath === path;
+    const isActive: boolean = activePath === path;
     const style =
       group != null
         ? {
@@ -267,12 +267,12 @@ export function ActiveFlyout(props: Readonly): React.JSX.Element {
           }
         : undefined;
 
-    const isdir = path.endsWith("/");
-    const isopen = openFiles.includes(path);
+    const isDir = path.endsWith("/");
+    const isOpen = openFiles.includes(path);
 
     // if it is a directory, remove the trailing slash
     // and if it starts with ".smc/root/", replace that by a "/"
-    const display = isdir
+    const display = isDir
       ? path.slice(0, -1).replace(/^\.smc\/root\//, "/")
       : undefined;
 
@@ -280,7 +280,7 @@ export function ActiveFlyout(props: Readonly): React.JSX.Element {
       ): React.JSX.Element {
           // we only toggle star, if it is currently opened!
           // otherwise, when closed and accidentally clicking on the star
           // the file unstarred and just vanishes
-          if (isopen) {
+          if (isOpen) {
             setStarredPath(path, starState);
           } else {
             handleFileClick(undefined, path, "star");
diff --git a/src/packages/frontend/project/page/flyouts/file-list-item.tsx b/src/packages/frontend/project/page/flyouts/file-list-item.tsx
index 341435e9d2..45755b57e1 100644
--- a/src/packages/frontend/project/page/flyouts/file-list-item.tsx
+++ b/src/packages/frontend/project/page/flyouts/file-list-item.tsx
@@ -107,14 +107,14 @@ const CLOSE_ICON_STYLE: CSS = {
 };
 
 interface Item {
-  isopen?: boolean;
-  isdir?: boolean;
-  isactive?: boolean;
-  is_public?: boolean;
+  isOpen?: boolean;
+  isDir?: boolean;
+  isActive?: boolean;
+  isPublic?: boolean;
   name: string;
   size?: number;
   mask?: boolean;
-  link_target?: string;
+  linkTarget?: string;
 }
 
 interface FileListItemProps {
@@ -184,7 +184,7 @@ export const FileListItem = React.memo((props: Readonly) => {
   const bodyRef = useRef(null);
 
   function renderCloseItem(item: Item): React.JSX.Element | null {
-    if (onClose == null || !item.isopen) return null;
+    if (onClose == null || !item.isOpen) return null;
 
     const { name } = item;
     return (
@@ -200,7 +200,7 @@ export const FileListItem = React.memo((props: Readonly) => {
   }
 
   function renderPublishedIcon(): React.JSX.Element | undefined {
-    if (!showPublish || !item.is_public) return undefined;
+    if (!showPublish || !item.isPublic) return undefined;
     return (
       
         

@@ -283,7 +283,7 @@ export const FileListItem = React.memo((props: Readonly) => {
         ? selected
           ? "check-square"
           : "square"
-        : item.isdir
+        : item.isDir
           ? "folder-open"
           : (file_options(item.name)?.icon ?? "file"));
 
@@ -315,7 +315,7 @@ export const FileListItem = React.memo((props: Readonly) => {
         name={icon}
         style={{
           ...ICON_STYLE,
-          color: isStarred && item.isopen ? COLORS.STAR : COLORS.GRAY_L,
+          color: isStarred && item.isOpen ? COLORS.STAR : COLORS.GRAY_L,
         }}
         onClick={(e: React.MouseEvent) => {
           e?.stopPropagation();
@@ -329,8 +329,8 @@ export const FileListItem = React.memo((props: Readonly) => {
     const currentExtra = type === 1 ? extra : extra2;
     if (currentExtra == null) return;
     // calculate extra margin to align the columns. if there is no "onClose", no margin
-    const closeMargin = onClose != null ? (item.isopen ? 0 : 18) : 0;
-    const publishMargin = showPublish ? (item.is_public ? 0 : 20) : 0;
+    const closeMargin = onClose != null ? (item.isOpen ? 0 : 18) : 0;
+    const publishMargin = showPublish ? (item.isPublic ? 0 : 20) : 0;
     const marginRight = type === 1 ? publishMargin + closeMargin : undefined;
     const widthPx = FLYOUT_DEFAULT_WIDTH_PX * 0.33;
     // if the 2nd extra shows up, fix the width to align the columns
@@ -405,14 +405,14 @@ export const FileListItem = React.memo((props: Readonly) => {
     item: Item,
     multiple: boolean,
   ) {
-    const { isdir, name: fileName } = item;
+    const { isDir, name: fileName } = item;
     const actionNames = multiple
       ? ACTION_BUTTONS_MULTI
-      : isdir
+      : isDir
         ? ACTION_BUTTONS_DIR
         : ACTION_BUTTONS_FILE;
     for (const key of actionNames) {
-      if (key === "download" && !item.isdir) continue;
+      if (key === "download" && !item.isDir) continue;
       const disabled =
         isDisabledSnapshots(key) &&
         (current_path?.startsWith(".snapshots") ?? false);
@@ -446,13 +446,13 @@ export const FileListItem = React.memo((props: Readonly) => {
   }
 
   function getContextMenu(): MenuProps["items"] {
-    const { name, isdir, is_public, size } = item;
+    const { name, isDir, isPublic, size } = item;
     const n = checked_files?.size ?? 0;
     const multiple = n > 1;
 
     const sizeStr = size ? human_readable_size(size) : "";
     const nameStr = trunc_middle(item.name, 30);
-    const typeStr = isdir ? "Folder" : "File";
+    const typeStr = isDir ? "Folder" : "File";
 
     const ctx: NonNullable = [];
 
@@ -466,7 +466,7 @@ export const FileListItem = React.memo((props: Readonly) => {
     } else {
       ctx.push({
         key: "header",
-        icon: ,
+        icon: ,
         label: `${typeStr} ${nameStr}${sizeStr ? ` (${sizeStr})` : ""}`,
         title: `${name}`,
         style: { fontWeight: "bold" },
@@ -474,14 +474,14 @@ export const FileListItem = React.memo((props: Readonly) => {
       ctx.push({
         key: "open",
         icon: ,
-        label: isdir ? "Open folder" : "Open file",
+        label: isDir ? "Open folder" : "Open file",
         onClick: () => onClick?.(),
       });
     }
 
     ctx.push({ key: "divider-header", type: "divider" });
 
-    if (is_public && typeof onPublic === "function") {
+    if (isPublic && typeof onPublic === "function") {
       ctx.push({
         key: "public",
         label: "Item is published",
@@ -495,7 +495,7 @@ export const FileListItem = React.memo((props: Readonly) => {
 
     // view/download buttons at the bottom
     const showDownload = !student_project_functionality.disableActions;
-    if (name !== ".." && !isdir && showDownload && !multiple) {
+    if (name !== ".." && !isDir && showDownload && !multiple) {
       const full_path = path_to_file(current_path, name);
       const ext = (filename_extension(name) ?? "").toLowerCase();
       const showView = VIEWABLE_FILE_EXT.includes(ext);
@@ -528,11 +528,11 @@ export const FileListItem = React.memo((props: Readonly) => {
   // because all those files are opened
   const activeStyle: CSS =
     mode === "active"
-      ? item.isactive
+      ? item.isActive
         ? FILE_ITEM_ACTIVE_STYLE_2
         : {}
-      : item.isopen
-        ? item.isactive
+      : item.isOpen
+        ? item.isActive
           ? FILE_ITEM_ACTIVE_STYLE
           : FILE_ITEM_OPENED_STYLE
         : {};
diff --git a/src/packages/frontend/project/page/flyouts/files-bottom.tsx b/src/packages/frontend/project/page/flyouts/files-bottom.tsx
index ea186390a4..7d075bc9e2 100644
--- a/src/packages/frontend/project/page/flyouts/files-bottom.tsx
+++ b/src/packages/frontend/project/page/flyouts/files-bottom.tsx
@@ -185,8 +185,8 @@ export function FilesBottom({
 
   function renderDownloadView() {
     if (!singleFile) return;
-    const { name, isdir, size = 0 } = singleFile;
-    if (isdir) return;
+    const { name, isDir, size = 0 } = singleFile;
+    if (isDir) return;
     const full_path = path_to_file(current_path, name);
     const ext = (filename_extension(name) ?? "").toLowerCase();
     const showView = VIEWABLE_FILE_EXT.includes(ext);
@@ -276,7 +276,7 @@ export function FilesBottom({
     if (checked_files.size === 0) {
       let totSize = 0;
       for (const f of directoryFiles) {
-        if (!f.isdir) totSize += f.size ?? 0;
+        if (!f.isDir) totSize += f.size ?? 0;
       }
       return (
         

@@ -292,7 +292,7 @@ export function FilesBottom({
     if (checked_files.size === 0) {
       let [nFiles, nDirs] = [0, 0];
       for (const f of directoryFiles) {
-        if (f.isdir) {
+        if (f.isDir) {
           nDirs++;
         } else {
           nFiles++;
@@ -307,7 +307,7 @@ export function FilesBottom({
       );
     } else if (singleFile) {
       const name = singleFile.name;
-      const iconName = singleFile.isdir
+      const iconName = singleFile.isDir
         ? "folder"
         : file_options(name)?.icon ?? "file";
       return (
diff --git a/src/packages/frontend/project/page/flyouts/files-controls.tsx b/src/packages/frontend/project/page/flyouts/files-controls.tsx
index 18c2b5db2a..5f150df239 100644
--- a/src/packages/frontend/project/page/flyouts/files-controls.tsx
+++ b/src/packages/frontend/project/page/flyouts/files-controls.tsx
@@ -6,7 +6,6 @@
 import { Button, Descriptions, Space, Tooltip } from "antd";
 import immutable from "immutable";
 import { useIntl } from "react-intl";
-
 import { useActions, useTypedRedux } from "@cocalc/frontend/app-framework";
 import { Icon, TimeAgo } from "@cocalc/frontend/components";
 import {
@@ -15,7 +14,7 @@ import {
   ACTION_BUTTONS_MULTI,
   isDisabledSnapshots,
 } from "@cocalc/frontend/project/explorer/action-bar";
-import {
+import type {
   DirectoryListing,
   DirectoryListingEntry,
 } from "@cocalc/frontend/project/explorer/types";
@@ -68,7 +67,7 @@ export function FilesSelectedControls({
       const basename = path_split(file).tail;
       const index = directoryFiles.findIndex((f) => f.name === basename);
       // skipping directories, because it makes no sense to flip through them rapidly
-      if (skipDirs && getFile(file)?.isdir) {
+      if (skipDirs && getFile(file)?.isDir) {
         open(e, index, true);
         continue;
       }
@@ -83,7 +82,7 @@ export function FilesSelectedControls({
 
     let [nFiles, nDirs] = [0, 0];
     for (const f of directoryFiles) {
-      if (f.isdir) {
+      if (f.isDir) {
         nDirs++;
       } else {
         nFiles++;
@@ -100,7 +99,7 @@ export function FilesSelectedControls({
 
   function renderFileInfoBottom() {
     if (singleFile != null) {
-      const { size, mtime, isdir } = singleFile;
+      const { size, mtime, isDir } = singleFile;
       const age = typeof mtime === "number" ? mtime : null;
       return (
         
@@ -109,7 +108,7 @@ export function FilesSelectedControls({
               
             
           ) : undefined}
-          {isdir ? (
+          {isDir ? (
             
               {size} {plural(size, "item")}
             
@@ -118,7 +117,7 @@ export function FilesSelectedControls({
               {human_readable_size(size)}
             
           )}
-          {singleFile.is_public ? (
+          {singleFile.isPublic ? (
             
               
     );
@@ -253,12 +260,10 @@ export default function Configure(props: Props) {
       
         <a
           onClick={() => {
-            redux
-              .getProjectActions(props.project_id)
-              ?.load_target("files/" + props.path);
+            redux.getProjectActions(project_id)?.load_target("files/" + path);
           }}
         >
-          {trunc_middle(props.path, 128)}
+          {trunc_middle(path, 128)}
         </a>
         <span style={{ float: "right" }}>{renderFinishedButton()}</span>
       
@@ -282,7 +287,7 @@ export default function Configure(props: Props) {
       
       
         
-          {!parent_is_public && (
+          {!parentIsPublic && (
             <>
               
                 
                       
                       {STATES.public_listed} - on the{" "}
                       
                         public search engine indexed server.{" "}
-                        {!props.has_network_access && (
+                        {!has_network_access && (
                           
                             (This project must be upgraded to have Internet
                             access.)
@@ -349,16 +354,16 @@ export default function Configure(props: Props) {
               
             
           )}
-          {parent_is_public && props.public != null && (
+          {parentIsPublic && publicInfo != null && (
             
-                  This {props.isdir ? "directory" : "file"} is public because it
-                  is in the public folder "{props.public.path}". Adjust the
-                  sharing configuration of that folder instead.
+                  This is public because it is in the public folder "
+                  {publicInfo.path}". Adjust the sharing configuration of that
+                  folder instead.
                 
               }
             />
@@ -393,11 +398,11 @@ export default function Configure(props: Props) {
                   style={{ paddingTop: "5px", margin: "15px 0" }}
                   value={description}
                   onChange={(e) => setDescription(e.target.value)}
-                  disabled={parent_is_public}
+                  disabled={parentIsPublic}
                   placeholder="Describe what you are sharing.  You can change this at any time."
-                  onKeyUp={props.action_key}
+                  onKeyUp={action_key}
                   onBlur={() => {
-                    props.set_public_path({ description });
+                    set_public_path({ description });
                   }}
                 />
               

@@ -415,11 +420,9 @@ export default function Configure(props: Props) {
                 
                 
                   
-                      props.set_public_path({ license })
-                    }
+                    set_license={(license) => set_public_path({ license })}
                   />
                 
 
@@ -431,7 +434,7 @@ export default function Configure(props: Props) {
                     licenseId={licenseId}
                     setLicenseId={(licenseId) => {
                       setLicenseId(licenseId);
-                      props.set_public_path({ site_license_id: licenseId });
+                      set_public_path({ site_license_id: licenseId });
                     }}
                   />
                   
@@ -446,10 +449,10 @@ export default function Configure(props: Props) {
                 
               

                {
-                  props.set_public_path({ jupyter_api });
+                  set_public_path({ jupyter_api });
                 }}
               />
             
@@ -477,12 +480,12 @@ export default function Configure(props: Props) {
{ - props.set_public_path({ redirect }); + set_public_path({ redirect }); }} - disabled={parent_is_public} + disabled={parentIsPublic} /> diff --git a/src/packages/util/types/directory-listing.ts b/src/packages/util/types/directory-listing.ts index 3352a99268..11e82d5472 100644 --- a/src/packages/util/types/directory-listing.ts +++ b/src/packages/util/types/directory-listing.ts @@ -1,12 +1,15 @@ export interface DirectoryListingEntry { + // relative path (to containing directory) name: string; - isdir?: boolean; - issymlink?: boolean; + // number of *bytes* used to store this path. + size: number; + // last modification time in ms of this file + mtime: number; + // true if it is a directory + isDir?: boolean; + // true if it is a symlink + isSymLink?: boolean; // set if issymlink is true and we're able to determine the target of the link - link_target?: string; - // bytes for file, number of entries for directory (*including* . and ..). - size?: number; - mtime?: number; + linkTarget?: string; error?: string; - mask?: boolean; } From 322fad2a82f997bc5ca89df1183d5a0767ad351c Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 30 Jul 2025 23:50:08 +0000 Subject: [PATCH 142/798] mainly fixing publish state view and editing - also remove all use of the abbrevation 'dflt' from our codebase for variable names; this is more consistent, since we basically don't use any abbreviations. --- .../file-server/btrfs/subvolume-bup.ts | 4 +- .../admin/site-settings/row-entry.tsx | 5 +- .../frontend/course/shared-project/actions.ts | 6 +- .../course/student-projects/actions.ts | 8 +- .../frontend/custom-software/selector.tsx | 20 ++-- .../frontend/project/explorer/action-box.tsx | 9 +- .../frontend/project/explorer/explorer.tsx | 23 +++- .../explorer/file-listing/file-listing.tsx | 7 +- .../explorer/file-listing/file-row.tsx | 3 +- .../frontend/project/explorer/types.ts | 10 +- .../project/page/flyouts/files-bottom.tsx | 5 +- .../project/page/flyouts/files-controls.tsx | 4 +- .../project/page/flyouts/files-header.tsx | 7 +- .../frontend/project/page/flyouts/files.tsx | 45 +++----- .../project/settings/software-env-info.tsx | 2 +- src/packages/frontend/project_actions.ts | 4 +- src/packages/frontend/project_store.ts | 105 ++++++++---------- src/packages/frontend/share/config.tsx | 94 +++++++--------- src/packages/frontend/share/license.tsx | 9 +- src/packages/jupyter/redux/store.ts | 4 +- .../next/components/store/quota-config.tsx | 7 +- .../components/store/quota-query-params.ts | 4 +- .../next/components/store/site-license.tsx | 12 +- src/packages/util/db-schema/llm-utils.ts | 6 +- src/packages/util/sanitize-software-envs.ts | 12 +- src/packages/util/upgrades/consts.ts | 14 +-- src/packages/util/upgrades/quota.ts | 24 ++-- .../smc_sagews/tests/test_sagews_modes.py | 24 ++-- 28 files changed, 239 insertions(+), 238 deletions(-) diff --git a/src/packages/file-server/btrfs/subvolume-bup.ts b/src/packages/file-server/btrfs/subvolume-bup.ts index 3237e32dba..a8ec09a945 100644 --- a/src/packages/file-server/btrfs/subvolume-bup.ts +++ b/src/packages/file-server/btrfs/subvolume-bup.ts @@ -126,10 +126,10 @@ export class SubvolumeBup { } const mtime = parseBupTime(name).valueOf() / 1000; newest = Math.max(mtime, newest); - v.push({ name, isDir: true, mtime }); + v.push({ name, isDir: true, mtime, size: -1 }); } if (v.length > 0) { - v.push({ name: "latest", isDir: true, mtime: newest }); + v.push({ name: "latest", isDir: true, mtime: newest, size: -1 }); } return v; } diff --git a/src/packages/frontend/admin/site-settings/row-entry.tsx b/src/packages/frontend/admin/site-settings/row-entry.tsx index 8c964e00ca..22e5ed4858 100644 --- a/src/packages/frontend/admin/site-settings/row-entry.tsx +++ b/src/packages/frontend/admin/site-settings/row-entry.tsx @@ -168,9 +168,8 @@ function VersionHint({ value }: { value: string }) { // The production site works differently. // TODO: make this a more sophisticated data editor. function JsonEntry({ name, data, readonly, onJsonEntryChange }) { - const jval = JSON.parse(data ?? "{}") ?? {}; - const dflt = FIELD_DEFAULTS[name]; - const quotas = { ...dflt, ...jval }; + const jsonValue = JSON.parse(data ?? "{}") ?? {}; + const quotas = { ...FIELD_DEFAULTS[name], ...jsonValue }; const value = JSON.stringify(quotas); return ( => { diff --git a/src/packages/frontend/course/student-projects/actions.ts b/src/packages/frontend/course/student-projects/actions.ts index aeb9ea37a3..a173dda6c0 100644 --- a/src/packages/frontend/course/student-projects/actions.ts +++ b/src/packages/frontend/course/student-projects/actions.ts @@ -66,13 +66,13 @@ export class StudentProjectsActions { const id = this.course_actions.set_activity({ desc: `Create project for ${store.get_student_name(student_id)}.`, }); - const dflt_img = await redux.getStore("customize").getDefaultComputeImage(); + const defaultImage = await redux.getStore("customize").getDefaultComputeImage(); let project_id: string; try { project_id = await redux.getActions("projects").create_project({ title: store.get("settings").get("title"), description: store.get("settings").get("description"), - image: store.get("settings").get("custom_image") ?? dflt_img, + image: store.get("settings").get("custom_image") ?? defaultImage, noPool: true, // student is unlikely to use the project right *now* }); } catch (err) { @@ -607,8 +607,8 @@ export class StudentProjectsActions { ): Promise => { const store = this.get_store(); if (store == null) return; - const dflt_img = await redux.getStore("customize").getDefaultComputeImage(); - const img_id = store.get("settings").get("custom_image") ?? dflt_img; + const defaultImage = await redux.getStore("customize").getDefaultComputeImage(); + const img_id = store.get("settings").get("custom_image") ?? defaultImage; const actions = redux.getProjectActions(student_project_id); await actions.set_compute_image(img_id); }; diff --git a/src/packages/frontend/custom-software/selector.tsx b/src/packages/frontend/custom-software/selector.tsx index 7e2f716369..5e7c9309d5 100644 --- a/src/packages/frontend/custom-software/selector.tsx +++ b/src/packages/frontend/custom-software/selector.tsx @@ -3,7 +3,7 @@ * License: MS-RSL – see LICENSE.md for details */ -// cSpell:ignore descr disp dflt +// cSpell:ignore descr disp import { Col, Form } from "antd"; import { FormattedMessage, useIntl } from "react-intl"; @@ -43,11 +43,11 @@ export async function derive_project_img_name( custom_software: SoftwareEnvironmentState, ): Promise { const { image_type, image_selected } = custom_software; - const dflt_software_img = await redux + const defaultSoftwareImage = await redux .getStore("customize") .getDefaultComputeImage(); if (image_selected == null || image_type == null) { - return dflt_software_img; + return defaultSoftwareImage; } switch (image_type) { case "custom": @@ -56,7 +56,7 @@ export async function derive_project_img_name( return image_selected; default: unreachable(image_type); - return dflt_software_img; // make TS happy + return defaultSoftwareImage; // make TS happy } } @@ -77,7 +77,7 @@ export function SoftwareEnvironment(props: Props) { const onCoCalcCom = customize_kucalc === KUCALC_COCALC_COM; const customize_software = useTypedRedux("customize", "software"); const organization_name = useTypedRedux("customize", "organization_name"); - const dflt_software_img = customize_software.get("default"); + const defaultSoftwareImage = customize_software.get("default"); const software_images = customize_software.get("environments"); const haveSoftwareImages: boolean = useMemo( @@ -109,7 +109,7 @@ export function SoftwareEnvironment(props: Props) { // initialize selection, if there is a default image set React.useEffect(() => { - if (default_image == null || default_image === dflt_software_img) { + if (default_image == null || default_image === defaultSoftwareImage) { // do nothing, that's the initial state already! } else if (is_custom_image(default_image)) { if (images == null) return; @@ -123,7 +123,7 @@ export function SoftwareEnvironment(props: Props) { } else { // must be standard image const img = software_images.get(default_image); - const display = img != null ? img.get("title") ?? "" : ""; + const display = img != null ? (img.get("title") ?? "") : ""; setState(default_image, display, "standard"); } }, []); @@ -170,7 +170,7 @@ export function SoftwareEnvironment(props: Props) { } function render_onprem() { - const selected = image_selected ?? dflt_software_img; + const selected = image_selected ?? defaultSoftwareImage; return ( <> @@ -219,7 +219,7 @@ export function SoftwareEnvironment(props: Props) { } function render_standard_image_selector() { - const isCustom = is_custom_image(image_selected ?? dflt_software_img); + const isCustom = is_custom_image(image_selected ?? defaultSoftwareImage); return ( <> @@ -230,7 +230,7 @@ export function SoftwareEnvironment(props: Props) { > { diff --git a/src/packages/frontend/project/explorer/action-box.tsx b/src/packages/frontend/project/explorer/action-box.tsx index 1e05857215..2a25f09bfd 100644 --- a/src/packages/frontend/project/explorer/action-box.tsx +++ b/src/packages/frontend/project/explorer/action-box.tsx @@ -22,7 +22,10 @@ import { Icon, Loading, LoginLink } from "@cocalc/frontend/components"; import SelectServer from "@cocalc/frontend/compute/select-server"; import ComputeServerTag from "@cocalc/frontend/compute/server-tag"; import { useRunQuota } from "@cocalc/frontend/project/settings/run-quota/hooks"; -import { file_actions, ProjectActions } from "@cocalc/frontend/project_store"; +import { + file_actions, + type ProjectActions, +} from "@cocalc/frontend/project_store"; import { SelectProject } from "@cocalc/frontend/projects/select-project"; import ConfigureShare from "@cocalc/frontend/share/config"; import * as misc from "@cocalc/util/misc"; @@ -550,8 +553,8 @@ export function ActionBox({ path={path} compute_server_id={compute_server_id} close={cancel_action} - action_key={action_key} - set_public_path={(opts) => actions.set_public_path(path, opts)} + onKeyUp={action_key} + actions={actions} has_network_access={!!runQuota.network} /> ); diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index 3ec833a4aa..47349a95c4 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -6,7 +6,13 @@ import * as _ from "lodash"; import { UsersViewing } from "@cocalc/frontend/account/avatar/users-viewing"; import { Col, Row } from "@cocalc/frontend/antd-bootstrap"; -import { type CSSProperties, useEffect, useRef, useState } from "react"; +import { + type CSSProperties, + useEffect, + useMemo, + useRef, + useState, +} from "react"; import { A, ActivityDisplay, @@ -41,6 +47,10 @@ import useListing, { import filterListing from "@cocalc/frontend/project/listing/filter-listing"; import ShowError from "@cocalc/frontend/components/error"; import { MainConfiguration } from "@cocalc/frontend/project_configuration"; +import { + getPublicFiles, + useStrippedPublicPaths, +} from "@cocalc/frontend/project_store"; const FLEX_ROW_STYLE = { display: "flex", @@ -109,6 +119,7 @@ export function Explorer() { "show_custom_software_reset", ); const show_library = useTypedRedux({ project_id }, "show_library"); + const [shiftIsDown, setShiftIsDown] = useState(false); const project_map = useTypedRedux("projects", "project_map"); @@ -139,6 +150,15 @@ export function Explorer() { actions?.setState({ numDisplayedFiles: listing?.length ?? 0 }); }, [listing?.length]); + // ensure that listing entries have isPublic set: + const strippedPublicPaths = useStrippedPublicPaths(project_id); + const publicFiles: Set = useMemo(() => { + if (listing == null) { + return new Set(); + } + return getPublicFiles(listing, strippedPublicPaths, current_path); + }, [listing, current_path, strippedPublicPaths]); + useEffect(() => { if (listing == null) { return; @@ -497,6 +517,7 @@ export function Explorer() { configuration_main={ configuration?.get("main") as MainConfiguration | undefined } + publicFiles={publicFiles} /> )} diff --git a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx index 4663119389..1430f51eca 100644 --- a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx +++ b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx @@ -35,6 +35,7 @@ interface Props { configuration_main?: MainConfiguration; isRunning?: boolean; // true if this project is running stale?: boolean; + publicFiles: Set; } export function FileListing({ @@ -47,6 +48,7 @@ export function FileListing({ configuration_main, file_search = "", stale, + publicFiles, }: Props) { const active_file_sort = useTypedRedux({ project_id }, "active_file_sort"); const computeServerId = useTypedRedux({ project_id }, "compute_server_id"); @@ -55,12 +57,15 @@ export function FileListing({ const name = actions.name; function renderRow(index, file) { - const checked = checked_files.has(misc.path_to_file(current_path, name)); + const checked = checked_files.has( + misc.path_to_file(current_path, file.name), + ); const color = misc.rowBackground({ index, checked }); return ( void; selectAllFiles: () => void; getFile: (path: string) => DirectoryListingEntry | undefined; + publicFiles: Set; } export function FilesBottom({ @@ -78,6 +79,7 @@ export function FilesBottom({ showFileSharingDialog, getFile, directoryFiles, + publicFiles, }: FilesBottomProps) { const [mode, setMode] = modeState; const current_path = useTypedRedux({ project_id }, "current_path"); @@ -268,6 +270,7 @@ export function FilesBottom({ getFile={getFile} mode="bottom" activeFile={activeFile} + publicFiles={publicFiles} /> ); } @@ -309,7 +312,7 @@ export function FilesBottom({ const name = singleFile.name; const iconName = singleFile.isDir ? "folder" - : file_options(name)?.icon ?? "file"; + : (file_options(name)?.icon ?? "file"); return ( ); diff --git a/src/packages/frontend/project/settings/software-env-info.tsx b/src/packages/frontend/project/settings/software-env-info.tsx index 25ba23a6c8..15616a227c 100644 --- a/src/packages/frontend/project/settings/software-env-info.tsx +++ b/src/packages/frontend/project/settings/software-env-info.tsx @@ -3,7 +3,7 @@ * License: MS-RSL – see LICENSE.md for details */ -// cSpell:ignore descr disp dflt +// cSpell:ignore descr disp import { join } from "path"; import { FormattedMessage } from "react-intl"; diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index 3b8d507b0c..177ed7c9c6 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -2922,13 +2922,13 @@ export class ProjectActions extends Actions { const id = client_db.sha1(project_id, path); const projects_store = redux.getStore("projects"); - const dflt_compute_img = await redux + const defaultComputeImage = await redux .getStore("customize") .getDefaultComputeImage(); const compute_image: string = projects_store.getIn(["project_map", project_id, "compute_image"]) ?? - dflt_compute_img; + defaultComputeImage; const table = this.redux.getProjectTable(project_id, "public_paths"); let obj: undefined | Map = table._table.get(id); diff --git a/src/packages/frontend/project_store.ts b/src/packages/frontend/project_store.ts index 78fb2e0aea..4e92521389 100644 --- a/src/packages/frontend/project_store.ts +++ b/src/packages/frontend/project_store.ts @@ -16,7 +16,6 @@ if (typeof window !== "undefined" && window !== null) { } import * as immutable from "immutable"; - import { AppRedux, project_redux_name, @@ -24,7 +23,9 @@ import { Store, Table, TypedMap, + useTypedRedux, } from "@cocalc/frontend/app-framework"; +import { useMemo } from "react"; import { fileURL } from "@cocalc/frontend/lib/cocalc-urls"; import { get_local_storage } from "@cocalc/frontend/misc"; import { QueryParams } from "@cocalc/frontend/misc/query-params"; @@ -40,7 +41,7 @@ import { isMainConfiguration, ProjectConfiguration, } from "@cocalc/frontend/project_configuration"; -import * as misc from "@cocalc/util/misc"; +import { containing_public_path, deep_copy } from "@cocalc/util/misc"; import { FixedTab } from "./project/page/file-tab"; import { FlyoutActiveMode, @@ -54,7 +55,8 @@ import { FLYOUT_LOG_FILTER_DEFAULT, FlyoutLogFilter, } from "./project/page/flyouts/utils"; - +import { type PublicPath } from "@cocalc/util/db-schema/public-paths"; +import { DirectoryListing } from "@cocalc/frontend/project/explorer/types"; export { FILE_ACTIONS as file_actions, ProjectActions }; export type ModalInfo = TypedMap<{ @@ -71,7 +73,7 @@ export interface ProjectStoreState { open_files: immutable.Map>; open_files_order: immutable.List; just_closed_files: immutable.List; - public_paths?: immutable.Map>; + public_paths?: immutable.Map>; show_upload: boolean; create_file_alert: boolean; @@ -145,7 +147,6 @@ export interface ProjectStoreState { // Project Settings get_public_path_id?: (path: string) => any; - stripped_public_paths: any; //computed(immutable.List) // Project Info show_project_info_explanation?: boolean; @@ -330,8 +331,6 @@ export class ProjectStore extends Store { most_recent_path: "", // Project Settings - stripped_public_paths: this.selectors.stripped_public_paths.fn, - other_settings: undefined, compute_server_id, @@ -360,26 +359,6 @@ export class ProjectStore extends Store { }; }, }, - - stripped_public_paths: { - dependencies: ["public_paths"] as const, - fn: () => { - const public_paths = this.get("public_paths"); - if (public_paths != null) { - return immutable.fromJS( - (() => { - const result: any[] = []; - const object = public_paths.toJS(); - for (const id in object) { - const x = object[id]; - result.push(misc.copy_without(x, ["id", "project_id"])); - } - return result; - })(), - ); - } - }, - }, }; // Returns the cursor positions for the given project_id/path, if that @@ -432,37 +411,41 @@ export class ProjectStore extends Store { } } -// Mutates data to include info on public paths. -export function mutate_data_to_compute_public_files( - data, - public_paths, - current_path, -) { - const { listing } = data; - const pub = data.public; - if (public_paths != null && public_paths.size > 0) { - const head = current_path ? current_path + "/" : ""; - const paths: string[] = []; - const public_path_data = {}; - for (const x of public_paths.toJS()) { - if (x.disabled) { - // Do not include disabled paths. Otherwise, it causes this confusing bug: - // https://github.com/sagemathinc/cocalc/issues/6159 - continue; - } - public_path_data[x.path] = x; - paths.push(x.path); - } - for (const x of listing) { - const full = head + x.name; - const p = misc.containing_public_path(full, paths); - if (p != null) { - x.public = public_path_data[p]; - x.is_public = !x.public.disabled; - pub[x.name] = public_path_data[p]; - } +// Returns set of paths that are public in the given +// listing, because they are in a public folder or are themselves public. +// This is used entirely to put an extra "public" label in the row of the file, +// when displaying it in a listing. +export function getPublicFiles( + listing: DirectoryListing, + public_paths: PublicPath[], + current_path: string, +): Set { + if ((public_paths?.length ?? 0) == 0) { + return new Set(); + } + const paths = public_paths + .filter(({ disabled }) => !disabled) + .map(({ path }) => path); + + if (paths.length == 0) { + return new Set(); + } + + const head = current_path ? current_path + "/" : ""; + if (containing_public_path(current_path, paths)) { + // fast special case: *every* file is public + return new Set(listing.map(({ name }) => name)); + } + + // maybe some files are public? + const X = new Set(); + for (const file of listing) { + const full = head + file.name; + if (containing_public_path(full, paths) != null) { + X.add(file.name); } } + return X; } export function init(project_id: string, redux: AppRedux): ProjectStore { @@ -486,7 +469,7 @@ export function init(project_id: string, redux: AppRedux): ProjectStore { actions.project_id = project_id; // so actions can assume this is available on the object store._init(); - const queries = misc.deep_copy(QUERIES); + const queries = deep_copy(QUERIES); const create_table = function (table_name, q) { //console.log("create_table", table_name) @@ -543,3 +526,11 @@ export function init(project_id: string, redux: AppRedux): ProjectStore { return store; } + +export function useStrippedPublicPaths(project_id: string): PublicPath[] { + const public_paths = useTypedRedux({ project_id }, "public_paths"); + return useMemo(() => { + const rows = public_paths?.valueSeq()?.toJS() ?? []; + return rows as unknown as PublicPath[]; + }, [public_paths]); +} diff --git a/src/packages/frontend/share/config.tsx b/src/packages/frontend/share/config.tsx index 527aa01524..48b0a407a8 100644 --- a/src/packages/frontend/share/config.tsx +++ b/src/packages/frontend/share/config.tsx @@ -33,8 +33,7 @@ import { Row, Space, } from "antd"; -import { useEffect, useState } from "react"; - +import { useEffect, useMemo, useState } from "react"; import { CSS, redux, useTypedRedux } from "@cocalc/frontend/app-framework"; import { A, @@ -60,45 +59,19 @@ import { COLORS } from "@cocalc/util/theme"; import { ConfigureName } from "./configure-name"; import { License } from "./license"; import { publicShareUrl, shareServerUrl } from "./util"; +import { containing_public_path } from "@cocalc/util/misc"; +import { type PublicPath } from "@cocalc/util/db-schema/public-paths"; +import { type ProjectActions } from "@cocalc/frontend/project_store"; // https://ant.design/components/grid/ const GUTTER: [number, number] = [20, 30]; -interface PublicInfo { - created: Date; - description: string; - disabled: boolean; - last_edited: Date; - path: string; - unlisted: boolean; - authenticated?: boolean; - license?: string; - name?: string; - site_license_id?: string; - redirect?: string; - jupyter_api?: boolean; -} - interface Props { project_id: string; path: string; - size: number; - mtime: number; - isPublic?: boolean; - publicInfo?: PublicInfo; close: (event: any) => void; - action_key: (event: any) => void; - site_license_id?: string; - set_public_path: (options: { - description?: string; - unlisted?: boolean; - license?: string; - disabled?: boolean; - authenticated?: boolean; - site_license_id?: string | null; - redirect?: string; - jupyter_api?: boolean; - }) => void; + onKeyUp?: (event: any) => void; + actions: ProjectActions; has_network_access?: boolean; compute_server_id?: number; } @@ -123,26 +96,40 @@ function SC({ children }) { export default function Configure({ project_id, path, - isPublic, - publicInfo, close, - action_key, - set_public_path, + onKeyUp, + actions, has_network_access, compute_server_id, }: Props) { + const publicPaths = useTypedRedux({ project_id }, "public_paths"); + const publicInfo: null | PublicPath = useMemo(() => { + for (const x of publicPaths?.valueSeq() ?? []) { + if ( + !x.get("disabled") && + containing_public_path(path, [x.get("path")]) != null + ) { + return x.toJS(); + } + } + return null; + }, [publicPaths]); + const student = useStudentProjectFunctionality(project_id); const [description, setDescription] = useState( publicInfo?.description ?? "", ); const [sharingOptionsState, setSharingOptionsState] = useState(() => { - if (isPublic && publicInfo?.unlisted) { + if (publicInfo == null) { + return "private"; + } + if (publicInfo?.unlisted) { return "public_unlisted"; } - if (isPublic && publicInfo?.authenticated) { + if (publicInfo?.authenticated) { return "authenticated"; } - if (isPublic && !publicInfo?.unlisted) { + if (!publicInfo?.unlisted) { return "public_listed"; } return "private"; @@ -176,17 +163,17 @@ export default function Configure({ setSharingOptionsState(state); switch (state) { case "private": - set_public_path(SHARE_FLAGS.DISABLED); + actions.set_public_path(path, SHARE_FLAGS.DISABLED); break; case "public_listed": // public is suppose to work in this state - set_public_path(SHARE_FLAGS.LISTED); + actions.set_public_path(path, SHARE_FLAGS.LISTED); break; case "public_unlisted": - set_public_path(SHARE_FLAGS.UNLISTED); + actions.set_public_path(path, SHARE_FLAGS.UNLISTED); break; case "authenticated": - set_public_path(SHARE_FLAGS.AUTHENTICATED); + actions.set_public_path(path, SHARE_FLAGS.AUTHENTICATED); break; default: unreachable(state); @@ -196,8 +183,7 @@ export default function Configure({ const license = publicInfo?.license ?? ""; // This path is public because some parent folder is public. - const parentIsPublic = - !!isPublic && publicInfo != null && publicInfo.path != path; + const parentIsPublic = publicInfo != null && publicInfo.path != path; const url = publicShareUrl( project_id, @@ -400,9 +386,9 @@ export default function Configure({ onChange={(e) => setDescription(e.target.value)} disabled={parentIsPublic} placeholder="Describe what you are sharing. You can change this at any time." - onKeyUp={action_key} + onKeyUp={onKeyUp} onBlur={() => { - set_public_path({ description }); + actions.set_public_path(path, { description }); }} />
{trunc_middle(name, 20)} diff --git a/src/packages/frontend/project/page/flyouts/files-controls.tsx b/src/packages/frontend/project/page/flyouts/files-controls.tsx index 5f150df239..7246a73666 100644 --- a/src/packages/frontend/project/page/flyouts/files-controls.tsx +++ b/src/packages/frontend/project/page/flyouts/files-controls.tsx @@ -37,6 +37,7 @@ interface FilesSelectedControlsProps { skip?: boolean, ) => void; activeFile: DirectoryListingEntry | null; + publicFiles: Set; } export function FilesSelectedControls({ @@ -48,6 +49,7 @@ export function FilesSelectedControls({ project_id, showFileSharingDialog, activeFile, + publicFiles, }: FilesSelectedControlsProps) { const intl = useIntl(); const current_path = useTypedRedux({ project_id }, "current_path"); @@ -117,7 +119,7 @@ export function FilesSelectedControls({ {human_readable_size(size)} )} - {singleFile.isPublic ? ( + {publicFiles.has(singleFile.name) ? (
@@ -422,7 +408,9 @@ export default function Configure({ set_public_path({ license })} + set_license={(license) => + actions.set_public_path(path, { license }) + } /> @@ -434,7 +422,9 @@ export default function Configure({ licenseId={licenseId} setLicenseId={(licenseId) => { setLicenseId(licenseId); - set_public_path({ site_license_id: licenseId }); + actions.set_public_path(path, { + site_license_id: licenseId, + }); }} /> @@ -452,7 +442,7 @@ export default function Configure({ disabled={parentIsPublic} jupyter_api={publicInfo?.jupyter_api} saveJupyterApi={(jupyter_api) => { - set_public_path({ jupyter_api }); + actions.set_public_path(path, { jupyter_api }); }} /> @@ -483,7 +473,7 @@ export default function Configure({ project_id={project_id} path={publicInfo?.path ?? path} saveRedirect={(redirect) => { - set_public_path({ redirect }); + actions.set_public_path(path, { redirect }); }} disabled={parentIsPublic} /> diff --git a/src/packages/frontend/share/license.tsx b/src/packages/frontend/share/license.tsx index 784e08d6b8..819c9f1e96 100644 --- a/src/packages/frontend/share/license.tsx +++ b/src/packages/frontend/share/license.tsx @@ -12,8 +12,7 @@ between them. I think this is acceptable, since it is unlikely for people to do that. */ -import { FC, memo, useMemo, useState } from "react"; - +import { useMemo, useState } from "react"; import { DropdownMenu } from "@cocalc/frontend/components"; import { MenuItems } from "../components/dropdown-menu"; import { LICENSES } from "./licenses"; @@ -24,9 +23,7 @@ interface Props { disabled?: boolean; } -export const License: FC = memo((props: Props) => { - const { license, set_license, disabled = false } = props; - +export function License({ license, set_license, disabled = false }: Props) { const [sel_license, set_sel_license] = useState(license); function select(license: string): void { @@ -65,4 +62,4 @@ export const License: FC = memo((props: Props) => { items={items} /> ); -}); +} diff --git a/src/packages/jupyter/redux/store.ts b/src/packages/jupyter/redux/store.ts index e611b59eec..a334be7de2 100644 --- a/src/packages/jupyter/redux/store.ts +++ b/src/packages/jupyter/redux/store.ts @@ -451,10 +451,10 @@ export class JupyterStore extends Store { // (??) return `${project_id}-${computeServerId}-default`; } - const dflt_img = await customize.getDefaultComputeImage(); + const defaultImage = await customize.getDefaultComputeImage(); const compute_image = projects_store.getIn( ["project_map", project_id, "compute_image"], - dflt_img, + defaultImage, ); const key = [project_id, `${computeServerId}`, compute_image].join("::"); // console.log("jupyter store / jupyter_kernel_key", key); diff --git a/src/packages/next/components/store/quota-config.tsx b/src/packages/next/components/store/quota-config.tsx index 56a65c62c5..91da4eeaf1 100644 --- a/src/packages/next/components/store/quota-config.tsx +++ b/src/packages/next/components/store/quota-config.tsx @@ -17,7 +17,6 @@ import { Typography, } from "antd"; import { useEffect, useRef, useState, type JSX } from "react"; - import { HelpIcon } from "@cocalc/frontend/components/help-icon"; import { Icon } from "@cocalc/frontend/components/icon"; import { displaySiteLicense } from "@cocalc/util/consts/site-license"; @@ -193,7 +192,7 @@ export const QuotaConfig: React.FC = (props: Props) => { = (props: Props) => { = (props: Props) => { diff --git a/src/packages/next/components/store/quota-query-params.ts b/src/packages/next/components/store/quota-query-params.ts index 92e85f30aa..c492089e6f 100644 --- a/src/packages/next/components/store/quota-query-params.ts +++ b/src/packages/next/components/store/quota-query-params.ts @@ -138,10 +138,10 @@ function decodeValue(val): boolean | number | string | DateRange { function fixNumVal( val: any, - param: { min: number; max: number; dflt: number }, + param: { min: number; max: number; default: number }, ): number { if (typeof val !== "number") { - return param.dflt; + return param.default; } else { return clamp(val, param.min, param.max); } diff --git a/src/packages/next/components/store/site-license.tsx b/src/packages/next/components/store/site-license.tsx index 1b1f99f858..4f46d568d8 100644 --- a/src/packages/next/components/store/site-license.tsx +++ b/src/packages/next/components/store/site-license.tsx @@ -292,14 +292,14 @@ function CreateSiteLicense({ })(); } else { const vals = decodeFormValues(router, "regular"); - const dflt = presets[DEFAULT_PRESET]; + const defaultPreset = presets[DEFAULT_PRESET]; // Only use the configuration fields from the default preset, not the entire object const defaultConfig = { - cpu: dflt.cpu, - ram: dflt.ram, - disk: dflt.disk, - uptime: dflt.uptime, - member: dflt.member, + cpu: defaultPreset.cpu, + ram: defaultPreset.ram, + disk: defaultPreset.disk, + uptime: defaultPreset.uptime, + member: defaultPreset.member, // Add other form fields that might be needed period: source === "course" ? "range" : "monthly", user: source === "course" ? "academic" : "business", diff --git a/src/packages/util/db-schema/llm-utils.ts b/src/packages/util/db-schema/llm-utils.ts index 0542461cfb..092c81681f 100644 --- a/src/packages/util/db-schema/llm-utils.ts +++ b/src/packages/util/db-schema/llm-utils.ts @@ -428,15 +428,15 @@ export function getValidLanguageModelName({ } for (const free of [true, false]) { - const dflt = getDefaultLLM( + const defaultModel = getDefaultLLM( selectable_llms, filter, ollama, custom_openai, free, ); - if (dflt != null) { - return dflt; + if (defaultModel != null) { + return defaultModel; } } return DEFAULT_MODEL; diff --git a/src/packages/util/sanitize-software-envs.ts b/src/packages/util/sanitize-software-envs.ts index 074ba1ac4d..8893d8b851 100644 --- a/src/packages/util/sanitize-software-envs.ts +++ b/src/packages/util/sanitize-software-envs.ts @@ -118,19 +118,19 @@ export function sanitizeSoftwareEnv( return null; } - const swDflt = software["default"]; + const swDefault = software["default"]; // we check that the default is a string and that it exists in envs - const dflt = - typeof swDflt === "string" && envs[swDflt] != null - ? swDflt + const defaultSoftware = + typeof swDefault === "string" && envs[swDefault] != null + ? swDefault : Object.keys(envs)[0]; // this is a fallback entry, when projects were created before the software env was configured if (envs[DEFAULT_COMPUTE_IMAGE] == null) { - envs[DEFAULT_COMPUTE_IMAGE] = { ...envs[dflt], hidden: true }; + envs[DEFAULT_COMPUTE_IMAGE] = { ...envs[defaultSoftware], hidden: true }; } - return { groups, default: dflt, environments: envs }; + return { groups, default: defaultSoftware, environments: envs }; } function fallback(a: any, b: any, c?: string): any { diff --git a/src/packages/util/upgrades/consts.ts b/src/packages/util/upgrades/consts.ts index 6e6ae8d409..90b4d248ea 100644 --- a/src/packages/util/upgrades/consts.ts +++ b/src/packages/util/upgrades/consts.ts @@ -22,7 +22,7 @@ export const MIN_DISK_GB = DISK_DEFAULT_GB; interface Values { min: number; - dflt: number; + default: number; max: number; } @@ -35,25 +35,25 @@ interface Limits { export const REGULAR: Limits = { cpu: { min: 1, - dflt: DEFAULT_CPU, + default: DEFAULT_CPU, max: MAX_CPU, }, ram: { min: 4, - dflt: RAM_DEFAULT_GB, + default: RAM_DEFAULT_GB, max: MAX_RAM_GB, }, disk: { min: MIN_DISK_GB, - dflt: DISK_DEFAULT_GB, + default: DISK_DEFAULT_GB, max: MAX_DISK_GB, }, } as const; export const BOOST: Limits = { - cpu: { min: 0, dflt: 0, max: MAX_CPU - 1 }, - ram: { min: 0, dflt: 0, max: MAX_RAM_GB - 1 }, - disk: { min: 0, dflt: 0, max: MAX_DISK_GB - 1 * DISK_DEFAULT_GB }, + cpu: { min: 0, default: 0, max: MAX_CPU - 1 }, + ram: { min: 0, default: 0, max: MAX_RAM_GB - 1 }, + disk: { min: 0, default: 0, max: MAX_DISK_GB - 1 * DISK_DEFAULT_GB }, } as const; // on-prem: this dedicated VM machine name is only used for cocalc-onprem diff --git a/src/packages/util/upgrades/quota.ts b/src/packages/util/upgrades/quota.ts index 8e532e3887..a7980a6428 100644 --- a/src/packages/util/upgrades/quota.ts +++ b/src/packages/util/upgrades/quota.ts @@ -637,21 +637,21 @@ function calcSiteLicenseQuotaIdleTimeout( // there is an old schema, inherited from SageMathCloud, etc. and newer iterations. // this helps by going from one schema to the newer one function upgrade2quota(up: Partial): RQuota { - const dflt_false = (x) => + const defaultFalse = (x) => x != null ? (typeof x === "boolean" ? x : to_int(x) >= 1) : false; - const dflt_num = (x) => + const defaultNumber = (x) => x != null ? (typeof x === "number" ? x : to_float(x)) : 0; return { - network: dflt_false(up.network), - member_host: dflt_false(up.member_host), - always_running: dflt_false(up.always_running), - disk_quota: dflt_num(up.disk_quota), - memory_limit: dflt_num(up.memory), - memory_request: dflt_num(up.memory_request), - cpu_limit: dflt_num(up.cores), - cpu_request: dflt_num(up.cpu_shares) / 1024, - privileged: dflt_false(up.privileged), - idle_timeout: dflt_num(up.mintime), + network: defaultFalse(up.network), + member_host: defaultFalse(up.member_host), + always_running: defaultFalse(up.always_running), + disk_quota: defaultNumber(up.disk_quota), + memory_limit: defaultNumber(up.memory), + memory_request: defaultNumber(up.memory_request), + cpu_limit: defaultNumber(up.cores), + cpu_request: defaultNumber(up.cpu_shares) / 1024, + privileged: defaultFalse(up.privileged), + idle_timeout: defaultNumber(up.mintime), dedicated_vm: false, // old schema has no dedicated_vm upgrades dedicated_disks: [] as DedicatedDisk[], // old schema has no dedicated_disk upgrades ext_rw: false, diff --git a/src/smc_sagews/smc_sagews/tests/test_sagews_modes.py b/src/smc_sagews/smc_sagews/tests/test_sagews_modes.py index 0f87827740..54ffd9a4f3 100644 --- a/src/smc_sagews/smc_sagews/tests/test_sagews_modes.py +++ b/src/smc_sagews/smc_sagews/tests/test_sagews_modes.py @@ -200,34 +200,34 @@ def test_bad_command(self, exec2): class TestShDefaultMode: - def test_start_sh_dflt(self, exec2): + def test_start_sh_default(self, exec2): exec2("%default_mode sh") - def test_multiline_dflt(self, exec2): + def test_multiline_default(self, exec2): exec2("FOO=hello\necho $FOO", pattern="^hello") def test_date(self, exec2): exec2("date +%Y-%m-%d", pattern=r'^\d{4}-\d{2}-\d{2}') - def test_capture_sh_01_dflt(self, exec2): + def test_capture_sh_01_default(self, exec2): exec2("%capture(stdout='output')\nuptime") - def test_capture_sh_02_dflt(self, exec2): + def test_capture_sh_02_default(self, exec2): exec2("%sage\noutput", pattern="up.*user.*load average") - def test_remember_settings_01_dflt(self, exec2): + def test_remember_settings_01_default(self, exec2): exec2("FOO='testing123'") - def test_remember_settings_02_dflt(self, exec2): + def test_remember_settings_02_default(self, exec2): exec2("echo $FOO", pattern=r"^testing123\s+") - def test_sh_display_dflt(self, execblob, image_file): + def test_sh_display_default(self, execblob, image_file): execblob("display < " + str(image_file), want_html=False) - def test_sh_autocomplete_01_dflt(self, exec2): + def test_sh_autocomplete_01_default(self, exec2): exec2("TESTVAR29=xyz") - def test_sh_autocomplete_02_dflt(self, execintrospect): + def test_sh_autocomplete_02_default(self, execintrospect): execintrospect('echo $TESTV', ["AR29"], '$TESTV') @@ -249,13 +249,13 @@ class TestRDefaultMode: def test_set_r_mode(self, exec2): exec2("%default_mode r") - def test_rdflt_assignment(self, exec2): + def test_rdefault_assignment(self, exec2): exec2("xx <- c(4,7,13)\nmean(xx)", html_pattern="^8$") - def test_dflt_capture_r_01(self, exec2): + def test_default_capture_r_01(self, exec2): exec2("%capture(stdout='output')\nsum(xx)") - def test_dflt_capture_r_02(self, exec2): + def test_default_capture_r_02(self, exec2): exec2("%sage\nprint(output)", "24\n") From 1f04e770b6061990ee937d6bbc78c5503d41598e Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 31 Jul 2025 00:30:04 +0000 Subject: [PATCH 143/798] fix isActive and isOpen for file listings to get updated; also make isOpen work for the main explorer --- .../explorer/file-listing/file-listing.tsx | 12 ++++--- .../explorer/file-listing/file-row.tsx | 11 +++++-- .../frontend/project/explorer/types.ts | 8 ----- .../project/page/flyouts/file-list-item.tsx | 2 +- .../frontend/project/page/flyouts/files.tsx | 33 +++++++++---------- 5 files changed, 32 insertions(+), 34 deletions(-) diff --git a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx index 1430f51eca..536bef1a62 100644 --- a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx +++ b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx @@ -16,7 +16,7 @@ import { TypedMap, useTypedRedux, redux } from "@cocalc/frontend/app-framework"; import useVirtuosoScrollHook from "@cocalc/frontend/components/virtuoso-scroll-hook"; import { ProjectActions } from "@cocalc/frontend/project_actions"; import { MainConfiguration } from "@cocalc/frontend/project_configuration"; -import * as misc from "@cocalc/util/misc"; +import { path_to_file, rowBackground } from "@cocalc/util/misc"; import { FileRow } from "./file-row"; import { ListingHeader } from "./listing-header"; import NoFiles from "./no-files"; @@ -55,16 +55,18 @@ export function FileListing({ const selected_file_index = useTypedRedux({ project_id }, "selected_file_index") ?? 0; const name = actions.name; + const openFiles = new Set( + useTypedRedux({ project_id }, "open_files_order")?.toJS() ?? [], + ); function renderRow(index, file) { - const checked = checked_files.has( - misc.path_to_file(current_path, file.name), - ); - const color = misc.rowBackground({ index, checked }); + const checked = checked_files.has(path_to_file(current_path, file.name)); + const color = rowBackground({ index, checked }); return ( = [ "md", @@ -49,6 +50,7 @@ interface Props { color: string; mask: boolean; isPublic: boolean; + isOpen: boolean; current_path: string; actions: ProjectActions; no_select: boolean; @@ -70,6 +72,7 @@ export function FileRow({ color, mask, isPublic, + isOpen, current_path, actions, no_select, @@ -149,12 +152,14 @@ export function FileRow({ (display_name != undefined && name0 !== display_name) || name0.length > 50; - const styles = { + const style = { whiteSpace: "pre-wrap", wordWrap: "break-word", overflowWrap: "break-word", verticalAlign: "middle", color: mask ? "#bbbbbb" : COLORS.TAB, + ...(isOpen ? FILE_ITEM_OPENED_STYLE : undefined), + backgroundColor: undefined, }; if (show_tip) { @@ -167,11 +172,11 @@ export function FileRow({ } tip={name0} > - {render_name_link(styles, name0, ext)} + {render_name_link(style, name0, ext)} ); } else { - return render_name_link(styles, name0, ext); + return render_name_link(style, name0, ext); } } diff --git a/src/packages/frontend/project/explorer/types.ts b/src/packages/frontend/project/explorer/types.ts index 563f12b4ab..cbab305a19 100644 --- a/src/packages/frontend/project/explorer/types.ts +++ b/src/packages/frontend/project/explorer/types.ts @@ -9,14 +9,6 @@ import type { DirectoryListingEntry as DirectoryListingEntry0 } from "@cocalc/ut export interface DirectoryListingEntry extends DirectoryListingEntry0 { // whether or not to mask this file in the UI mask?: boolean; - - // This is used in flyout panels. TODO: Mutating listings based on status info - // like this that randomly changes (unlik mask above) will lead to subtle state bugs or requiring - // inefficient frequent rerenders. Instead one should make a separate - // Set of the paths of open files and active files and use that in the UI. I'm not fixing - // this now since it is only used in the flyout panels and not the main explorer. - isOpen?: boolean; - isActive?: boolean; } export type DirectoryListing = DirectoryListingEntry[]; diff --git a/src/packages/frontend/project/page/flyouts/file-list-item.tsx b/src/packages/frontend/project/page/flyouts/file-list-item.tsx index 45755b57e1..44c74426ce 100644 --- a/src/packages/frontend/project/page/flyouts/file-list-item.tsx +++ b/src/packages/frontend/project/page/flyouts/file-list-item.tsx @@ -42,7 +42,7 @@ const FILE_ITEM_SELECTED_STYLE: CSS = { backgroundColor: COLORS.BLUE_LLL, // bit darker than .cc-project-flyout-file-item:hover } as const; -const FILE_ITEM_OPENED_STYLE: CSS = { +export const FILE_ITEM_OPENED_STYLE: CSS = { fontWeight: "bold", backgroundColor: COLORS.GRAY_LL, color: COLORS.PROJECT.FIXED_LEFT_ACTIVE, diff --git a/src/packages/frontend/project/page/flyouts/files.tsx b/src/packages/frontend/project/page/flyouts/files.tsx index 2b0e73bec7..5834d2ec0c 100644 --- a/src/packages/frontend/project/page/flyouts/files.tsx +++ b/src/packages/frontend/project/page/flyouts/files.tsx @@ -100,7 +100,9 @@ export function FilesFlyout({ const show_masked = useTypedRedux({ project_id }, "show_masked"); const hidden = useTypedRedux({ project_id }, "show_hidden"); const checked_files = useTypedRedux({ project_id }, "checked_files"); - const openFiles = useTypedRedux({ project_id }, "open_files_order"); + const openFiles = new Set( + useTypedRedux({ project_id }, "open_files_order")?.toJS() ?? [], + ); // mainly controls what a single click does, plus additional UI elements const [mode, setMode] = useState<"open" | "select">("open"); const [prevSelected, setPrevSelected] = useState(null); @@ -192,17 +194,6 @@ export function FilesFlyout({ } }); - for (const file of processedFiles) { - const fullPath = path_to_file(current_path, file.name); - if (openFiles.some((path) => path == fullPath)) { - file.isOpen = true; - } - if (activePath === fullPath) { - file.isActive = true; - activeFile = file; - } - } - if (activeFileSort.get("is_descending")) { processedFiles.reverse(); // inplace op } @@ -229,12 +220,15 @@ export function FilesFlyout({ activeFileSort, hidden, file_search, - openFiles, show_masked, current_path, strippedPublicPaths, ]); + const isOpen = (file) => openFiles.has(path_to_file(current_path, file.name)); + const isActive = (file) => + activePath == path_to_file(current_path, file.name); + const publicFiles = getPublicFiles( directoryFiles, strippedPublicPaths, @@ -393,7 +387,7 @@ export function FilesFlyout({ } // similar, if in open mode and already opened, just switch to it as well - if (mode === "open" && file.isOpen && !e.shiftKey && !e.ctrlKey) { + if (mode === "open" && isOpen(file) && !e.shiftKey && !e.ctrlKey) { setPrevSelected(index); open(e, index); return; @@ -457,13 +451,13 @@ export function FilesFlyout({ } function renderTimeAgo(item: DirectoryListingEntry) { - const { mtime, isOpen = false } = item; + const { mtime } = item; if (typeof mtime === "number") { return ( ); } @@ -515,7 +509,12 @@ export function FilesFlyout({ return ( Date: Thu, 31 Jul 2025 00:34:27 +0000 Subject: [PATCH 144/798] fix a test --- src/packages/test/project/listing/use-files.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/packages/test/project/listing/use-files.test.ts b/src/packages/test/project/listing/use-files.test.ts index f36ad76040..ca113bbbb8 100644 --- a/src/packages/test/project/listing/use-files.test.ts +++ b/src/packages/test/project/listing/use-files.test.ts @@ -69,6 +69,9 @@ describe("the useFiles hook", () => { await waitFor(() => { expect(result.current.files?.["hello.txt"]).not.toBeDefined(); }); + await waitFor(() => { + expect(result.current.error).not.toBe(null); + }); expect(result.current.error?.code).toBe("ENOENT"); await act(async () => { From 0e23bab2b56b50e1b1a5f1d3380c0bd473f03a03 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 31 Jul 2025 00:44:14 +0000 Subject: [PATCH 145/798] change tests for how we changed hash of saved version and is read only --- src/packages/sync/editor/generic/sync-doc.ts | 29 ++++++++++--------- .../sync/editor/string/test/sync.0.test.ts | 3 +- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 2fe3749193..b949d1b529 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -2291,19 +2291,22 @@ export class SyncDoc extends EventEmitter { ); wait_until_read_only_known = async (): Promise => { - await until(async () => { - if (this.isClosed()) { - return true; - } - if (this.stats != null) { - return true; - } - try { - await this.stat(); - return true; - } catch {} - return false; - }); + await until( + async () => { + if (this.isClosed()) { + return true; + } + if (this.stats != null) { + return true; + } + try { + await this.stat(); + return true; + } catch {} + return false; + }, + { min: 3000 }, + ); }; /* Returns true if the current live version of this document has diff --git a/src/packages/sync/editor/string/test/sync.0.test.ts b/src/packages/sync/editor/string/test/sync.0.test.ts index 82f080dac9..0dc3607e5f 100644 --- a/src/packages/sync/editor/string/test/sync.0.test.ts +++ b/src/packages/sync/editor/string/test/sync.0.test.ts @@ -142,12 +142,11 @@ describe("create a blank minimal string SyncDoc and call public methods on it", }); it("read only checks", async () => { - await syncstring.wait_until_read_only_known(); // no-op expect(syncstring.is_read_only()).toBe(false); }); it("hashes of versions", () => { - expect(syncstring.hash_of_saved_version()).toBe(0); + expect(syncstring.hash_of_saved_version()).toBe(undefined); expect(syncstring.hash_of_live_version()).toBe(0); expect(syncstring.has_uncommitted_changes()).toBe(false); }); From 102cae04da58850a504acc47822e3f6655e7381f Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 31 Jul 2025 04:25:45 +0000 Subject: [PATCH 146/798] remove the entire open-files implementation, front to back - we have chosen a different path --- src/packages/backend/conat/sync.ts | 4 - .../conat/test/sync/open-files.test.ts | 132 ----- src/packages/conat/sync/open-files.ts | 302 ---------- src/packages/frontend/client/client.ts | 20 - src/packages/frontend/conat/client.ts | 54 -- .../terminal-editor/conat-terminal.ts | 21 - src/packages/frontend/project_actions.ts | 10 - src/packages/project/client.ts | 11 - src/packages/project/conat/api/index.ts | 7 +- src/packages/project/conat/index.ts | 4 +- src/packages/project/conat/open-files.ts | 516 ------------------ src/packages/project/conat/sync.ts | 11 +- src/packages/sync/editor/generic/types.ts | 6 - 13 files changed, 4 insertions(+), 1094 deletions(-) delete mode 100644 src/packages/backend/conat/test/sync/open-files.test.ts delete mode 100644 src/packages/conat/sync/open-files.ts delete mode 100644 src/packages/project/conat/open-files.ts diff --git a/src/packages/backend/conat/sync.ts b/src/packages/backend/conat/sync.ts index 3bccd54978..50963ac696 100644 --- a/src/packages/backend/conat/sync.ts +++ b/src/packages/backend/conat/sync.ts @@ -7,7 +7,6 @@ import { dkv as createDKV, type DKV, type DKVOptions } from "@cocalc/conat/sync/ import { dko as createDKO, type DKO } from "@cocalc/conat/sync/dko"; import { akv as createAKV, type AKV } from "@cocalc/conat/sync/akv"; import { astream as createAStream, type AStream } from "@cocalc/conat/sync/astream"; -import { createOpenFiles, type OpenFiles } from "@cocalc/conat/sync/open-files"; export { inventory } from "@cocalc/conat/sync/inventory"; import "./index"; @@ -35,6 +34,3 @@ export async function dko(opts: DKVOptions): Promise> { return await createDKO(opts); } -export async function openFiles(project_id: string, opts?): Promise { - return await createOpenFiles({ project_id, ...opts }); -} diff --git a/src/packages/backend/conat/test/sync/open-files.test.ts b/src/packages/backend/conat/test/sync/open-files.test.ts deleted file mode 100644 index 371ba6a476..0000000000 --- a/src/packages/backend/conat/test/sync/open-files.test.ts +++ /dev/null @@ -1,132 +0,0 @@ -/* -Unit test basic functionality of the openFiles distributed key:value -store. Projects and compute servers use this to know what files -to open so they can fulfill their backend responsibilities: - - computation - - save to disk - - load from disk when file changes - -DEVELOPMENT: - -pnpm test ./open-files.test.ts - -*/ - -import { openFiles as createOpenFiles } from "@cocalc/backend/conat/sync"; -import { once } from "@cocalc/util/async-utils"; -import { delay } from "awaiting"; -import { before, after, wait } from "@cocalc/backend/conat/test/setup"; - -beforeAll(before); - -const project_id = "00000000-0000-4000-8000-000000000000"; -async function create() { - return await createOpenFiles(project_id, { noAutosave: true, noCache: true }); -} - -describe("create open file tracker and do some basic operations", () => { - let o1, o2; - let file1 = `${Math.random()}.txt`; - let file2 = `${Math.random()}.txt`; - - it("creates two open files trackers (tracking same project) and clear them", async () => { - o1 = await create(); - o2 = await create(); - // ensure caching disabled so our sync tests are real - expect(o1.getKv() === o2.getKv()).toBe(false); - o1.clear(); - await o1.save(); - expect(o1.hasUnsavedChanges()).toBe(false); - o2.clear(); - while (o2.hasUnsavedChanges() || o1.hasUnsavedChanges()) { - try { - // expected due to merge conflict and autosave being disabled. - await o2.save(); - } catch { - await delay(10); - } - } - }); - - it("confirm they are cleared", async () => { - expect(o1.getAll()).toEqual([]); - expect(o2.getAll()).toEqual([]); - }); - - it("touch file in one and observe change and timestamp getting assigned by server", async () => { - o1.touch(file1); - expect(o1.get(file1).time).toBeCloseTo(Date.now(), -3); - }); - - it("touches file in one and observes change by OTHER", async () => { - o1.touch(file2); - expect(o1.get(file2)?.path).toBe(file2); - expect(o2.get(file2)).toBe(undefined); - await o1.save(); - if (o2.get(file2) == null) { - await once(o2, "change", 250); - expect(o2.get(file2).path).toBe(file2); - expect(o2.get(file2).time == null).toBe(false); - } - }); - - it("get all in o2 sees both file1 and file2", async () => { - const v = o2.getAll(); - expect(v[0].path).toBe(file1); - expect(v[1].path).toBe(file2); - expect(v.length).toBe(2); - }); - - it("delete file1 and verify fact that it is deleted is sync'd", async () => { - o1.delete(file1); - expect(o1.get(file1)).toBe(undefined); - expect(o1.getAll().length).toBe(1); - await o1.save(); - - // verify file is gone in o2, at least after waiting (if necessary) - await wait({ - until: () => { - return o2.getAll().length == 1; - }, - }); - expect(o2.get(file1)).toBe(undefined); - // should be 1 due to file2 still being there: - expect(o2.getAll().length).toBe(1); - - // Also confirm file1 is gone in a newly opened one: - const o3 = await create(); - expect(o3.get(file1)).toBe(undefined); - // should be 1 due to file2 still being there, but not file1. - expect(o3.getAll().length).toBe(1); - o3.close(); - }); - - it("sets an error", async () => { - o2.setError(file2, Error("test error")); - expect(o2.get(file2).error.error).toBe("Error: test error"); - expect(typeof o2.get(file2).error.time == "number").toBe(true); - expect(Math.abs(Date.now() - o2.get(file2).error.time)).toBeLessThan(10000); - try { - // get a conflict due to above so resolve it... - await o2.save(); - } catch { - await o2.save(); - } - if (!o1.get(file2).error) { - await once(o1, "change", 250); - } - expect(o1.get(file2).error.error).toBe("Error: test error"); - }); - - it("clears the error", async () => { - o1.setError(file2); - expect(o1.get(file2).error).toBe(undefined); - await o1.save(); - if (o2.get(file2).error) { - await once(o2, "change", 250); - } - expect(o2.get(file2).error).toBe(undefined); - }); -}); - -afterAll(after); diff --git a/src/packages/conat/sync/open-files.ts b/src/packages/conat/sync/open-files.ts deleted file mode 100644 index b82afd8578..0000000000 --- a/src/packages/conat/sync/open-files.ts +++ /dev/null @@ -1,302 +0,0 @@ -/* -Keep track of open files. - -We use the "dko" distributed key:value store because of the potential of merge -conflicts, e.g,. one client changes the compute server id and another changes -whether a file is deleted. By using dko, only the field that changed is sync'd -out, so we get last-write-wins on the level of fields. - -WARNINGS: -An old version use dkv with merge conflict resolution, but with multiple clients -and the project, feedback loops or something happened and it would start getting -slow -- basically, merge conflicts could take a few seconds to resolve, which would -make opening a file start to be slow. Instead we use DKO data type, where fields -are treated separately atomically by the storage system. A *subtle issue* is -that when you set an object, this is NOT treated atomically. E.g., if you -set 2 fields in a set operation, then 2 distinct changes are emitted as the -two fields get set. - -DEVELOPMENT: - -Change to packages/backend, since packages/conat doesn't have a way to connect: - -~/cocalc/src/packages/backend$ node - -> z = await require('@cocalc/backend/conat/sync').openFiles({project_id:cc.current().project_id}) -> z.touch({path:'a.txt'}) -> z.get({path:'a.txt'}) -{ open: true, count: 1, time:2025-02-09T16:37:20.713Z } -> z.touch({path:'a.txt'}) -> z.get({path:'a.txt'}) -{ open: true, count: 2 } -> z.time({path:'a.txt'}) -2025-02-09T16:36:58.510Z -> z.touch({path:'foo/b.md',id:0}) -> z.getAll() -{ - 'a.txt': { open: true, count: 3 }, - 'foo/b.md': { open: true, count: 1 } - -Frontend Dev in browser: - -z = await cc.client.conat_client.openFiles({project_id:cc.current().project_id)) -z.getAll() -} -*/ - -import { type State } from "@cocalc/conat/types"; -import { dko, type DKO } from "@cocalc/conat/sync/dko"; -import { EventEmitter } from "events"; -import getTime, { getSkew } from "@cocalc/conat/time"; - -// info about interest in open files (and also what was explicitly deleted) older -// than this is automatically purged. -const MAX_AGE_MS = 1000 * 60 * 60 * 24; - -interface Deleted { - // what deleted state is - deleted: boolean; - // when deleted state set - time: number; -} - -interface Backend { - // who has it opened -- the compute_server_id (0 for project) - id: number; - // when they last reported having it opened - time: number; -} - -export interface KVEntry { - // a web browser has the file open at this point in time (in ms) - time?: number; - // if the file was removed from disk (and not immmediately written back), - // then deleted gets set to the time when this happened (in ms since epoch) - // and the file is closed on the backend. It won't be re-opened until - // either (1) the file is created on disk again, or (2) deleted is cleared. - // Note: the actual time here isn't really important -- what matter is the number - // is nonzero. It's just used for a display to the user. - // We store the deleted state *and* when this was set, so that in case - // of merge conflict we can do something sensible. - deleted?: Deleted; - - // if file is actively opened on a compute server/project, then it sets - // this entry. Right when it closes the file, it clears this. - // If it gets killed/broken and doesn't have a chance to clear it, then - // backend.time can be used to decide this isn't valid. - backend?: Backend; - - // optional information - doctype?; -} - -export interface Entry extends KVEntry { - // path to file relative to HOME - path: string; -} - -interface Options { - project_id: string; - noAutosave?: boolean; - noCache?: boolean; -} - -export async function createOpenFiles(opts: Options) { - const openFiles = new OpenFiles(opts); - await openFiles.init(); - return openFiles; -} - -export class OpenFiles extends EventEmitter { - private project_id: string; - private noCache?: boolean; - private noAutosave?: boolean; - private kv?: DKO; - public state: "disconnected" | "connected" | "closed" = "disconnected"; - - constructor({ project_id, noAutosave, noCache }: Options) { - super(); - if (!project_id) { - throw Error("project_id must be specified"); - } - this.project_id = project_id; - this.noAutosave = noAutosave; - this.noCache = noCache; - } - - private setState = (state: State) => { - this.state = state; - this.emit(state); - }; - - private initialized = false; - init = async () => { - if (this.initialized) { - throw Error("init can only be called once"); - } - this.initialized = true; - const d = await dko({ - name: "open-files", - project_id: this.project_id, - config: { - max_age: MAX_AGE_MS, - }, - noAutosave: this.noAutosave, - noCache: this.noCache, - noInventory: true, - }); - this.kv = d; - d.on("change", this.handleChange); - // ensure clock is synchronized - await getSkew(); - this.setState("connected"); - }; - - private handleChange = ({ key: path }) => { - const entry = this.get(path); - if (entry != null) { - // not deleted and timestamp is set: - this.emit("change", entry as Entry); - } - }; - - close = () => { - if (this.kv == null) { - return; - } - this.setState("closed"); - this.removeAllListeners(); - this.kv.removeListener("change", this.handleChange); - this.kv.close(); - delete this.kv; - // @ts-ignore - delete this.project_id; - }; - - private getKv = () => { - const { kv } = this; - if (kv == null) { - throw Error("closed"); - } - return kv; - }; - - private set = (path, entry: KVEntry) => { - this.getKv().set(path, entry); - }; - - // When a client has a file open, they should periodically - // touch it to indicate that it is open. - // updates timestamp and ensures open=true. - touch = (path: string, doctype?) => { - if (!path) { - throw Error("path must be specified"); - } - const kv = this.getKv(); - const cur = kv.get(path); - const time = getTime(); - if (doctype) { - this.set(path, { - ...cur, - time, - doctype, - }); - } else { - this.set(path, { - ...cur, - time, - }); - } - }; - - setError = (path: string, err?: any) => { - const kv = this.getKv(); - if (!err) { - const current = { ...kv.get(path) }; - delete current.error; - this.set(path, current); - } else { - const current = { ...kv.get(path) }; - current.error = { time: Date.now(), error: `${err}` }; - this.set(path, current); - } - }; - - setDeleted = (path: string) => { - const kv = this.getKv(); - this.set(path, { - ...kv.get(path), - deleted: { deleted: true, time: getTime() }, - }); - }; - - isDeleted = (path: string) => { - return !!this.getKv().get(path)?.deleted?.deleted; - }; - - setNotDeleted = (path: string) => { - const kv = this.getKv(); - this.set(path, { - ...kv.get(path), - deleted: { deleted: false, time: getTime() }, - }); - }; - - // set that id is the backend with the file open. - // This should be called by that backend periodically - // when it has the file opened. - setBackend = (path: string, id: number) => { - const kv = this.getKv(); - this.set(path, { - ...kv.get(path), - backend: { id, time: getTime() }, - }); - }; - - // get current backend that has file opened. - getBackend = (path: string): Backend | undefined => { - return this.getKv().get(path)?.backend; - }; - - // ONLY if backend for path is currently set to id, then clear - // the backend field. - setNotBackend = (path: string, id: number) => { - const kv = this.getKv(); - const cur = { ...kv.get(path) }; - if (cur?.backend?.id == id) { - delete cur.backend; - this.set(path, cur); - } - }; - - getAll = (): Entry[] => { - const x = this.getKv().getAll(); - return Object.keys(x).map((path) => { - return { ...x[path], path }; - }); - }; - - get = (path: string): Entry | undefined => { - const x = this.getKv().get(path); - if (x == null) { - return x; - } - return { ...x, path }; - }; - - delete = (path) => { - this.getKv().delete(path); - }; - - clear = () => { - this.getKv().clear(); - }; - - save = async () => { - await this.getKv().save(); - }; - - hasUnsavedChanges = () => { - return this.getKv().hasUnsavedChanges(); - }; -} diff --git a/src/packages/frontend/client/client.ts b/src/packages/frontend/client/client.ts index 08dec77e57..786f57ef0e 100644 --- a/src/packages/frontend/client/client.ts +++ b/src/packages/frontend/client/client.ts @@ -317,26 +317,6 @@ class Client extends EventEmitter implements WebappClient { public set_deleted(): void { throw Error("not implemented for frontend"); } - - touchOpenFile = async ({ - project_id, - path, - setNotDeleted, - doctype, - }: { - project_id: string; - path: string; - id?: number; - doctype?; - // if file is deleted, this explicitly undeletes it. - setNotDeleted?: boolean; - }) => { - const x = await this.conat_client.openFiles(project_id); - if (setNotDeleted) { - x.setNotDeleted(path); - } - x.touch(path, doctype); - }; } export const webapp_client = new Client(); diff --git a/src/packages/frontend/conat/client.ts b/src/packages/frontend/conat/client.ts index 516c138143..eb9c7ca613 100644 --- a/src/packages/frontend/conat/client.ts +++ b/src/packages/frontend/conat/client.ts @@ -11,7 +11,6 @@ import { parseQueryWithOptions } from "@cocalc/sync/table/util"; import { type HubApi, initHubApi } from "@cocalc/conat/hub/api"; import { type ProjectApi, initProjectApi } from "@cocalc/conat/project/api"; import { isValidUUID } from "@cocalc/util/misc"; -import { createOpenFiles, OpenFiles } from "@cocalc/conat/sync/open-files"; import { PubSub } from "@cocalc/conat/sync/pubsub"; import type { ChatOptions } from "@cocalc/util/types/llm"; import { dkv } from "@cocalc/conat/sync/dkv"; @@ -62,7 +61,6 @@ export class ConatClient extends EventEmitter { client: WebappClient; public hub: HubApi; public sessionId = randomId(); - private openFilesCache: { [project_id: string]: OpenFiles } = {}; private clientWithState: ClientWithState; private _conatClient: null | ReturnType; public numConnectionAttempts = 0; @@ -428,42 +426,6 @@ export class ConatClient extends EventEmitter { }); }; - openFiles = reuseInFlight(async (project_id: string) => { - if (this.openFilesCache[project_id] == null) { - const openFiles = await createOpenFiles({ - project_id, - }); - this.openFilesCache[project_id] = openFiles; - openFiles.on("closed", () => { - delete this.openFilesCache[project_id]; - }); - openFiles.on("change", (entry) => { - if (entry.deleted?.deleted) { - setDeleted({ - project_id, - path: entry.path, - deleted: entry.deleted.time, - }); - } else { - setNotDeleted({ project_id, path: entry.path }); - } - }); - const recentlyDeletedPaths: any = {}; - for (const { path, deleted } of openFiles.getAll()) { - if (deleted?.deleted) { - recentlyDeletedPaths[path] = deleted.time; - } - } - const store = redux.getProjectStore(project_id); - store.setState({ recentlyDeletedPaths }); - } - return this.openFilesCache[project_id]!; - }); - - closeOpenFiles = (project_id) => { - this.openFilesCache[project_id]?.close(); - }; - pubsub = async ({ project_id, path, @@ -520,22 +482,6 @@ export class ConatClient extends EventEmitter { refCacheInfo = () => refCacheInfo(); } -function setDeleted({ project_id, path, deleted }) { - if (!redux.hasProjectStore(project_id)) { - return; - } - const actions = redux.getProjectActions(project_id); - actions.setRecentlyDeleted(path, deleted); -} - -function setNotDeleted({ project_id, path }) { - if (!redux.hasProjectStore(project_id)) { - return; - } - const actions = redux.getProjectActions(project_id); - actions?.setRecentlyDeleted(path, 0); -} - async function waitForOnline(): Promise { if (navigator.onLine) return; await new Promise((resolve) => { diff --git a/src/packages/frontend/frame-editors/terminal-editor/conat-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/conat-terminal.ts index 58e0e1d51e..3734b56808 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/conat-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/conat-terminal.ts @@ -10,7 +10,6 @@ import { SIZE_TIMEOUT_MS, createBrowserClient, } from "@cocalc/conat/service/terminal"; -import { CONAT_OPEN_FILE_TOUCH_INTERVAL } from "@cocalc/util/conat"; import { until } from "@cocalc/util/async-utils"; type State = "disconnected" | "init" | "running" | "closed"; @@ -58,7 +57,6 @@ export class ConatTerminal extends EventEmitter { this.path = path; this.termPath = termPath; this.options = options; - this.touchLoop({ project_id, path: termPath }); this.sizeLoop(measureSize); this.api = createTerminalClient({ project_id, termPath }); this.createBrowserService(); @@ -143,25 +141,6 @@ export class ConatTerminal extends EventEmitter { } }; - touchLoop = async ({ project_id, path }) => { - while (this.state != ("closed" as State)) { - try { - // this marks the path as being of interest for editing and starts - // the service; it doesn't actually create a file on disk. - await webapp_client.touchOpenFile({ - project_id, - path, - }); - } catch (err) { - console.warn(err); - } - if (this.state == ("closed" as State)) { - break; - } - await delay(CONAT_OPEN_FILE_TOUCH_INTERVAL); - } - }; - sizeLoop = async (measureSize) => { while (this.state != ("closed" as State)) { measureSize(); diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index 177ed7c9c6..4a4b91d80f 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -382,8 +382,6 @@ export class ProjectActions extends Actions { this.remove_table(table); } - webapp_client.conat_client.closeOpenFiles(this.project_id); - const store = this.get_store(); store?.close_all_tables(); }; @@ -3539,14 +3537,6 @@ export class ProjectActions extends Actions { const store = this.get_store(); if (store == null) return; this.setRecentlyDeleted(path, 0); - (async () => { - try { - const o = await webapp_client.conat_client.openFiles(this.project_id); - o.setNotDeleted(path); - } catch (err) { - console.log("WARNING: issue undeleting file", err); - } - })(); }; private initProjectStatus = async () => { diff --git a/src/packages/project/client.ts b/src/packages/project/client.ts index ea631ddc47..9b1fa744e7 100644 --- a/src/packages/project/client.ts +++ b/src/packages/project/client.ts @@ -27,7 +27,6 @@ import { join } from "node:path"; import { FileSystemClient } from "@cocalc/sync-client/lib/client-fs"; import { execute_code, uuidsha1 } from "@cocalc/backend/misc_node"; import { CoCalcSocket } from "@cocalc/backend/tcp/enable-messaging-protocol"; -import type { SyncDoc } from "@cocalc/sync/editor/generic/sync-doc"; import type { ProjectClient as ProjectClientInterface } from "@cocalc/sync/editor/generic/types"; import { SyncString } from "@cocalc/sync/editor/string/sync"; import * as synctable2 from "@cocalc/sync/table"; @@ -54,7 +53,6 @@ import { type CreateConatServiceFunction, } from "@cocalc/conat/service"; import { connectToConat } from "./conat/connection"; -import { getSyncDoc } from "@cocalc/project/conat/open-files"; import { isDeleted } from "@cocalc/project/conat/listings"; const winston = getLogger("client"); @@ -520,15 +518,6 @@ export class Client extends EventEmitter implements ProjectClientInterface { }); }; - // WARNING: making two of the exact same sync_string or sync_db will definitely - // lead to corruption! - - // Get the synchronized doc with the given path. Returns undefined - // if currently no such sync-doc. - syncdoc = ({ path }: { path: string }): SyncDoc | undefined => { - return getSyncDoc(path); - }; - public path_access(opts: { path: string; mode: string; cb: CB }): void { // mode: sub-sequence of 'rwxf' -- see https://nodejs.org/api/fs.html#fs_class_fs_stats // cb(err); err = if any access fails; err=undefined if all access is OK diff --git a/src/packages/project/conat/api/index.ts b/src/packages/project/conat/api/index.ts index a523d594d0..ef45cb1eb8 100644 --- a/src/packages/project/conat/api/index.ts +++ b/src/packages/project/conat/api/index.ts @@ -53,7 +53,6 @@ Remember, if you don't set API_KEY, then the project MUST be running so that the import { type ProjectApi } from "@cocalc/conat/project/api"; import { connectToConat } from "@cocalc/project/conat/connection"; import { getSubject } from "../names"; -import { terminate as terminateOpenFiles } from "@cocalc/project/conat/open-files"; import { close as closeListings } from "@cocalc/project/conat/listings"; import { project_id } from "@cocalc/project/data"; import { close as closeFilesRead } from "@cocalc/project/conat/files/read"; @@ -102,11 +101,7 @@ async function handleMessage(api, subject, mesg) { // TODO: should be part of handleApiRequest below, but done differently because // one case halts this loop const { service } = request.args[0] ?? {}; - if (service == "open-files") { - terminateOpenFiles(); - await mesg.respond({ status: "terminated", service }); - return; - } else if (service == "listings") { + if (service == "listings") { closeListings(); await mesg.respond({ status: "terminated", service }); return; diff --git a/src/packages/project/conat/index.ts b/src/packages/project/conat/index.ts index a93de46d88..2c2d9121eb 100644 --- a/src/packages/project/conat/index.ts +++ b/src/packages/project/conat/index.ts @@ -9,7 +9,7 @@ Start the NATS servers: import "./connection"; import { getLogger } from "@cocalc/project/logger"; import { init as initAPI } from "./api"; -import { init as initOpenFiles } from "./open-files"; +// import { init as initOpenFiles } from "./open-files"; // TODO: initWebsocketApi is temporary import { init as initWebsocketApi } from "./browser-websocket-api"; import { init as initListings } from "./listings"; @@ -25,7 +25,7 @@ export default async function init() { logger.debug("starting Conat project services"); await initAPI(); await initJupyter(); - await initOpenFiles(); + // await initOpenFiles(); initWebsocketApi(); await initListings(); await initRead(); diff --git a/src/packages/project/conat/open-files.ts b/src/packages/project/conat/open-files.ts deleted file mode 100644 index 1b93b8c3e5..0000000000 --- a/src/packages/project/conat/open-files.ts +++ /dev/null @@ -1,516 +0,0 @@ -/* -Handle opening files in a project to save/load from disk and also enable compute capabilities. - -DEVELOPMENT: - -0. From the browser with the project opened, terminate the open-files api service: - - - await cc.client.conat_client.projectApi(cc.current()).system.terminate({service:'open-files'}) - - - -Set env variables as in a project (see api/index.ts ), then in nodejs: - -DEBUG_CONSOLE=yes DEBUG=cocalc:debug:project:conat:* node - - x = await require("@cocalc/project/conat/open-files").init(); Object.keys(x) - - -[ 'openFiles', 'openDocs', 'terminate', 'computeServers', 'cc' ] - -> x.openFiles.getAll(); - -> Object.keys(x.openDocs) - -> s = x.openDocs['z4.tasks'] -// now you can directly work with the syncdoc for a given file, -// but from the perspective of the project, not the browser! -// -// - -OR: - - echo "require('@cocalc/project/conat/open-files').init(); require('@cocalc/project/bug-counter').init()" | node - -COMPUTE SERVER: - -To simulate a compute server, do exactly as above, but also set the environment -variable COMPUTE_SERVER_ID to the *global* (not project specific) id of the compute -server: - - COMPUTE_SERVER_ID=84 node - -In this case, you aso don't need to use the terminate command if the compute -server isn't actually running. To terminate a compute server open files service though: - - (TODO) - - -EDITOR ACTIONS: - -Stop the open-files server and define x as above in a terminal. You can -then get the actions or store in a nodejs terminal for a particular document -as follows: - -project_id = '00847397-d6a8-4cb0-96a8-6ef64ac3e6cf'; path = '2025-03-21-100921.ipynb'; -redux = require("@cocalc/jupyter/redux/app").redux; a = redux.getEditorActions(project_id, path); s = redux.getEditorStore(project_id, path); 0; - - -IN A LIVE RUNNING PROJECT IN KUCALC: - -Ssh in to the project itself. You can use a terminal because that very terminal will be broken by -doing this! Then: - -/cocalc/github/src/packages/project$ . /cocalc/nvm/nvm.sh -/cocalc/github/src/packages/project$ COCALC_PROJECT_ID=... COCALC_SECRET_TOKEN="/secrets/secret-token/token" CONAT_SERVER=hub-conat node # not sure about CONAT_SERVER -Welcome to Node.js v20.19.0. -Type ".help" for more information. -> x = await require("@cocalc/project/conat/open-files").init(); Object.keys(x) -[ 'openFiles', 'openDocs', 'terminate', 'computeServers' ] -> - - -*/ - -import { - openFiles as createOpenFiles, - type OpenFiles, - type OpenFileEntry, -} from "@cocalc/project/conat/sync"; -import { CONAT_OPEN_FILE_TOUCH_INTERVAL } from "@cocalc/util/conat"; -import { compute_server_id, project_id } from "@cocalc/project/data"; -import type { SyncDoc } from "@cocalc/sync/editor/generic/sync-doc"; -import { getClient } from "@cocalc/project/client"; -import { SyncString } from "@cocalc/sync/editor/string/sync"; -import { SyncDB } from "@cocalc/sync/editor/db/sync"; -import getLogger from "@cocalc/backend/logger"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { delay } from "awaiting"; -import { initJupyterRedux, removeJupyterRedux } from "@cocalc/jupyter/kernel"; -import { filename_extension, original_path } from "@cocalc/util/misc"; -import { type ConatService } from "@cocalc/conat/service/service"; -import { exists } from "@cocalc/backend/misc/async-utils-node"; -import { map as awaitMap } from "awaiting"; -import { unlink } from "fs/promises"; -import { join } from "path"; -import { - computeServerManager, - ComputeServerManager, -} from "@cocalc/conat/compute/manager"; -import { JUPYTER_SYNCDB_EXTENSIONS } from "@cocalc/util/jupyter/names"; -import { connectToConat } from "@cocalc/project/conat/connection"; - -// ensure conat connection stuff is initialized -import "@cocalc/project/conat/env"; -import { chdir } from "node:process"; - -const logger = getLogger("project:conat:open-files"); - -// we check all files we are currently managing this frequently to -// see if they exist on the filesystem: -const FILE_DELETION_CHECK_INTERVAL = 5000; - -// once we determine that a file does not exist for some reason, we -// wait this long and check *again* just to be sure. If it is still missing, -// then we close the file in memory and set the file as deleted in the -// shared openfile state. -const FILE_DELETION_GRACE_PERIOD = 2000; - -// We NEVER check a file for deletion for this long after first opening it. -// This is VERY important, since some documents, e.g., jupyter notebooks, -// can take a while to get created on disk the first time. -const FILE_DELETION_INITIAL_DELAY = 15000; - -let openFiles: OpenFiles | null = null; -const openDocs: { [path: string]: SyncDoc | ConatService } = {}; -let computeServers: ComputeServerManager | null = null; -const openTimes: { [path: string]: number } = {}; - -export function getSyncDoc(path: string): SyncDoc | undefined { - const doc = openDocs[path]; - if (doc instanceof SyncString || doc instanceof SyncDB) { - return doc; - } - return undefined; -} - -export async function init() { - logger.debug("init"); - - if (process.env.HOME) { - chdir(process.env.HOME); - } - - openFiles = await createOpenFiles(); - - computeServers = computeServerManager({ project_id }); - await computeServers.waitUntilReady(); - computeServers.on("change", async ({ path, id }) => { - if (openFiles == null) { - return; - } - const entry = openFiles?.get(path); - if (entry != null) { - await handleChange({ ...entry, id }); - } else { - await closeDoc(path); - } - }); - - // initialize - for (const entry of openFiles.getAll()) { - handleChange(entry); - } - - // start loop to watch for and close files that aren't touched frequently: - closeIgnoredFilesLoop(); - - // periodically update timestamp on backend for files we have open - touchOpenFilesLoop(); - // watch if any file that is currently opened on this host gets deleted, - // and if so, mark it as such, and set it to closed. - watchForFileDeletionLoop(); - - // handle changes - openFiles.on("change", (entry) => { - // we ONLY actually try to open the file here if there - // is a doctype set. When it is first being created, - // the doctype won't be the first field set, and we don't - // want to launch this until it is set. - if (entry.doctype) { - handleChange(entry); - } - }); - - // useful for development - return { - openFiles, - openDocs, - terminate, - computeServers, - cc: connectToConat(), - }; -} - -export function terminate() { - logger.debug("terminating open-files service"); - for (const path in openDocs) { - closeDoc(path); - } - openFiles?.close(); - openFiles = null; - - computeServers?.close(); - computeServers = null; -} - -function getCutoff(): number { - return Date.now() - 2.5 * CONAT_OPEN_FILE_TOUCH_INTERVAL; -} - -function computeServerId(path: string): number { - return computeServers?.get(path) ?? 0; -} - -function hasBackendState(path) { - return ( - path.endsWith("." + JUPYTER_SYNCDB_EXTENSIONS) || path.endsWith(".sagews") - ); -} - -async function handleChange({ - path, - time, - deleted, - backend, - doctype, - id, -}: OpenFileEntry & { id?: number }) { - // DEPRECATED! - return; - if (!hasBackendState(path)) { - return; - } - try { - if (id == null) { - id = computeServerId(path); - } - logger.debug("handleChange", { path, time, deleted, backend, doctype, id }); - const syncDoc = openDocs[path]; - const isOpenHere = syncDoc != null; - - if (id != compute_server_id) { - if (backend?.id == compute_server_id) { - // we are definitely not the backend right now. - openFiles?.setNotBackend(path, compute_server_id); - } - // only thing we should do is close it if it is open. - if (isOpenHere) { - await closeDoc(path); - } - return; - } - - if (deleted?.deleted) { - if (await exists(path)) { - // it's back - openFiles?.setNotDeleted(path); - } else { - if (isOpenHere) { - await closeDoc(path); - } - return; - } - } - - // @ts-ignore - if (time != null && time >= getCutoff()) { - if (!isOpenHere) { - logger.debug("handleChange: opening", { path }); - // users actively care about this file being opened HERE, but it isn't - await openDoc(path); - } - return; - } - } catch (err) { - console.trace(err); - logger.debug(`handleChange: WARNING - error opening ${path} -- ${err}`); - } -} - -function supportAutoclose(path: string): boolean { - // this feels way too "hard coded"; alternatively, maybe we make the kernel or whatever - // actually update the interest? or something else... - if ( - path.endsWith("." + JUPYTER_SYNCDB_EXTENSIONS) || - path.endsWith(".sagews") || - path.endsWith(".term") - ) { - return false; - } - return true; -} - -async function closeIgnoredFilesLoop() { - while (openFiles?.state == "connected") { - await delay(CONAT_OPEN_FILE_TOUCH_INTERVAL); - if (openFiles?.state != "connected") { - return; - } - const paths = Object.keys(openDocs); - if (paths.length == 0) { - logger.debug("closeIgnoredFiles: no paths currently open"); - continue; - } - logger.debug( - "closeIgnoredFiles: checking", - paths.length, - "currently open paths...", - ); - const cutoff = getCutoff(); - for (const entry of openFiles.getAll()) { - if ( - entry != null && - entry.time != null && - openDocs[entry.path] != null && - entry.time <= cutoff && - supportAutoclose(entry.path) - ) { - logger.debug("closeIgnoredFiles: closing due to inactivity", entry); - closeDoc(entry.path); - } - } - } -} - -async function touchOpenFilesLoop() { - while (openFiles?.state == "connected" && openDocs != null) { - for (const path in openDocs) { - openFiles.setBackend(path, compute_server_id); - } - await delay(CONAT_OPEN_FILE_TOUCH_INTERVAL); - } -} - -async function checkForFileDeletion(path: string) { - if (openFiles == null) { - return; - } - if (Date.now() - (openTimes[path] ?? 0) <= FILE_DELETION_INITIAL_DELAY) { - return; - } - const id = computeServerId(path); - if (id != compute_server_id) { - // not our concern - return; - } - - if (path.endsWith(".term")) { - // term files are exempt -- we don't save data in them and often - // don't actually make the hidden ones for each frame in the - // filesystem at all. - return; - } - const entry = openFiles.get(path); - if (entry == null) { - return; - } - if (entry.deleted?.deleted) { - // already set as deleted -- shouldn't still be opened - await closeDoc(entry.path); - } else { - if (!process.env.HOME) { - // too dangerous - return; - } - const fullPath = join(process.env.HOME, entry.path); - // if file doesn't exist and still doesn't exist in a while, - // mark deleted, which also causes a close. - if (await exists(fullPath)) { - return; - } - // still doesn't exist? - // We must give things a reasonable amount of time, e.g., otherwise - // creating a file (e.g., jupyter notebook) might take too long and - // we randomly think it is deleted before we even make it! - await delay(FILE_DELETION_GRACE_PERIOD); - if (await exists(fullPath)) { - return; - } - // still doesn't exist - if (openFiles != null) { - logger.debug("checkForFileDeletion: marking as deleted -- ", entry); - openFiles.setDeleted(entry.path); - await closeDoc(fullPath); - // closing a file may cause it to try to save to disk the last version, - // so we delete it if that happens. - // TODO: add an option to close everywhere to not do this, and/or make - // it not save on close if the file doesn't exist. - try { - if (await exists(fullPath)) { - await unlink(fullPath); - } - } catch {} - } - } -} - -async function watchForFileDeletionLoop() { - while (openFiles != null && openFiles.state == "connected") { - await delay(FILE_DELETION_CHECK_INTERVAL); - if (openFiles?.state != "connected") { - return; - } - const paths = Object.keys(openDocs); - if (paths.length == 0) { - // logger.debug("watchForFileDeletionLoop: no paths currently open"); - continue; - } - // logger.debug( - // "watchForFileDeletionLoop: checking", - // paths.length, - // "currently open paths to see if any were deleted", - // ); - await awaitMap(paths, 20, checkForFileDeletion); - } -} - -const closeDoc = reuseInFlight(async (path: string) => { - logger.debug("close", { path }); - try { - const doc = openDocs[path]; - if (doc == null) { - return; - } - delete openDocs[path]; - delete openTimes[path]; - try { - await doc.close(); - } catch (err) { - logger.debug(`WARNING -- issue closing doc -- ${err}`); - openFiles?.setError(path, err); - } - } finally { - if (openDocs[path] == null) { - openFiles?.setNotBackend(path, compute_server_id); - } - } -}); - -const openDoc = reuseInFlight(async (path: string) => { - logger.debug("openDoc", { path }); - try { - const doc = openDocs[path]; - if (doc != null) { - return; - } - openTimes[path] = Date.now(); - - if (path.endsWith(".term")) { - // terminals are handled directly by the project api -- also since - // doctype probably not set for them, they won't end up here. - // (this could change though, e.g., we might use doctype to - // set the terminal command). - return; - } - - const client = getClient(); - let doctype: any = openFiles?.get(path)?.doctype; - logger.debug("openDoc: open files table knows ", openFiles?.get(path), { - path, - }); - if (doctype == null) { - logger.debug("openDoc: doctype must be set but isn't, so bailing", { - path, - }); - } else { - logger.debug("openDoc: got doctype from openFiles table", { - path, - doctype, - }); - } - - let syncdoc; - if (doctype.type == "string") { - syncdoc = new SyncString({ - ...doctype.opts, - project_id, - path, - client, - }); - } else { - syncdoc = new SyncDB({ - ...doctype.opts, - project_id, - path, - client, - }); - } - openDocs[path] = syncdoc; - - syncdoc.on("error", (err) => { - closeDoc(path); - openFiles?.setError(path, err); - logger.debug(`syncdoc error -- ${err}`, path); - }); - - // Extra backend support in some cases, e.g., Jupyter, Sage, etc. - const ext = filename_extension(path); - switch (ext) { - case JUPYTER_SYNCDB_EXTENSIONS: - logger.debug("initializing Jupyter backend for ", path); - await initJupyterRedux(syncdoc, client); - const path1 = original_path(syncdoc.get_path()); - syncdoc.on("closed", async () => { - logger.debug("removing Jupyter backend for ", path1); - await removeJupyterRedux(path1, project_id); - }); - break; - } - } finally { - if (openDocs[path] != null) { - openFiles?.setBackend(path, compute_server_id); - } - } -}); diff --git a/src/packages/project/conat/sync.ts b/src/packages/project/conat/sync.ts index ca2c343501..b705b4b3ee 100644 --- a/src/packages/project/conat/sync.ts +++ b/src/packages/project/conat/sync.ts @@ -10,11 +10,6 @@ import { } from "@cocalc/conat/sync/dkv"; import { dko as createDKO, type DKO } from "@cocalc/conat/sync/dko"; import { project_id } from "@cocalc/project/data"; -import { - createOpenFiles, - type OpenFiles, - Entry as OpenFileEntry, -} from "@cocalc/conat/sync/open-files"; import { inventory as createInventory, type Inventory, @@ -26,7 +21,7 @@ import { type AStream, } from "@cocalc/conat/sync/astream"; -export type { DStream, DKV, OpenFiles, OpenFileEntry }; +export type { DStream, DKV }; export async function dstream( opts: DStreamOptions, @@ -50,10 +45,6 @@ export async function dko(opts: DKVOptions): Promise> { return await createDKO({ project_id, ...opts }); } -export async function openFiles(): Promise { - return await createOpenFiles({ project_id }); -} - export async function inventory(): Promise { return await createInventory({ project_id }); } diff --git a/src/packages/sync/editor/generic/types.ts b/src/packages/sync/editor/generic/types.ts index 9222636e92..f840cec3a1 100644 --- a/src/packages/sync/editor/generic/types.ts +++ b/src/packages/sync/editor/generic/types.ts @@ -169,12 +169,6 @@ export interface Client extends ProjectClient { sage_session: (opts: { path: string }) => any; - touchOpenFile?: (opts: { - project_id: string; - path: string; - doctype?; - }) => Promise; - touch_project?: (path: string) => void; } From 192627a53ac6df41f1195bab4806c27169aba9a8 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 31 Jul 2025 05:09:12 +0000 Subject: [PATCH 147/798] jupyter: when evaluating a cell don't have the input get reset as new output appears --- src/packages/conat/project/jupyter/run-code.ts | 5 +++++ src/packages/frontend/jupyter/browser-actions.ts | 13 +++++++++---- src/packages/jupyter/control.ts | 8 +++++++- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/packages/conat/project/jupyter/run-code.ts b/src/packages/conat/project/jupyter/run-code.ts index 6efe51fbdf..04c7eeb53f 100644 --- a/src/packages/conat/project/jupyter/run-code.ts +++ b/src/packages/conat/project/jupyter/run-code.ts @@ -30,6 +30,11 @@ function getSubject({ interface InputCell { id: string; input: string; + output?: { [n: string]: OutputMessage } | null; + state?: "done" | "busy" | "run"; + exec_count?: number | null; + start?: number | null; + end?: number | null; } export interface OutputMessage { diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index f325724127..34e2626133 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -1469,11 +1469,16 @@ export class JupyterActions extends JupyterActions0 { getOutputHandler = (cell) => { const handler = new OutputHandler({ cell }); + + // save first time, so that other clients know this cell is running. let first = true; const f = throttle( () => { - // save first so that other clients know this cell is running. - this._set(cell, first); + // we ONLY set certain fields; e.g., setting the input would be + // extremely annoying since the user can edit the input while the + // cell is running. + const { id, state, output, start, end, exec_count } = cell; + this._set({ id, state, output, start, end, exec_count }, first); first = false; }, 1000 / OUTPUT_FPS, @@ -1575,8 +1580,8 @@ export class JupyterActions extends JupyterActions0 { cell.output[n] = null; } // time last evaluation took - cell.last = cell.start && cell.end ? cell.end - cell.start : null; - this._set(cell, false); + const last = cell.start && cell.end ? cell.end - cell.start : null; + this._set({ id: cell.id, last, output: cell.output }, false); } cells.push(cell); } diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts index 457efe81ec..58cf958bfa 100644 --- a/src/packages/jupyter/control.ts +++ b/src/packages/jupyter/control.ts @@ -122,7 +122,13 @@ class MulticellOutputHandler { this.handler?.done(); this.handler = new OutputHandler({ cell }); const f = throttle( - () => this.actions._set({ ...cell, type: "cell" }, true), + () => { + const { id, state, output, start, end, exec_count } = cell; + this.actions._set( + { type:"cell", id, state, output, start, end, exec_count }, + true, + ); + }, 1000 / BACKEND_OUTPUT_FPS, { leading: true, From 22c35e3873606af9b6b14fc96b538f445be5e3c3 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 31 Jul 2025 20:12:11 +0000 Subject: [PATCH 148/798] surprise subtle improvement to EventIterator --- .../test/project/jupyter/run-code.test.ts | 27 +++++++++++++++++-- .../conat/project/jupyter/run-code.ts | 11 +++++--- .../frontend/jupyter/browser-actions.ts | 9 ++++--- src/packages/util/event-iterator.ts | 16 ++++++++++- 4 files changed, 53 insertions(+), 10 deletions(-) diff --git a/src/packages/backend/conat/test/project/jupyter/run-code.test.ts b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts index fdb960d326..9cc2535124 100644 --- a/src/packages/backend/conat/test/project/jupyter/run-code.test.ts +++ b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts @@ -6,13 +6,18 @@ pnpm test `pwd`/run-code.test.ts */ -import { before, after, connect, wait } from "@cocalc/backend/conat/test/setup"; +import { + before, + after, + connect, + delay, + wait, +} from "@cocalc/backend/conat/test/setup"; import { jupyterClient, jupyterServer, } from "@cocalc/conat/project/jupyter/run-code"; import { uuid } from "@cocalc/util/misc"; -import { delay } from "awaiting"; // it's really 100+, but tests fails if less than this. const MIN_EVALS_PER_SECOND = 10; @@ -54,6 +59,24 @@ describe("create very simple mocked jupyter runner and test evaluating code", () expect(v).toEqual([[{ path, id: "0" }], [{ cells, id: "0" }]]); }); + it("start iterating over the output after waiting", async () => { + // this is the same as the previous test, except we insert a + // delay from when we create the iterator, and when we start + // reading values out of it. This is important to test, because + // it was broken in my first implementation, and is a common mistake + // when implementing async iterators. + client.verbose = true; + const iter = await client.run(cells); + iter.verbose = true; + const v: any[] = []; + await delay(500); + for await (const output of iter) { + v.push(output); + } + client.verbose = false; + expect(v).toEqual([[{ path, id: "0" }], [{ cells, id: "0" }]]); + }); + const count = 100; it(`run ${count} evaluations to ensure that the speed is reasonable (and also everything is kept properly ordered, etc.)`, async () => { const start = Date.now(); diff --git a/src/packages/conat/project/jupyter/run-code.ts b/src/packages/conat/project/jupyter/run-code.ts index 04c7eeb53f..483d52d0c2 100644 --- a/src/packages/conat/project/jupyter/run-code.ts +++ b/src/packages/conat/project/jupyter/run-code.ts @@ -27,14 +27,15 @@ function getSubject({ return `jupyter.project-${project_id}.${compute_server_id}`; } -interface InputCell { +export interface InputCell { id: string; input: string; - output?: { [n: string]: OutputMessage } | null; + output?: { [n: string]: OutputMessage | null } | null; state?: "done" | "busy" | "run"; exec_count?: number | null; start?: number | null; end?: number | null; + cell_type?: "code"; } export interface OutputMessage { @@ -189,7 +190,9 @@ class JupyterClient { run = async (cells: InputCell[], opts: { noHalt?: boolean } = {}) => { if (this.iter) { - // one evaluation at a time. + // one evaluation at a time -- starting a new one ends the previous one. + // Each client browser has a separate instance of JupyterClient, so + // a properly implemented frontend client would never hit this. this.iter.end(); delete this.iter; } @@ -201,7 +204,7 @@ class JupyterClient { } if (args[0] == null) { this.iter?.end(); - return null; + return; } else { return args[0]; } diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 34e2626133..8092f5e326 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -58,7 +58,10 @@ import getKernelSpec from "@cocalc/frontend/jupyter/kernelspecs"; import { get as getUsageInfo } from "@cocalc/conat/project/usage-info"; import { delay } from "awaiting"; import { until } from "@cocalc/util/async-utils"; -import { jupyterClient } from "@cocalc/conat/project/jupyter/run-code"; +import { + jupyterClient, + type InputCell, +} from "@cocalc/conat/project/jupyter/run-code"; import { OutputHandler } from "@cocalc/jupyter/execute/output-handler"; import { throttle } from "lodash"; @@ -1556,11 +1559,11 @@ export class JupyterActions extends JupyterActions0 { if (client == null) { throw Error("bug"); } - const cells: any[] = []; + const cells: InputCell[] = []; const kernel = this.store.get("kernel"); for (const id of ids) { - const cell = this.store.getIn(["cells", id])?.toJS(); + const cell = this.store.getIn(["cells", id])?.toJS() as InputCell; if ((cell?.cell_type ?? "code") != "code") { // code is the default type continue; diff --git a/src/packages/util/event-iterator.ts b/src/packages/util/event-iterator.ts index 8dcca43803..2d9ec62210 100644 --- a/src/packages/util/event-iterator.ts +++ b/src/packages/util/event-iterator.ts @@ -169,7 +169,6 @@ export class EventIterator if (this.#ended) return; this.resolveNext?.(); this.#ended = true; - this.#queue = []; this.emitter.off(this.event, this.#push); const maxListeners = this.emitter.getMaxListeners(); @@ -189,6 +188,9 @@ export class EventIterator * The next value that's received from the EventEmitter. */ public async next(): Promise> { + // if (this.verbose) { + // console.log("next", this.#queue); + // } if (this.err) { const err = this.err; delete this.err; @@ -279,11 +281,23 @@ export class EventIterator * Pushes a value into the queue. */ protected push(...args): void { + // if (this.verbose) { + // console.log("push", args, this.#queue); + // } if (this.err) { return; } try { const value = this.map(args); + if (this.#ended) { + // the this.map... call could have decided to end + // the iterator, by calling this.end() instead of returning a value. + if (value !== undefined) { + // not undefined so at least give the user the opportunity to get this final value. + this.#queue.push(value); + } + return; + } this.#queue.push(value); while (this.#queue.length > this.#maxQueue && this.#queue.length > 0) { if (this.#overflow == "throw") { From c22f8d44b7d4f62c0ab12a3818a49996f9a42bd8 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 31 Jul 2025 20:54:56 +0000 Subject: [PATCH 149/798] jupyter run -- handle tcp buffer issue --- .../backend/conat/test/socket/basic.test.ts | 2 +- .../conat/project/jupyter/run-code.ts | 19 +++++++++++++++++-- src/packages/conat/socket/client.ts | 4 ++-- src/packages/conat/socket/server-socket.ts | 4 ++-- src/packages/conat/socket/tcp.ts | 2 +- src/packages/conat/socket/util.ts | 5 +++-- .../jupyter/output-messages/message.tsx | 7 ++++--- 7 files changed, 30 insertions(+), 13 deletions(-) diff --git a/src/packages/backend/conat/test/socket/basic.test.ts b/src/packages/backend/conat/test/socket/basic.test.ts index 1ae6e364b2..36e4467b2a 100644 --- a/src/packages/backend/conat/test/socket/basic.test.ts +++ b/src/packages/backend/conat/test/socket/basic.test.ts @@ -168,7 +168,7 @@ describe("create a client first and write more messages than the queue size resu }); it("wait for client to drain; then we can now send another message without an error", async () => { - await client.waitUntilDrain(); + await client.drain(); client.write("foo"); }); diff --git a/src/packages/conat/project/jupyter/run-code.ts b/src/packages/conat/project/jupyter/run-code.ts index 483d52d0c2..866f0c9aab 100644 --- a/src/packages/conat/project/jupyter/run-code.ts +++ b/src/packages/conat/project/jupyter/run-code.ts @@ -120,7 +120,11 @@ export function jupyterServer({ //console.log(err); logger.debug("server: failed response -- ", err); if (socket.state != "closed") { - socket.write(null, { headers: { error: `${err}` } }); + try { + socket.write(null, { headers: { error: `${err}` } }); + } catch { + // an error trying to report an error shouldn't crash everything + } } } }); @@ -146,6 +150,7 @@ async function handleRequest({ let handler: OutputHandler | null = null; for await (const mesg of runner) { if (socket.state == "closed") { + // client socket has closed -- the backend server must take over! if (handler == null) { logger.debug("socket closed -- server must handle output"); if (outputHandler == null) { @@ -163,7 +168,17 @@ async function handleRequest({ handler.process(mesg); } else { output.push(mesg); - socket.write([mesg]); + try { + socket.write([mesg]); + } catch (err) { + if (err.code == "ENOBUFS") { + // wait for the over-filled socket to finish writing out data. + await socket.drain(); + socket.write([mesg]); + } else { + throw err; + } + } } } handler?.done(); diff --git a/src/packages/conat/socket/client.ts b/src/packages/conat/socket/client.ts index db8598a0e2..caa1a45fb9 100644 --- a/src/packages/conat/socket/client.ts +++ b/src/packages/conat/socket/client.ts @@ -91,8 +91,8 @@ export class ConatSocketClient extends ConatSocketBase { }); } - waitUntilDrain = async () => { - await this.tcp?.send.waitUntilDrain(); + drain = async () => { + await this.tcp?.send.drain(); }; private sendCommandToServer = async ( diff --git a/src/packages/conat/socket/server-socket.ts b/src/packages/conat/socket/server-socket.ts index 531e151949..0b678b05db 100644 --- a/src/packages/conat/socket/server-socket.ts +++ b/src/packages/conat/socket/server-socket.ts @@ -238,7 +238,7 @@ export class ServerSocket extends EventEmitter { } }); - waitUntilDrain = async () => { - await this.tcp?.send.waitUntilDrain(); + drain = async () => { + await this.tcp?.send.drain(); }; } diff --git a/src/packages/conat/socket/tcp.ts b/src/packages/conat/socket/tcp.ts index 7f0e2b4994..da55e76a6a 100644 --- a/src/packages/conat/socket/tcp.ts +++ b/src/packages/conat/socket/tcp.ts @@ -275,7 +275,7 @@ export class Sender extends EventEmitter { } }; - waitUntilDrain = reuseInFlight(async () => { + drain = reuseInFlight(async () => { if (this.unsent == 0) { return; } diff --git a/src/packages/conat/socket/util.ts b/src/packages/conat/socket/util.ts index 9e81f08439..2baef9fbea 100644 --- a/src/packages/conat/socket/util.ts +++ b/src/packages/conat/socket/util.ts @@ -20,8 +20,9 @@ export const PING_PONG_INTERVAL = 90000; // NOTE: in nodejs the default for exactly this is "infinite=use up all RAM", so // maybe we should make this even larger (?). // Also note that this is just the *number* of messages, and a message can have -// any size. -export const DEFAULT_MAX_QUEUE_SIZE = 1000; +// any size. But determining message size is very difficult without serializing the +// message, which costs. +export const DEFAULT_MAX_QUEUE_SIZE = 10_000; export let DEFAULT_COMMAND_TIMEOUT = 10_000; export let DEFAULT_KEEP_ALIVE = 25_000; diff --git a/src/packages/frontend/jupyter/output-messages/message.tsx b/src/packages/frontend/jupyter/output-messages/message.tsx index 69596b4623..27d59f0081 100644 --- a/src/packages/frontend/jupyter/output-messages/message.tsx +++ b/src/packages/frontend/jupyter/output-messages/message.tsx @@ -10,7 +10,6 @@ Handling of output messages. import Anser from "anser"; import type { Map } from "immutable"; import React from "react"; - import type { JupyterActions } from "@cocalc/jupyter/redux/actions"; import { LLMTools } from "@cocalc/jupyter/types"; import { Input } from "./input"; @@ -130,8 +129,10 @@ export const CellOutputMessages: React.FC = React.memo( const mesg = obj[n]; if (mesg != null) { if (mesg.get("traceback")) { - hasError = true; - traceback += mesg.get("traceback").join("\n") + "\n"; + const t = mesg.get("traceback").join("\n"); + // if user clicks "Stop" there is a traceback, but it's not an error to fix with AI. + hasError = !t.includes("KeyboardInterrupt"); + traceback += t + "\n"; } if (scrolled && !hasIframes && mesg.getIn(["data", "iframe"])) { hasIframes = true; From 14fc7f8f0a66f9f6ae7380251f906771f12117b4 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 31 Jul 2025 22:00:51 +0000 Subject: [PATCH 150/798] jupyter run -- buffer the jupyter output messages --- .../test/project/jupyter/run-code.test.ts | 30 ++++--- .../conat/project/jupyter/run-code.ts | 82 ++++++++++++------- src/packages/conat/socket/util.ts | 4 +- .../project/conat/terminal/session.ts | 6 +- src/packages/util/throttle.test.ts | 74 +++++++++++++++++ src/packages/util/throttle.ts | 32 ++++++-- 6 files changed, 176 insertions(+), 52 deletions(-) create mode 100644 src/packages/util/throttle.test.ts diff --git a/src/packages/backend/conat/test/project/jupyter/run-code.test.ts b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts index 9cc2535124..95dcb1400e 100644 --- a/src/packages/backend/conat/test/project/jupyter/run-code.test.ts +++ b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts @@ -54,9 +54,12 @@ describe("create very simple mocked jupyter runner and test evaluating code", () const iter = await client.run(cells); const v: any[] = []; for await (const output of iter) { - v.push(output); + v.push(...output); } - expect(v).toEqual([[{ path, id: "0" }], [{ cells, id: "0" }]]); + expect(v).toEqual([ + { path, id: "0" }, + { cells, id: "0" }, + ]); }); it("start iterating over the output after waiting", async () => { @@ -67,14 +70,15 @@ describe("create very simple mocked jupyter runner and test evaluating code", () // when implementing async iterators. client.verbose = true; const iter = await client.run(cells); - iter.verbose = true; const v: any[] = []; await delay(500); for await (const output of iter) { - v.push(output); + v.push(...output); } - client.verbose = false; - expect(v).toEqual([[{ path, id: "0" }], [{ cells, id: "0" }]]); + expect(v).toEqual([ + { path, id: "0" }, + { cells, id: "0" }, + ]); }); const count = 100; @@ -84,9 +88,12 @@ describe("create very simple mocked jupyter runner and test evaluating code", () const v: any[] = []; const cells = [{ id: `${i}`, input: `${i} + ${i}` }]; for await (const output of await client.run(cells)) { - v.push(output); + v.push(...output); } - expect(v).toEqual([[{ path, id: "0" }], [{ cells, id: "0" }]]); + expect(v).toEqual([ + { path, id: "0" }, + { cells, id: "0" }, + ]); } const evalsPerSecond = Math.floor((1000 * count) / (Date.now() - start)); if (process.env.BENCH) { @@ -146,9 +153,12 @@ describe("create simple mocked jupyter runner that does actually eval an express const iter = await client.run(cells); const v: any[] = []; for await (const output of iter) { - v.push(output); + v.push(...output); } - expect(v).toEqual([[{ id: "a", output: 5 }], [{ id: "b", output: 243 }]]); + expect(v).toEqual([ + { id: "a", output: 5 }, + { id: "b", output: 243 }, + ]); }); it("run code that FAILS and see error is visible to client properly", async () => { diff --git a/src/packages/conat/project/jupyter/run-code.ts b/src/packages/conat/project/jupyter/run-code.ts index 866f0c9aab..0417350eef 100644 --- a/src/packages/conat/project/jupyter/run-code.ts +++ b/src/packages/conat/project/jupyter/run-code.ts @@ -14,7 +14,10 @@ import { } from "@cocalc/conat/socket"; import { EventIterator } from "@cocalc/util/event-iterator"; import { getLogger } from "@cocalc/conat/client"; - +import { Throttle } from "@cocalc/util/throttle"; +const MAX_MSGS_PER_SECOND = parseInt( + process.env.COCALC_JUPYTER_MAX_MSGS_PER_SECOND ?? "20", +); const logger = getLogger("conat:project:jupyter:run-code"); function getSubject({ @@ -147,42 +150,59 @@ async function handleRequest({ }) { const runner = await jupyterRun({ path, cells, noHalt }); const output: OutputMessage[] = []; - let handler: OutputHandler | null = null; - for await (const mesg of runner) { - if (socket.state == "closed") { - // client socket has closed -- the backend server must take over! - if (handler == null) { - logger.debug("socket closed -- server must handle output"); - if (outputHandler == null) { - throw Error("no output handler available"); - } - handler = outputHandler({ path, cells }); + + const throttle = new Throttle(MAX_MSGS_PER_SECOND); + let unhandledClientWriteError: any = undefined; + throttle.on("data", async (mesgs) => { + try { + socket.write(mesgs); + } catch (err) { + if (err.code == "ENOBUFS") { + // wait for the over-filled socket to finish writing out data. + await socket.drain(); + socket.write(mesgs); + } else { + unhandledClientWriteError = err; + } + } + }); + + try { + let handler: OutputHandler | null = null; + for await (const mesg of runner) { + if (socket.state == "closed") { + // client socket has closed -- the backend server must take over! if (handler == null) { - throw Error("bug -- outputHandler must return a handler"); - } - for (const prev of output) { - handler.process(prev); + logger.debug("socket closed -- server must handle output"); + if (outputHandler == null) { + throw Error("no output handler available"); + } + handler = outputHandler({ path, cells }); + if (handler == null) { + throw Error("bug -- outputHandler must return a handler"); + } + for (const prev of output) { + handler.process(prev); + } + output.length = 0; } - output.length = 0; - } - handler.process(mesg); - } else { - output.push(mesg); - try { - socket.write([mesg]); - } catch (err) { - if (err.code == "ENOBUFS") { - // wait for the over-filled socket to finish writing out data. - await socket.drain(); - socket.write([mesg]); - } else { - throw err; + handler.process(mesg); + } else { + if (unhandledClientWriteError) { + throw unhandledClientWriteError; } + output.push(mesg); + throttle.write(mesg); } } + handler?.done(); + } finally { + if (socket.state != "closed" && !unhandledClientWriteError) { + throttle.flush(); + socket.write(null); + } + throttle.close(); } - handler?.done(); - socket.write(null); } class JupyterClient { diff --git a/src/packages/conat/socket/util.ts b/src/packages/conat/socket/util.ts index 2baef9fbea..5274ba1206 100644 --- a/src/packages/conat/socket/util.ts +++ b/src/packages/conat/socket/util.ts @@ -13,7 +13,7 @@ export type Role = "client" | "server"; // socketio and use those to manage things. This ping // is entirely a "just in case" backup if some event // were missed (e.g., a kill -9'd process...) -export const PING_PONG_INTERVAL = 90000; +export const PING_PONG_INTERVAL = 90_000; // We queue up unsent writes, but only up to a point (to not have a huge memory issue). // Any write beyond this size result in an exception. @@ -22,7 +22,7 @@ export const PING_PONG_INTERVAL = 90000; // Also note that this is just the *number* of messages, and a message can have // any size. But determining message size is very difficult without serializing the // message, which costs. -export const DEFAULT_MAX_QUEUE_SIZE = 10_000; +export const DEFAULT_MAX_QUEUE_SIZE = 1_000; export let DEFAULT_COMMAND_TIMEOUT = 10_000; export let DEFAULT_KEEP_ALIVE = 25_000; diff --git a/src/packages/project/conat/terminal/session.ts b/src/packages/project/conat/terminal/session.ts index 63a0097857..8afd87b1c9 100644 --- a/src/packages/project/conat/terminal/session.ts +++ b/src/packages/project/conat/terminal/session.ts @@ -12,7 +12,7 @@ import { } from "@cocalc/conat/service/terminal"; import { project_id, compute_server_id } from "@cocalc/project/data"; import { throttle } from "lodash"; -import { ThrottleString as Throttle } from "@cocalc/util/throttle"; +import { ThrottleString } from "@cocalc/util/throttle"; import { join } from "path"; import type { CreateTerminalOptions } from "@cocalc/conat/project/api/editor"; import { delay } from "awaiting"; @@ -55,7 +55,7 @@ const MAX_BYTES_PER_SECOND = parseInt( // having to discard writes. This is basically the "frame rate" // we are supporting for users. const MAX_MSGS_PER_SECOND = parseInt( - process.env.COCALC_TERMINAL_MAX_MSGS_PER_SECOND ?? "24", + process.env.COCALC_TERMINAL_MAX_MSGS_PER_SECOND ?? "20", ); type State = "running" | "off" | "closed"; @@ -236,7 +236,7 @@ export class Session { // use slighlty less than MAX_MSGS_PER_SECOND to avoid reject // due to being *slightly* off. - const throttle = new Throttle(1000 / (MAX_MSGS_PER_SECOND - 3)); + const throttle = new ThrottleString(MAX_MSGS_PER_SECOND - 3); throttle.on("data", (data: string) => { // logger.debug("got data out of pty"); this.handleBackendMessages(data); diff --git a/src/packages/util/throttle.test.ts b/src/packages/util/throttle.test.ts new file mode 100644 index 0000000000..ac357c544f --- /dev/null +++ b/src/packages/util/throttle.test.ts @@ -0,0 +1,74 @@ +import { ThrottleString, Throttle } from "./throttle"; +import { delay } from "awaiting"; + +describe("a throttled string", () => { + let t; + let output = ""; + it("creates a throttled string", () => { + // emits 10 times a second or once very 100ms. + t = new ThrottleString(10); + t.on("data", (data) => { + output += data; + }); + }); + + it("write 3 times and wait 50ms and get nothing, then 70 more ms and get all", async () => { + t.write("a"); + t.write("b"); + t.write("c"); + await delay(50); + expect(output).toBe(""); + // this "d" also gets included -- it makes it in before the cutoff. + t.write("d"); + await delay(70); + expect(output).toBe("abcd"); + }); + + it("do the same again", async () => { + t.write("a"); + t.write("b"); + t.write("c"); + await delay(50); + expect(output).toBe("abcd"); + t.write("d"); + await delay(70); + expect(output).toBe("abcdabcd"); + }); +}); + +describe("a throttled list of objects", () => { + let t; + let output: any[] = []; + + it("creates a throttled any[]", () => { + // emits 10 times a second or once very 100ms. + t = new Throttle(10); + t.on("data", (data: any[]) => { + output = output.concat(data); + }); + }); + + it("write 3 times and wait 50ms and get nothing, then 70 more ms and get all", async () => { + t.write("a"); + t.write("b"); + t.write("c"); + await delay(50); + expect(output).toEqual([]); + // this "d" also gets included -- it makes it in before the cutoff. + t.write("d"); + await delay(70); + expect(output).toEqual(["a", "b", "c", "d"]); + }); + + it("do it again", async () => { + t.write("a"); + t.write("b"); + t.write("c"); + await delay(50); + expect(output).toEqual(["a", "b", "c", "d"]); + // this "d" also gets included -- it makes it in before the cutoff. + t.write("d"); + await delay(70); + expect(output).toEqual(["a", "b", "c", "d", "a", "b", "c", "d"]); + }); +}); diff --git a/src/packages/util/throttle.ts b/src/packages/util/throttle.ts index 2fff3d2596..92e64dfab9 100644 --- a/src/packages/util/throttle.ts +++ b/src/packages/util/throttle.ts @@ -3,15 +3,22 @@ This is a really simple but incredibly useful little class. See packages/project/conat/terminal.ts for how to use it to make it so the terminal sends output at a rate of say "24 frames per second". + +This could also be called "buffering"... */ import { EventEmitter } from "events"; +const DEFAULT_MESSAGES_PER_SECOND = 24; + +// Throttling a string where use "+" to add more to our buffer export class ThrottleString extends EventEmitter { private buf: string = ""; private last = Date.now(); + private interval: number; - constructor(private interval: number) { + constructor(messagesPerSecond: number = DEFAULT_MESSAGES_PER_SECOND) { super(); + this.interval = 1000 / messagesPerSecond; } write = (data: string) => { @@ -34,20 +41,33 @@ export class ThrottleString extends EventEmitter { }; } -export class ThrottleAny extends EventEmitter { - private buf: any[] = []; +// Throttle a list of objects, where push them into an array to add more to our buffer. +export class Throttle extends EventEmitter { + private buf: T[] = []; private last = Date.now(); + private interval: number; - constructor(private interval: number) { + constructor(messagesPerSecond: number = DEFAULT_MESSAGES_PER_SECOND) { super(); + this.interval = 1000 / messagesPerSecond; } - write = (data: any) => { + // if you want data to be sent be sure to flush before closing + close = () => { + this.removeAllListeners(); + this.buf.length = 0; + }; + + write = (data: T) => { this.buf.push(data); + this.update(); + }; + + private update = () => { const now = Date.now(); const timeUntilEmit = this.interval - (now - this.last); if (timeUntilEmit > 0) { - setTimeout(() => this.write([]), timeUntilEmit); + setTimeout(() => this.update(), timeUntilEmit); } else { this.flush(); } From f18e917931bead026c582d88ef35231b11236a1e Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 1 Aug 2025 05:01:15 +0000 Subject: [PATCH 151/798] mainly fix using File --> Duplicate, etc.; did this by being more careful about timing using async instead of callbacks more. improved typings as well. --- .../course/assignments/assignment.tsx | 4 +- .../frontend/editors/file-info-dropdown.tsx | 6 +- .../frame-tree/commands/generic-commands.tsx | 3 +- .../frontend/project/explorer/action-bar.tsx | 3 +- .../frontend/project/explorer/action-box.tsx | 80 ++++---- .../frontend/project/explorer/explorer.tsx | 1 - .../project/explorer/file-listing/utils.ts | 4 +- src/packages/frontend/project/open-file.ts | 3 +- .../frontend/project/page/share-indicator.tsx | 2 +- src/packages/frontend/project_actions.ts | 188 +++++++++--------- src/packages/frontend/project_store.ts | 5 +- src/packages/frontend/projects/store.ts | 12 +- 12 files changed, 156 insertions(+), 155 deletions(-) diff --git a/src/packages/frontend/course/assignments/assignment.tsx b/src/packages/frontend/course/assignments/assignment.tsx index 276aaa1364..9849c2592e 100644 --- a/src/packages/frontend/course/assignments/assignment.tsx +++ b/src/packages/frontend/course/assignments/assignment.tsx @@ -407,7 +407,7 @@ export function Assignment({ ); } - function open_assignment_path(): void { + async function open_assignment_path() { if (assignment.get("listing")?.size == 0) { // there are no files yet, so we *close* the assignment // details panel. This is just **a hack** so that the user @@ -421,7 +421,7 @@ export function Assignment({ assignment.get("assignment_id"), ); } - return redux + await redux .getProjectActions(project_id) .open_directory(assignment.get("path")); } diff --git a/src/packages/frontend/editors/file-info-dropdown.tsx b/src/packages/frontend/editors/file-info-dropdown.tsx index bcbf3530a2..e25de0188d 100644 --- a/src/packages/frontend/editors/file-info-dropdown.tsx +++ b/src/packages/frontend/editors/file-info-dropdown.tsx @@ -11,7 +11,7 @@ import { CSS, React, useActions } from "@cocalc/frontend/app-framework"; import { DropdownMenu, Icon, IconName } from "@cocalc/frontend/components"; import { MenuItems } from "@cocalc/frontend/components/dropdown-menu"; import { useStudentProjectFunctionality } from "@cocalc/frontend/course"; -import { file_actions } from "@cocalc/frontend/project_store"; +import { file_actions, type FileAction } from "@cocalc/frontend/project_store"; import { capitalize, filename_extension } from "@cocalc/util/misc"; interface Props { @@ -57,9 +57,9 @@ const EditorFileInfoDropdown: React.FC = React.memo( } for (const key in file_actions) { if (key === name) { - actions.show_file_action_panel({ + actions.showFileActionPanel({ path: filename, - action: key, + action: key as FileAction, }); break; } diff --git a/src/packages/frontend/frame-editors/frame-tree/commands/generic-commands.tsx b/src/packages/frontend/frame-editors/frame-tree/commands/generic-commands.tsx index 1de09b438a..0bb95fecce 100644 --- a/src/packages/frontend/frame-editors/frame-tree/commands/generic-commands.tsx +++ b/src/packages/frontend/frame-editors/frame-tree/commands/generic-commands.tsx @@ -7,7 +7,6 @@ import { Input } from "antd"; import { debounce } from "lodash"; import { useEffect, useRef } from "react"; import { defineMessage, IntlShape, useIntl } from "react-intl"; - import { set_account_table } from "@cocalc/frontend/account/util"; import { redux } from "@cocalc/frontend/app-framework"; import { Icon } from "@cocalc/frontend/components"; @@ -1467,7 +1466,7 @@ function fileAction(action) { alwaysShow: true, onClick: ({ props }) => { const actions = redux.getProjectActions(props.project_id); - actions.show_file_action_panel({ + actions.showFileActionPanel({ path: props.path, action, }); diff --git a/src/packages/frontend/project/explorer/action-bar.tsx b/src/packages/frontend/project/explorer/action-bar.tsx index 80977d97dd..58a30dbfe2 100644 --- a/src/packages/frontend/project/explorer/action-bar.tsx +++ b/src/packages/frontend/project/explorer/action-bar.tsx @@ -18,6 +18,7 @@ import { labels } from "@cocalc/frontend/i18n"; import { file_actions, type ProjectActions, + type FileAction, } from "@cocalc/frontend/project_store"; import * as misc from "@cocalc/util/misc"; import { COLORS } from "@cocalc/util/theme"; @@ -195,7 +196,7 @@ export function ActionBar({ } } - function render_action_button(name: string): React.JSX.Element { + function render_action_button(name: FileAction): React.JSX.Element { const disabled = isDisabledSnapshots(name) && (current_path != null diff --git a/src/packages/frontend/project/explorer/action-box.tsx b/src/packages/frontend/project/explorer/action-box.tsx index 2a25f09bfd..110bdf2e5b 100644 --- a/src/packages/frontend/project/explorer/action-box.tsx +++ b/src/packages/frontend/project/explorer/action-box.tsx @@ -18,7 +18,7 @@ import { Well, } from "@cocalc/frontend/antd-bootstrap"; import { useRedux, useTypedRedux } from "@cocalc/frontend/app-framework"; -import { Icon, Loading, LoginLink } from "@cocalc/frontend/components"; +import { Icon, LoginLink } from "@cocalc/frontend/components"; import SelectServer from "@cocalc/frontend/compute/select-server"; import ComputeServerTag from "@cocalc/frontend/compute/server-tag"; import { useRunQuota } from "@cocalc/frontend/project/settings/run-quota/hooks"; @@ -54,7 +54,6 @@ interface Props { file_action: FileAction; current_path: string; project_id: string; - file_map: object; actions: ProjectActions; } @@ -63,7 +62,6 @@ export function ActionBox({ file_action, current_path, project_id, - file_map, actions, }: Props) { const intl = useIntl(); @@ -588,44 +586,40 @@ export function ActionBox({ if (action_button == undefined) { return
Undefined action
; } - if (file_map == undefined) { - return ; - } else { - return ( - - - - {" "} - {intl.formatMessage(action_button.name)} -
- - - -
- {!!compute_server_id && ( - - )} - - {render_action_box(action)} - - - ); - } + return ( + + + + {" "} + {intl.formatMessage(action_button.name)} +
+ + + +
+ {!!compute_server_id && ( + + )} + + {render_action_box(action)} + + + ); } diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index 47349a95c4..4694e6ac5d 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -467,7 +467,6 @@ export function Explorer() { = React.memo( -
-
- ); - }); + const v: JSX.Element[] = []; + const attachments = cell.get("attachments"); + if (attachments) { + attachments.forEach((_, name) => { + if (v.length > 0) { + v.push(
); } - } - if (v.length === 0) { - return ( - - There are no attachments. To attach images, use Edit → Insert - Image. - + return v.push( +
+
{name}
+
+ +
+
, ); - } - return v; + }); + } + if (v.length === 0) { + return ( + + There are no attachments. To attach images, use Edit → Insert + Image. + + ); + } + + function close() { + actions.setState({ edit_attachments: undefined }); + actions.focus(true); } return ( @@ -79,7 +78,7 @@ export function EditAttachments({ actions, cell }: EditAttachmentsProps) { } > - {renderAttachments()} + {v} ); } diff --git a/src/packages/frontend/jupyter/main.tsx b/src/packages/frontend/jupyter/main.tsx index aca7b02f86..77a96a80c4 100644 --- a/src/packages/frontend/jupyter/main.tsx +++ b/src/packages/frontend/jupyter/main.tsx @@ -12,11 +12,10 @@ import { CSS, React, redux, - Rendered, useRedux, - useRef, useTypedRedux, } from "@cocalc/frontend/app-framework"; +import { useRef } from "react"; // Support for all the MIME types import { Button, Tooltip } from "antd"; import "./output-messages/mime-types/init-frontend"; @@ -117,7 +116,6 @@ export const JupyterEditor: React.FC = React.memo((props: Props) => { "show_kernel_selector", ]); // string name of the kernel - const kernels: undefined | KernelsType = useRedux([name, "kernels"]); const kernelspec = useRedux([name, "kernel_info"]); const error: undefined | KernelsType = useRedux([name, "error"]); // settings for all the codemirror editors @@ -261,25 +259,11 @@ export const JupyterEditor: React.FC = React.memo((props: Props) => { ); } - function render_loading(): Rendered { - return ( - - ); - } - function render_cells() { if ( cell_list == null || font_size == null || cm_options == null || - kernels == null || cells == null ) { return ( @@ -327,95 +311,20 @@ export const JupyterEditor: React.FC = React.memo((props: Props) => { ); } - function render_about() { - return ( - - ); - } - - function render_nbconvert() { - if (path == null || project_id == null) return; - return ( - - ); - } - - function render_edit_attachments() { - if (edit_attachments == null || cells == null) { - return; - } - const cell = cells.get(edit_attachments); - if (cell == null) { - return; - } - return ; - } - - function render_edit_cell_metadata() { - if (edit_cell_metadata == null) { - return; - } - return ( - - ); - } - - function render_find_and_replace() { - if (cells == null || cur_id == null) { - return; - } - return ( - - ); - } - - function render_confirm_dialog() { - if (confirm_dialog == null || actions == null) return; - return ; - } - function render_select_kernel() { return ; } - function render_keyboard_shortcuts() { - if (actions == null) return; - return ( - - ); - } - function render_main() { if (!check_select_kernel_init) { - return render_loading(); + ; } else if (show_kernel_selector) { return render_select_kernel(); } else { @@ -427,13 +336,58 @@ export const JupyterEditor: React.FC = React.memo((props: Props) => { if (!is_focused) return; return ( <> - {render_about()} - {render_nbconvert()} - {render_edit_attachments()} - {render_edit_cell_metadata()} - {render_find_and_replace()} - {render_keyboard_shortcuts()} - {render_confirm_dialog()} + + {path != null && project_id != null && ( + + )} + {edit_attachments != null && ( + + )} + {edit_cell_metadata != null && ( + + )} + {cells != null && cur_id != null && ( + + )} + {actions != null && ( + + )} + {actions != null && confirm_dialog != null && ( + + )} ); } diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index debebb2a4f..41615bac30 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -688,6 +688,7 @@ export abstract class JupyterActions extends Actions { this.set_cell_list(); } + this.ensureThereIsACell(); this.__syncdb_change_post_hook(doInit); }; @@ -2505,6 +2506,25 @@ export abstract class JupyterActions extends Actions { } this.setState({ runProgress: total > 0 ? (100 * ran) / total : 100 }); }; + + ensureThereIsACell = () => { + if (this._state !== "ready") { + return; + } + const cells = this.store.get("cells"); + if (cells == null || cells.size === 0) { + this._set({ + type: "cell", + // by using the same id across clients we solve the problem of multiple + // clients creating a cell at the same time. + id: "alpha", + pos: 0, + input: "", + }); + // We are obviously contributing content to this (empty!) notebook. + return this.set_trust_notebook(true); + } + }; } function extractLabel(content: string): string { diff --git a/src/packages/jupyter/redux/project-actions.ts b/src/packages/jupyter/redux/project-actions.ts index b30b01d779..a334f5f3a9 100644 --- a/src/packages/jupyter/redux/project-actions.ts +++ b/src/packages/jupyter/redux/project-actions.ts @@ -287,7 +287,6 @@ export class JupyterActions extends JupyterActions0 { }); } - this.ensure_there_is_a_cell(); this._throttled_ensure_positions_are_unique(); }; @@ -784,23 +783,6 @@ export class JupyterActions extends JupyterActions0 { } }; - ensure_there_is_a_cell = () => { - if (this._state !== "ready") { - return; - } - const cells = this.store.get("cells"); - if (cells == null || cells.size === 0) { - this._set({ - type: "cell", - id: this.new_id(), - pos: 0, - input: "", - }); - // We are obviously contributing content to this (empty!) notebook. - return this.set_trust_notebook(true); - } - }; - private handle_all_cell_attachments() { // Check if any cell attachments need to be loaded. const cells = this.store.get("cells"); From 84077c318afa20ea3c9ff96adedbe4e6a5cc3c98 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 1 Aug 2025 17:19:32 +0000 Subject: [PATCH 155/798] delete the old project actions entirely -- starting from scratch with something much more direct --- src/packages/jupyter/control.ts | 4 +- src/packages/jupyter/kernel/kernel.ts | 3 +- src/packages/jupyter/redux/actions.ts | 31 +- src/packages/jupyter/redux/project-actions.ts | 981 +----------------- src/packages/project/conat/jupyter.ts | 19 + src/packages/sync/editor/generic/sync-doc.ts | 4 +- 6 files changed, 52 insertions(+), 990 deletions(-) diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts index 58cf958bfa..1020c94ec7 100644 --- a/src/packages/jupyter/control.ts +++ b/src/packages/jupyter/control.ts @@ -84,7 +84,7 @@ export async function jupyterRun({ path, cells, noHalt }: RunOptions) { logger.debug("jupyterRun: running"); async function* run() { for (const cell of cells) { - actions.ensure_backend_kernel_setup(); + actions.initKernel(); const output = actions.jupyter_kernel.execute_code({ halt_on_error: !noHalt, code: cell.input, @@ -125,7 +125,7 @@ class MulticellOutputHandler { () => { const { id, state, output, start, end, exec_count } = cell; this.actions._set( - { type:"cell", id, state, output, start, end, exec_count }, + { type: "cell", id, state, output, start, end, exec_count }, true, ); }, diff --git a/src/packages/jupyter/kernel/kernel.ts b/src/packages/jupyter/kernel/kernel.ts index a3e67c6e9a..f8ef7ac1fa 100644 --- a/src/packages/jupyter/kernel/kernel.ts +++ b/src/packages/jupyter/kernel/kernel.ts @@ -36,7 +36,6 @@ import { jupyterSockets, type JupyterSockets } from "@cocalc/jupyter/zmq"; import { EventEmitter } from "node:events"; import { unlink } from "@cocalc/backend/misc/async-utils-node"; import { remove_redundant_reps } from "@cocalc/jupyter/ipynb/import-from-ipynb"; -import { JupyterActions } from "@cocalc/jupyter/redux/project-actions"; import { type BlobStoreInterface, CodeExecutionEmitterInterface, @@ -44,6 +43,7 @@ import { JupyterKernelInterface, KernelInfo, } from "@cocalc/jupyter/types/project-interface"; +import { JupyterActions } from "@cocalc/jupyter/redux/project-actions"; import { JupyterStore } from "@cocalc/jupyter/redux/store"; import { JUPYTER_MIMETYPES } from "@cocalc/jupyter/util/misc"; import type { SyncDB } from "@cocalc/sync/editor/db/sync"; @@ -151,7 +151,6 @@ export function initJupyterRedux(syncdb: SyncDB, client: Client) { } const store = redux.createStore(name, JupyterStore); const actions = redux.createActions(name, JupyterActions); - actions._init(project_id, path, syncdb, store, client); syncdb.once("error", (err) => diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index 41615bac30..7c3c9f85ba 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -64,7 +64,7 @@ const CellDeleteProtectedException = new Error("CellDeleteProtectedException"); type State = "init" | "load" | "ready" | "closed"; -export abstract class JupyterActions extends Actions { +export class JupyterActions extends Actions { public is_project: boolean; public is_compute_server?: boolean; readonly path: string; @@ -95,6 +95,7 @@ export abstract class JupyterActions extends Actions { store: any, client: Client, ): void { + console.log("jupyter actions: _init", { path }); this._client = client; const dbg = this.dbg("_init"); dbg("Initializing Jupyter Actions"); @@ -113,20 +114,17 @@ export abstract class JupyterActions extends Actions { this.path = path; store.syncdb = syncdb; this.syncdb = syncdb; - // the project client is designated to manage execution/conflict, etc. this.is_project = client.is_project(); - if (this.is_project) { - this.syncdb.on("first-load", () => { - dbg("handling first load of syncdb in project"); - // Clear settings the first time the syncdb is ever - // loaded, since it has settings like "ipynb last save" - // and trust, which shouldn't be initialized to - // what they were before. Not doing this caused - // https://github.com/sagemathinc/cocalc/issues/7074 - this.syncdb.delete({ type: "settings" }); - this.syncdb.commit(); - }); - } + this.syncdb.on("first-load", () => { + dbg("handling first load of syncdb"); + // Clear settings the first time the syncdb is ever + // loaded, since it has settings like "ipynb last save" + // and trust, which shouldn't be initialized to + // what they were before. Not doing this caused + // https://github.com/sagemathinc/cocalc/issues/7074 + this.syncdb.delete({ type: "settings" }); + this.syncdb.commit(); + }); this.is_compute_server = client.is_compute_server(); let directory: any; @@ -983,7 +981,10 @@ export abstract class JupyterActions extends Actions { this.deprecated("run_selected_cells"); }; - abstract runCells(ids: string[], opts?: { noHalt?: boolean }): Promise; + runCells(_ids: string[], _opts?: { noHalt?: boolean }): Promise { + // defined in derived class (e.g., frontend browser). + throw Error("DEPRECATED"); + } run_all_cells = (no_halt: boolean = false): void => { this.runCells(this.store.get_cell_list().toJS(), { noHalt: no_halt }); diff --git a/src/packages/jupyter/redux/project-actions.ts b/src/packages/jupyter/redux/project-actions.ts index a334f5f3a9..8befaacec2 100644 --- a/src/packages/jupyter/redux/project-actions.ts +++ b/src/packages/jupyter/redux/project-actions.ts @@ -14,993 +14,34 @@ fully unit test it via mocking of components. NOTE: this is also now the actions used by remote compute servers as well. */ -import { get_kernel_data } from "@cocalc/jupyter/kernel/kernel-data"; -import * as immutable from "immutable"; -import json_stable from "json-stable-stringify"; -import { debounce } from "lodash"; import { JupyterActions as JupyterActions0 } from "@cocalc/jupyter/redux/actions"; -import { callback2, once } from "@cocalc/util/async-utils"; -import * as misc from "@cocalc/util/misc"; -import { RunAllLoop } from "./run-all-loop"; -import nbconvertChange from "./handle-nbconvert-change"; -import type { ClientFs } from "@cocalc/sync/client/types"; import { kernel as createJupyterKernel } from "@cocalc/jupyter/kernel"; -import { removeJupyterRedux } from "@cocalc/jupyter/kernel"; -import { initConatService } from "@cocalc/jupyter/kernel/conat-service"; -import { type DKV, dkv } from "@cocalc/conat/sync/dkv"; -import { computeServerManager } from "@cocalc/conat/compute/manager"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; - -// refuse to open an ipynb that is bigger than this: -const MAX_SIZE_IPYNB_MB = 150; - -type BackendState = "init" | "ready" | "spawning" | "starting" | "running"; export class JupyterActions extends JupyterActions0 { - private _backend_state: BackendState = "init"; - private lastSavedBackendState?: BackendState; - private _initialize_manager_already_done: any; - private _kernel_state: any; - private _running_cells: { [id: string]: string }; - private _throttled_ensure_positions_are_unique: any; - private run_all_loop?: RunAllLoop; - private clear_kernel_error?: any; - private last_ipynb_save: number = 0; - protected _client: ClientFs; // this has filesystem access, etc. - public blobs: DKV; - private computeServers?; - - private initBlobStore = async () => { - this.blobs = await dkv(this.blobStoreOptions()); - }; - - // uncomment for verbose logging of everything here to the console. - // dbg(f: string) { - // return (...args) => console.log(f, args); - // } - - async runCells( - _ids: string[], - _opts: { noHalt?: boolean } = {}, - ): Promise { - throw Error("DEPRECATED"); - } - - private set_backend_state(backend_state: BackendState): void { - this.dbg("set_backend_state")(backend_state); - - /* - The backend states, which are put in the syncdb so clients - can display this: - - - 'init' -- the backend is checking the file on disk, etc. - - 'ready' -- the backend is setup and ready to use; kernel isn't running though - - 'starting' -- the kernel itself is actived and currently starting up (e.g., Sage is starting up) - - 'running' -- the kernel is running and ready to evaluate code - - - 'init' --> 'ready' --> 'spawning' --> 'starting' --> 'running' - /|\ | - |-----------------------------------------| - - Going from ready to starting happens first when a code execution is requested. - */ - - // Check just in case Typescript doesn't catch something: - if ( - ["init", "ready", "spawning", "starting", "running"].indexOf( - backend_state, - ) === -1 - ) { - throw Error(`invalid backend state '${backend_state}'`); - } - if (backend_state == "init" && this._backend_state != "init") { - // Do NOT allow changing the state to init from any other state. - throw Error( - `illegal state change '${this._backend_state}' --> '${backend_state}'`, - ); - } - this._backend_state = backend_state; - - if (this.lastSavedBackendState != backend_state) { - this._set({ - type: "settings", - backend_state, - last_backend_state: Date.now(), - }); - this.save_asap(); - this.lastSavedBackendState = backend_state; - } - - // The following is to clear kernel_error if things are working only. - if (backend_state == "running") { - // clear kernel error if kernel successfully starts and stays - // in running state for a while. - this.clear_kernel_error = setTimeout(() => { - this._set({ - type: "settings", - kernel_error: "", - }); - }, 3000); - } else { - // change to a different state; cancel attempt to clear kernel error - if (this.clear_kernel_error) { - clearTimeout(this.clear_kernel_error); - delete this.clear_kernel_error; - } - } - } - - set_kernel_state = (state: any, save = false) => { - this._kernel_state = state; - this._set({ type: "settings", kernel_state: state }, save); - }; - - // Called exactly once when the manager first starts up after the store is initialized. - // Here we ensure everything is in a consistent state so that we can react - // to changes later. - async initialize_manager() { - if (this._initialize_manager_already_done) { - return; - } - const dbg = this.dbg("initialize_manager"); - dbg(); - this._initialize_manager_already_done = true; - - dbg("initialize Jupyter Conat api handler"); - await this.initConatApi(); - - dbg("initializing blob store"); - await this.initBlobStore(); - - this._throttled_ensure_positions_are_unique = debounce( - this.ensure_positions_are_unique, - 5000, - ); - // Listen for changes... - this.syncdb.on("change", this.backendSyncdbChange); - - this.setState({ - // used by the kernel_info function of this.jupyter_kernel - start_time: this._client.server_time().valueOf(), - }); - - // clear nbconvert start on init, since no nbconvert can be running yet - this.syncdb.delete({ type: "nbconvert" }); - - // Initialize info about available kernels, which is used e.g., for - // saving to ipynb format. - this.init_kernel_info(); - - // We try once to load from disk. If it fails, then - // a record with type:'fatal' - // is created in the database; if it succeeds, that record is deleted. - // Try again only when the file changes. - await this._first_load(); - - // Listen for model state changes... - if (this.syncdb.ipywidgets_state == null) { - throw Error("syncdb's ipywidgets_state must be defined!"); - } - this.syncdb.ipywidgets_state.on( - "change", - this.handle_ipywidgets_state_change, - ); - } - - private conatService?; - private initConatApi = reuseInFlight(async () => { - if (this.conatService != null) { - this.conatService.close(); - this.conatService = null; - } - const service = (this.conatService = await initConatService({ - project_id: this.project_id, - path: this.path, - })); - this.syncdb.on("closed", () => { - service.close(); - }); - }); - - private _first_load = async () => { - const dbg = this.dbg("_first_load"); - dbg("doing load"); - if (this.is_closed()) { - throw Error("actions must not be closed"); - } - try { - await this.loadFromDiskIfNewer(); - } catch (err) { - dbg(`load failed -- ${err}; wait for file change and try again`); - const path = this.store.get("path"); - const watcher = this._client.watch_file({ path }); - await once(watcher, "change"); - dbg("file changed"); - watcher.close(); - await this._first_load(); - return; - } - dbg("loading worked"); - this._init_after_first_load(); - }; - - private _init_after_first_load = () => { - const dbg = this.dbg("_init_after_first_load"); - - dbg("initializing"); - // this may change the syncdb. - this.ensure_backend_kernel_setup(); - - this.init_file_watcher(); - - this._state = "ready"; + public blobs = { + set: (_k, _v) => {}, + get: (_k): any => {}, }; + save_ipynb_file = async (_opts?) => {}; + capture_output_message = (_opts) => {}; + process_comm_message_from_kernel = (_mesg) => {}; - private backendSyncdbChange = (changes: any) => { - if (this.is_closed()) { - return; - } - const dbg = this.dbg("backendSyncdbChange"); - if (changes != null) { - changes.forEach((key) => { - switch (key.get("type")) { - case "settings": - dbg("settings change"); - var record = this.syncdb.get_one(key); - if (record != null) { - // ensure kernel is properly configured - this.ensure_backend_kernel_setup(); - // only the backend should change kernel and backend state; - // however, our security model allows otherwise (e.g., via TimeTravel). - if ( - record.get("kernel_state") !== this._kernel_state && - this._kernel_state != null - ) { - this.set_kernel_state(this._kernel_state, true); - } - if (record.get("backend_state") !== this._backend_state) { - this.set_backend_state(this._backend_state); - } - - if (record.get("run_all_loop_s")) { - if (this.run_all_loop == null) { - this.run_all_loop = new RunAllLoop( - this, - record.get("run_all_loop_s"), - ); - } else { - // ensure interval is correct - this.run_all_loop.set_interval(record.get("run_all_loop_s")); - } - } else if ( - !record.get("run_all_loop_s") && - this.run_all_loop != null - ) { - // stop it. - this.run_all_loop.close(); - delete this.run_all_loop; - } - } - break; - } - }); - } - - this._throttled_ensure_positions_are_unique(); - }; - - // ensure_backend_kernel_setup ensures that we have a connection - // to the selected Jupyter kernel, if any. - ensure_backend_kernel_setup = () => { - const dbg = this.dbg("ensure_backend_kernel_setup"); - if (this.isDeleted()) { - dbg("file is deleted"); + initKernel = () => { + if (this.jupyter_kernel != null) { return; } - const kernel = this.store.get("kernel"); - dbg("ensure_backend_kernel_setup", { kernel }); - - let current: string | undefined = undefined; - if (this.jupyter_kernel != null) { - current = this.jupyter_kernel.name; - if (current == kernel) { - const state = this.jupyter_kernel.get_state(); - if (state == "error") { - dbg("kernel is broken"); - // nothing to do -- let user ponder the error they should see. - return; - } - if (state != "closed") { - dbg("everything is properly setup and working"); - return; - } - } - } - - dbg(`kernel='${kernel}', current='${current}'`); - if ( - this.jupyter_kernel != null && - this.jupyter_kernel.get_state() != "closed" - ) { - if (current != kernel) { - dbg("kernel changed -- kill running kernel to trigger switch"); - this.jupyter_kernel.close(); - return; - } else { - dbg("nothing to do"); - return; - } - } - - dbg("make a new kernel"); - + console.log("initKernel", { kernel, path: this.path }); // No kernel wrapper object setup at all. Make one. this.jupyter_kernel = createJupyterKernel({ name: kernel, - path: this.store.get("path"), + path: this.path, actions: this, }); - - if (this.syncdb.ipywidgets_state == null) { - throw Error("syncdb's ipywidgets_state must be defined!"); - } - this.syncdb.ipywidgets_state.clear(); - - if (this.jupyter_kernel == null) { - // to satisfy typescript. - throw Error("jupyter_kernel must be defined"); - } - dbg("kernel created -- installing handlers"); - - // save so gets reported to frontend, and surfaced to user: - // https://github.com/sagemathinc/cocalc/issues/4847 - this.jupyter_kernel.on("kernel_error", (error) => { - this.set_kernel_error(error); - }); - - this.restartKernelOnClose = () => { - // When the kernel closes, make sure a new kernel gets setup. - if (this.store == null || this._state !== "ready") { - // This event can also happen when this actions is being closed, - // in which case obviously we shouldn't make a new kernel. - return; - } - dbg("kernel closed -- make new one."); - this.ensure_backend_kernel_setup(); - }; - - this.jupyter_kernel.once("closed", this.restartKernelOnClose); - - // Track backend state changes other than closing, so they - // are visible to user etc. - // TODO: Maybe all these need to move to ephemeral table? - // There's a good argument that recording these is useful though, so when - // looking at time travel or debugging, you know what was going on. - this.jupyter_kernel.on("state", (state) => { - dbg("jupyter_kernel state --> ", state); - switch (state) { - case "off": - case "closed": - // things went wrong. - this.set_backend_state("ready"); - this.jupyter_kernel?.close(); - delete this.jupyter_kernel; - return; - case "spawning": - case "starting": - this.set_connection_file(); // yes, fall through - case "running": - this.set_backend_state(state); - } - }); - - this.jupyter_kernel.on("execution_state", this.set_kernel_state); - - this.handle_all_cell_attachments(); - dbg("ready"); - this.set_backend_state("ready"); - }; - - set_connection_file = () => { - const connection_file = this.jupyter_kernel?.get_connection_file() ?? ""; - this._set({ - type: "settings", - connection_file, - }); }; - init_kernel_info = async () => { - let kernels0 = this.store.get("kernels"); - if (kernels0 != null) { - return; - } - const dbg = this.dbg("init_kernel_info"); - dbg("getting"); - let kernels; - try { - kernels = await get_kernel_data(); - dbg("success"); - } catch (err) { - dbg(`FAILED to get kernel info: ${err}`); - // TODO: what to do?? Saving will be broken... - return; - } - this.setState({ - kernels: immutable.fromJS(kernels), - }); - }; - - async ensure_backend_kernel_is_running() { - const dbg = this.dbg("ensure_backend_kernel_is_running"); - if (this._backend_state == "ready") { - dbg("in state 'ready', so kick it into gear"); - await this.set_backend_kernel_info(); - dbg("done getting kernel info"); - } - const is_running = (s): boolean => { - if (this._state === "closed") { - return true; - } - const t = s.get_one({ type: "settings" }); - if (t == null) { - dbg("no settings"); - return false; - } else { - const state = t.get("backend_state"); - dbg(`state = ${state}`); - return state == "running"; - } - }; - await this.syncdb.wait(is_running, 60); - } - - protected __syncdb_change_post_hook(doInit: boolean) { - if (doInit) { - // Since just opening the actions in the project, definitely the kernel - // isn't running so set this fact in the shared database. It will make - // things always be in the right initial state. - this.syncdb.set({ - type: "settings", - backend_state: "init", - kernel_state: "idle", - kernel_usage: { memory: 0, cpu: 0 }, - }); - this.syncdb.commit(); - - // Also initialize the execution manager, which runs cells that have been - // requested to run. - this.initialize_manager(); - } - } - - _cancel_run = (id: any) => { - const dbg = this.dbg(`_cancel_run ${id}`); - // All these checks are so we only cancel if it is actually running - // with the current kernel... - if (this._running_cells == null || this.jupyter_kernel == null) return; - const identity = this._running_cells[id]; - if (identity == null) return; - if (this.jupyter_kernel.identity == identity) { - dbg("canceling"); - this.jupyter_kernel.cancel_execute(id); - } else { - dbg("not canceling since wrong identity"); - } - }; - - private init_file_watcher = () => { - const dbg = this.dbg("file_watcher"); - dbg(); - this._file_watcher = this._client.watch_file({ - path: this.store.get("path"), - debounce: 1000, - }); - - this._file_watcher.on("change", async () => { - dbg("change"); - try { - await this.loadFromDiskIfNewer(); - } catch (err) { - dbg("failed to load on change", err); - } - }); - }; - - // Load file from disk if it is newer than - // the last we saved to disk. - private loadFromDiskIfNewer = async () => { - const dbg = this.dbg("loadFromDiskIfNewer"); - // Get mtime of last .ipynb file that we explicitly saved. - - // TODO: breaking the syncdb typescript data hiding. The - // right fix will be to move - // this info to a new ephemeral state table. - const last_ipynb_save = await this.get_last_ipynb_save(); - dbg(`syncdb last_ipynb_save=${last_ipynb_save}`); - let file_changed; - if (last_ipynb_save == 0) { - // we MUST load from file the first time, of course. - file_changed = true; - dbg("file changed because FIRST TIME"); - } else { - const path = this.store.get("path"); - let stats; - try { - stats = await callback2(this._client.path_stat, { path }); - dbg(`stats.mtime = ${stats.mtime}`); - } catch (err) { - // This err just means the file doesn't exist. - // We set the 'last load' to now in this case, since - // the frontend clients need to know that we - // have already scanned the disk. - this.set_last_load(); - return; - } - const mtime = stats.mtime.getTime(); - file_changed = mtime > last_ipynb_save; - dbg({ mtime, last_ipynb_save }); - } - if (file_changed) { - dbg(".ipynb disk file changed ==> loading state from disk"); - try { - await this.load_ipynb_file(); - } catch (err) { - dbg("failed to load on change", err); - } - } else { - dbg("disk file NOT changed: NOT loading"); - } - }; - - // if also set load is true, we also set the "last_ipynb_save" time. - set_last_load = (alsoSetLoad: boolean = false) => { - const last_load = new Date().getTime(); - this.syncdb.set({ - type: "file", - last_load, - }); - if (alsoSetLoad) { - // yes, load v save is inconsistent! - this.syncdb.set({ type: "settings", last_ipynb_save: last_load }); - } - this.syncdb.commit(); - }; - - /* Determine timestamp of aux .ipynb file, and record it here, - so we know that we do not have to load exactly that file - back from disk. */ - private set_last_ipynb_save = async () => { - let stats; - try { - stats = await callback2(this._client.path_stat, { - path: this.store.get("path"), - }); - } catch (err) { - // no-op -- nothing to do. - this.dbg("set_last_ipynb_save")(`WARNING -- issue in path_stat ${err}`); - return; - } - - // This is ugly (i.e., how we get access), but I need to get this done. - // This is the RIGHT place to save the info though. - // TODO: move this state info to new ephemeral table. - try { - const last_ipynb_save = stats.mtime.getTime(); - this.last_ipynb_save = last_ipynb_save; - this._set({ - type: "settings", - last_ipynb_save, - }); - this.dbg("stats.mtime.getTime()")( - `set_last_ipynb_save = ${last_ipynb_save}`, - ); - } catch (err) { - this.dbg("set_last_ipynb_save")( - `WARNING -- issue in set_last_ipynb_save ${err}`, - ); - return; - } - }; - - private get_last_ipynb_save = async () => { - const x = - this.syncdb.get_one({ type: "settings" })?.get("last_ipynb_save") ?? 0; - return Math.max(x, this.last_ipynb_save); - }; - - load_ipynb_file = async () => { - /* - Read the ipynb file from disk. Fully use the ipynb file to - set the syncdb's state. We do this when opening a new file, or when - the file changes on disk (e.g., a git checkout or something). - */ - const dbg = this.dbg(`load_ipynb_file`); - dbg("reading file"); - const path = this.store.get("path"); - let content: string; - try { - content = await callback2(this._client.path_read, { - path, - maxsize_MB: MAX_SIZE_IPYNB_MB, - }); - } catch (err) { - // possibly file doesn't exist -- set notebook to empty. - const exists = await callback2(this._client.path_exists, { - path, - }); - if (!exists) { - content = ""; - } else { - // It would be better to have a button to push instead of - // suggesting running a command in the terminal, but - // adding that took 1 second. Better than both would be - // making it possible to edit huge files :-). - const error = `Error reading ipynb file '${path}': ${err.toString()}. Fix this to continue. You can delete all output by typing cc-jupyter-no-output [filename].ipynb in a terminal.`; - this.syncdb.set({ type: "fatal", error }); - throw Error(error); - } - } - if (content.length === 0) { - // Blank file, e.g., when creating in CoCalc. - // This is good, works, etc. -- just clear state, including error. - this.syncdb.delete(); - this.set_last_load(true); - return; - } - - // File is nontrivial -- parse and load. - let parsed_content; - try { - parsed_content = JSON.parse(content); - } catch (err) { - const error = `Error parsing the ipynb file '${path}': ${err}. You must fix the ipynb file somehow before continuing, or use TimeTravel to revert to a recent version.`; - dbg(error); - this.syncdb.set({ type: "fatal", error }); - throw Error(error); - } - this.syncdb.delete({ type: "fatal" }); - await this.set_to_ipynb(parsed_content); - this.set_last_load(true); - }; - - private fetch_jupyter_kernels = async () => { - const data = await get_kernel_data(); - const kernels = immutable.fromJS(data as any); - this.setState({ kernels }); - }; - - save_ipynb_file = async ({ - version = 0, - timeout = 15000, - }: { - // if version is given, waits (up to timeout ms) for syncdb to - // contain that exact version before writing the ipynb to disk. - // This may be needed to ensure that ipynb saved to disk - // reflects given frontend state. This comes up, e.g., in - // generating the nbgrader version of a document. - version?: number; - timeout?: number; - } = {}) => { - const dbg = this.dbg("save_ipynb_file"); - if (version && !this.syncdb.hasVersion(version)) { - dbg(`frontend needs ${version}, which we do not yet have`); - const start = Date.now(); - while (true) { - if (this.is_closed()) { - return; - } - if (Date.now() - start >= timeout) { - dbg("timed out waiting"); - break; - } - try { - dbg(`waiting for version ${version}`); - await once(this.syncdb, "change", timeout - (Date.now() - start)); - } catch { - dbg("timed out waiting"); - break; - } - if (this.syncdb.hasVersion(version)) { - dbg("now have the version"); - break; - } - } - } - if (this.is_closed()) { - return; - } - dbg("saving to file"); - - // Check first if file was deleted, in which case instead of saving to disk, - // we should terminate and clean up everything. - if (this.isDeleted()) { - dbg("ipynb file is deleted, so NOT saving to disk and closing"); - this.close(); - return; - } - - if (this.jupyter_kernel == null) { - // The kernel is needed to get access to the blob store, which - // may be needed to save to disk. - this.ensure_backend_kernel_setup(); - if (this.jupyter_kernel == null) { - // still not null? This would happen if no kernel is set at all, - // in which case it's OK that saving isn't possible. - throw Error("no kernel so cannot save"); - } - } - if (this.store.get("kernels") == null) { - await this.init_kernel_info(); - if (this.store.get("kernels") == null) { - // This should never happen, but maybe could in case of a very - // messed up compute environment where the kernelspecs can't be listed. - throw Error( - "kernel info not known and can't be determined, so can't save", - ); - } - } - dbg("going to try to save: getting ipynb object..."); - const blob_store = this.jupyter_kernel.get_blob_store(); - let ipynb = this.store.get_ipynb(blob_store); - if (this.store.get("kernel")) { - // if a kernel is set, check that it was sufficiently known that - // we can fill in data about it -- - // see https://github.com/sagemathinc/cocalc/issues/7286 - if (ipynb?.metadata?.kernelspec?.name == null) { - dbg("kernelspec not known -- try loading kernels again"); - await this.fetch_jupyter_kernels(); - // and again grab the ipynb - ipynb = this.store.get_ipynb(blob_store); - if (ipynb?.metadata?.kernelspec?.name == null) { - dbg("kernelspec STILL not known: metadata will be incomplete"); - } - } - } - dbg("got ipynb object"); - // We use json_stable (and indent 1) to be more diff friendly to user, - // and more consistent with official Jupyter. - const data = json_stable(ipynb, { space: 1 }); - if (data == null) { - dbg("failed -- ipynb not defined yet"); - throw Error("ipynb not defined yet; can't save"); - } - dbg("converted ipynb to stable JSON string", data?.length); - //dbg(`got string version '${data}'`) - try { - dbg("writing to disk..."); - await callback2(this._client.write_file, { - path: this.store.get("path"), - data, - }); - dbg("succeeded at saving"); - await this.set_last_ipynb_save(); - } catch (err) { - const e = `error writing file: ${err}`; - dbg(e); - throw Error(e); - } - }; - - private handle_all_cell_attachments() { - // Check if any cell attachments need to be loaded. - const cells = this.store.get("cells"); - cells?.forEach((cell) => { - this.handle_cell_attachments(cell); - }); - } - - private handle_cell_attachments(cell) { - if (this.jupyter_kernel == null) { - // can't do anything - return; - } - const dbg = this.dbg(`handle_cell_attachments(id=${cell.get("id")})`); - dbg(); - - const attachments = cell.get("attachments"); - if (attachments == null) return; // nothing to do - attachments.forEach(async (x, name) => { - if (x == null) return; - if (x.get("type") === "load") { - if (this.jupyter_kernel == null) return; // try later - // need to load from disk - this.set_cell_attachment(cell.get("id"), name, { - type: "loading", - value: null, - }); - let sha1: string; - try { - sha1 = await this.jupyter_kernel.load_attachment(x.get("value")); - } catch (err) { - this.set_cell_attachment(cell.get("id"), name, { - type: "error", - value: `${err}`, - }); - return; - } - this.set_cell_attachment(cell.get("id"), name, { - type: "sha1", - value: sha1, - }); - } - }); - } - - // handle_ipywidgets_state_change is called when the project ipywidgets_state - // object changes, e.g., in response to a user moving a slider in the browser. - // It crafts a comm message that is sent to the running Jupyter kernel telling - // it about this change by calling send_comm_message_to_kernel. - private handle_ipywidgets_state_change = (keys): void => { - if (this.is_closed()) { - return; - } - const dbg = this.dbg("handle_ipywidgets_state_change"); - dbg(keys); - if (this.jupyter_kernel == null) { - dbg("no kernel, so ignoring changes to ipywidgets"); - return; - } - if (this.syncdb.ipywidgets_state == null) { - throw Error("syncdb's ipywidgets_state must be defined!"); - } - for (const key of keys) { - const [, model_id, type] = JSON.parse(key); - dbg({ key, model_id, type }); - let data: any; - if (type === "value") { - const state = this.syncdb.ipywidgets_state.get_model_value(model_id); - // Saving the buffers on change is critical since otherwise this breaks: - // https://ipywidgets.readthedocs.io/en/latest/examples/Widget%20List.html#file-upload - // Note that stupidly the buffer (e.g., image upload) gets sent to the kernel twice. - // But it does work robustly, and the kernel and nodejs server processes next to each - // other so this isn't so bad. - const { buffer_paths, buffers } = - this.syncdb.ipywidgets_state.getKnownBuffers(model_id); - data = { method: "update", state, buffer_paths }; - this.jupyter_kernel.send_comm_message_to_kernel({ - msg_id: misc.uuid(), - target_name: "jupyter.widget", - comm_id: model_id, - data, - buffers, - }); - } else if (type === "buffers") { - // TODO: we MIGHT need implement this... but MAYBE NOT. An example where this seems like it might be - // required is by the file upload widget, but actually that just uses the value type above, since - // we explicitly fill in the widgets there; also there is an explicit comm upload message that - // the widget sends out that updates the buffer, and in send_comm_message_to_kernel in jupyter/kernel/kernel.ts - // when processing that message, we saves those buffers and make sure they are set in the - // value case above (otherwise they would get removed). - // https://ipywidgets.readthedocs.io/en/latest/examples/Widget%20List.html#file-upload - // which creates a buffer from the content of the file, then sends it to the backend, - // which sees a change and has to write that buffer to the kernel (here) so that - // the running python process can actually do something with the file contents (e.g., - // process data, save file to disk, etc). - // We need to be careful though to not send buffers to the kernel that the kernel sent us, - // since that would be a waste. - } else if (type === "state") { - // TODO: currently ignoring this, since it seems chatty and pointless, - // and could lead to race conditions probably with multiple users, etc. - // It happens right when the widget is created. - /* - const state = this.syncdb.ipywidgets_state.getModelSerializedState(model_id); - data = { method: "update", state }; - this.jupyter_kernel.send_comm_message_to_kernel( - misc.uuid(), - model_id, - data - ); - */ - } else { - const m = `Jupyter: unknown type '${type}'`; - console.warn(m); - dbg(m); - } - } - }; - - async process_comm_message_from_kernel(mesg: any): Promise { - const dbg = this.dbg("process_comm_message_from_kernel"); - // serializing the full message could cause enormous load on the server, since - // the mesg may contain large buffers. Only do for low level debugging! - // dbg(mesg); // EXTREME DANGER! - // This should be safe: - dbg(JSON.stringify(mesg.header)); - if (this.syncdb.ipywidgets_state == null) { - throw Error("syncdb's ipywidgets_state must be defined!"); - } - await this.syncdb.ipywidgets_state.process_comm_message_from_kernel(mesg); - } - - capture_output_message(mesg: any): boolean { - if (this.syncdb.ipywidgets_state == null) { - throw Error("syncdb's ipywidgets_state must be defined!"); - } - return this.syncdb.ipywidgets_state.capture_output_message(mesg); - } - - close_project_only() { - const dbg = this.dbg("close_project_only"); - dbg(); - if (this.run_all_loop) { - this.run_all_loop.close(); - delete this.run_all_loop; - } - // this stops the kernel and cleans everything up - // so no resources are wasted and next time starting - // is clean - (async () => { - try { - await removeJupyterRedux(this.store.get("path"), this.project_id); - } catch (err) { - dbg("WARNING -- issue removing jupyter redux", err); - } - })(); - - this.blobs?.close(); - } - // not actually async... - async signal(signal = "SIGINT"): Promise { + signal = async (signal = "SIGINT"): Promise => { this.jupyter_kernel?.signal(signal); - } - - handle_nbconvert_change(oldVal, newVal): void { - nbconvertChange(this, oldVal?.toJS(), newVal?.toJS()); - } - - // Handle transient cell messages. - handleTransientUpdate = (mesg) => { - const display_id = mesg.content?.transient?.display_id; - if (!display_id) { - return false; - } - - let matched = false; - // are there any transient outputs in the entire document that - // have this display_id? search to find them. - // TODO: we could use a clever data structure to make - // this faster and more likely to have bugs. - const cells = this.syncdb.get({ type: "cell" }); - for (let cell of cells) { - let output = cell.get("output"); - if (output != null) { - for (const [n, val] of output) { - if (val.getIn(["transient", "display_id"]) == display_id) { - // found a match -- replace it - output = output.set(n, immutable.fromJS(mesg.content)); - this.syncdb.set({ type: "cell", id: cell.get("id"), output }); - matched = true; - } - } - } - } - if (matched) { - this.syncdb.commit(); - } - }; - - getComputeServers = () => { - // we don't bother worrying about freeing this since it is only - // run in the project or compute server, which needs the underlying - // dkv for its entire lifetime anyways. - if (this.computeServers == null) { - this.computeServers = computeServerManager({ - project_id: this.project_id, - }); - } - return this.computeServers; - }; - - getComputeServerIdSync = (): number => { - const c = this.getComputeServers(); - return c.get(this.syncdb.path) ?? 0; - }; - - getComputeServerId = async (): Promise => { - const c = this.getComputeServers(); - return (await c.getServerIdForPath(this.syncdb.path)) ?? 0; }; } diff --git a/src/packages/project/conat/jupyter.ts b/src/packages/project/conat/jupyter.ts index 1405684198..80f8482acc 100644 --- a/src/packages/project/conat/jupyter.ts +++ b/src/packages/project/conat/jupyter.ts @@ -1,3 +1,22 @@ +/* + +To run just this for a project in a console, from the browser, terminate the jupyter server by running this +in your browser with the project open: + + await cc.client.conat_client.projectApi(cc.current()).system.terminate({service:'jupyter'}) + +As explained in packages/project/conat/api/index.ts setup your environment as for the project. + +Then run this code in nodejs: + + require("@cocalc/project/conat/jupyter").init() + + + + +*/ + + import { jupyterRun } from "@cocalc/project/conat/api/editor"; import { outputHandler } from "@cocalc/jupyter/control"; import { jupyterServer } from "@cocalc/conat/project/jupyter/run-code"; diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 868eecdf52..22bd8bedfd 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -1299,7 +1299,9 @@ export class SyncDoc extends EventEmitter { await this.readFile(); if (firstLoad) { dbg("emitting first-load event"); - // this event is emited the first time the document is ever loaded from disk. + // this event is emited the first time the document is ever + // loaded from disk. It's used, e.g., for notebook "trust" state, + // so important from a security POV. this.emit("first-load"); } dbg("loaded"); From 7cca2c1a75253a9d63b24c50b080d1e08f5baa23 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 1 Aug 2025 18:00:58 +0000 Subject: [PATCH 156/798] refactor: move the jupyter functionality from project api.editor to new api.jupyter --- .../test/project/jupyter/run-code.test.ts | 12 +++---- src/packages/conat/project/api/editor.ts | 28 ---------------- src/packages/conat/project/api/index.ts | 3 ++ src/packages/conat/project/api/jupyter.ts | 32 +++++++++++++++++++ .../conat/project/jupyter/run-code.ts | 12 +++---- .../frontend/jupyter/browser-actions.ts | 4 +-- src/packages/frontend/jupyter/kernelspecs.ts | 2 +- src/packages/frontend/jupyter/logo.tsx | 2 +- .../frontend/project/websocket/api.ts | 6 ++-- src/packages/jupyter/control.ts | 20 ++++++------ src/packages/project/conat/api/editor.ts | 32 ------------------- src/packages/project/conat/api/index.ts | 2 ++ src/packages/project/conat/jupyter.ts | 5 ++- 13 files changed, 68 insertions(+), 92 deletions(-) create mode 100644 src/packages/conat/project/api/jupyter.ts diff --git a/src/packages/backend/conat/test/project/jupyter/run-code.test.ts b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts index 95dcb1400e..460442324b 100644 --- a/src/packages/backend/conat/test/project/jupyter/run-code.test.ts +++ b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts @@ -35,7 +35,7 @@ describe("create very simple mocked jupyter runner and test evaluating code", () const project_id = uuid(); it("create jupyter code run server", () => { // running code with this just results in two responses: the path and the cells - async function jupyterRun({ path, cells }) { + async function run({ path, cells }) { async function* runner() { yield { path, id: "0" }; yield { cells, id: "0" }; @@ -43,7 +43,7 @@ describe("create very simple mocked jupyter runner and test evaluating code", () return runner(); } - server = jupyterServer({ client: client1, project_id, jupyterRun }); + server = jupyterServer({ client: client1, project_id, run }); }); let client; @@ -120,7 +120,7 @@ describe("create simple mocked jupyter runner that does actually eval an express const compute_server_id = 3; it("create jupyter code run server", () => { // running code with this just results in two responses: the path and the cells - async function jupyterRun({ cells }) { + async function run({ cells }) { async function* runner() { for (const { id, input } of cells) { yield { id, output: eval(input) }; @@ -132,7 +132,7 @@ describe("create simple mocked jupyter runner that does actually eval an express server = jupyterServer({ client: client1, project_id, - jupyterRun, + run, compute_server_id, }); }); @@ -197,7 +197,7 @@ describe("create mocked jupyter runner that does failover to backend output mana let handler: any = null; it("create jupyter code run server that also takes as long as the output to run", () => { - async function jupyterRun({ cells }) { + async function run({ cells }) { async function* runner() { for (const { id, input } of cells) { const output = eval(input); @@ -232,7 +232,7 @@ describe("create mocked jupyter runner that does failover to backend output mana server = jupyterServer({ client: client1, project_id, - jupyterRun, + run, outputHandler, }); }); diff --git a/src/packages/conat/project/api/editor.ts b/src/packages/conat/project/api/editor.ts index e17d597e8a..66750e2c51 100644 --- a/src/packages/conat/project/api/editor.ts +++ b/src/packages/conat/project/api/editor.ts @@ -1,19 +1,8 @@ -import type { NbconvertParams } from "@cocalc/util/jupyter/types"; -import type { RunNotebookOptions } from "@cocalc/util/jupyter/nbgrader-types"; import type { Options as FormatterOptions } from "@cocalc/util/code-formatter"; -import type { KernelSpec } from "@cocalc/util/jupyter/types"; export const editor = { newFile: true, - jupyterStart: true, - jupyterStop: true, - jupyterStripNotebook: true, - jupyterNbconvert: true, - jupyterRunNotebook: true, - jupyterKernelLogo: true, - jupyterKernels: true, - formatString: true, printSageWS: true, @@ -41,23 +30,6 @@ export interface Editor { // context of our editors. newFile: (path: string) => Promise; - jupyterStripNotebook: (path_ipynb: string) => Promise; - - // path = the syncdb path (not *.ipynb) - jupyterStart: (path: string) => Promise; - jupyterStop: (path: string) => Promise; - - jupyterNbconvert: (opts: NbconvertParams) => Promise; - - jupyterRunNotebook: (opts: RunNotebookOptions) => Promise; - - jupyterKernelLogo: ( - kernelName: string, - opts?: { noCache?: boolean }, - ) => Promise<{ filename: string; base64: string }>; - - jupyterKernels: (opts?: { noCache?: boolean }) => Promise; - // returns formatted version of str. formatString: (opts: { str: string; diff --git a/src/packages/conat/project/api/index.ts b/src/packages/conat/project/api/index.ts index 694229b2bc..96febf9172 100644 --- a/src/packages/conat/project/api/index.ts +++ b/src/packages/conat/project/api/index.ts @@ -1,17 +1,20 @@ import { type System, system } from "./system"; import { type Editor, editor } from "./editor"; +import { type Jupyter, jupyter } from "./jupyter"; import { type Sync, sync } from "./sync"; import { handleErrorMessage } from "@cocalc/conat/util"; export interface ProjectApi { system: System; editor: Editor; + jupyter: Jupyter; sync: Sync; } const ProjectApiStructure = { system, editor, + jupyter, sync, } as const; diff --git a/src/packages/conat/project/api/jupyter.ts b/src/packages/conat/project/api/jupyter.ts new file mode 100644 index 0000000000..20ec07b4bf --- /dev/null +++ b/src/packages/conat/project/api/jupyter.ts @@ -0,0 +1,32 @@ +import type { NbconvertParams } from "@cocalc/util/jupyter/types"; +import type { RunNotebookOptions } from "@cocalc/util/jupyter/nbgrader-types"; +import type { KernelSpec } from "@cocalc/util/jupyter/types"; + +export const jupyter = { + start: true, + stop: true, + stripNotebook: true, + nbconvert: true, + runNotebook: true, + kernelLogo: true, + kernels: true, +}; + +export interface Jupyter { + stripNotebook: (path_ipynb: string) => Promise; + + // path = the syncdb path (not *.ipynb) + start: (path: string) => Promise; + stop: (path: string) => Promise; + + nbconvert: (opts: NbconvertParams) => Promise; + + runNotebook: (opts: RunNotebookOptions) => Promise; + + kernelLogo: ( + kernelName: string, + opts?: { noCache?: boolean }, + ) => Promise<{ filename: string; base64: string }>; + + kernels: (opts?: { noCache?: boolean }) => Promise; +} diff --git a/src/packages/conat/project/jupyter/run-code.ts b/src/packages/conat/project/jupyter/run-code.ts index 0417350eef..ef78773bf9 100644 --- a/src/packages/conat/project/jupyter/run-code.ts +++ b/src/packages/conat/project/jupyter/run-code.ts @@ -79,9 +79,9 @@ export function jupyterServer({ client, project_id, compute_server_id = 0, - // jupyterRun takes a path and cells to run and returns an async iterator + // run takes a path and cells to run and returns an async iterator // over the outputs. - jupyterRun, + run, // outputHandler takes a path and returns an OutputHandler, which can be // used to process the output and include it in the notebook. It is used // as a fallback in case the client that initiated running cells is @@ -91,7 +91,7 @@ export function jupyterServer({ client: ConatClient; project_id: string; compute_server_id?: number; - jupyterRun: JupyterCodeRunner; + run: JupyterCodeRunner; outputHandler?: CreateOutputHandler; }) { const subject = getSubject({ project_id, compute_server_id }); @@ -113,7 +113,7 @@ export function jupyterServer({ mesg.respondSync(null); await handleRequest({ socket, - jupyterRun, + run, outputHandler, path, cells, @@ -142,13 +142,13 @@ export function jupyterServer({ async function handleRequest({ socket, - jupyterRun, + run, outputHandler, path, cells, noHalt, }) { - const runner = await jupyterRun({ path, cells, noHalt }); + const runner = await run({ path, cells, noHalt }); const output: OutputMessage[] = []; const throttle = new Throttle(MAX_MSGS_PER_SECOND); diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 0415bd3d37..a2341cf7f6 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -1464,7 +1464,7 @@ export class JupyterActions extends JupyterActions0 { } try { const api = await this.conatApi(); - await api.editor.jupyterStart(this.syncdbPath); + await api.jupyter.start(this.syncdbPath); return true; } catch (err) { console.log("failed to initialize ", this.path, err); @@ -1477,7 +1477,7 @@ export class JupyterActions extends JupyterActions0 { stopBackend = async () => { const api = await this.conatApi(); - await api.editor.jupyterStop(this.syncdbPath); + await api.jupyter.stop(this.syncdbPath); }; getOutputHandler = (cell) => { diff --git a/src/packages/frontend/jupyter/kernelspecs.ts b/src/packages/frontend/jupyter/kernelspecs.ts index 596e55b57a..5a1db32a2b 100644 --- a/src/packages/frontend/jupyter/kernelspecs.ts +++ b/src/packages/frontend/jupyter/kernelspecs.ts @@ -38,7 +38,7 @@ const getKernelSpec = reuseInFlight( compute_server_id, timeout: 7500, }); - const spec = await api.editor.jupyterKernels(); + const spec = await api.jupyter.kernels(); cache.set(key, spec); return spec; }, diff --git a/src/packages/frontend/jupyter/logo.tsx b/src/packages/frontend/jupyter/logo.tsx index fe26b5f15e..963a47650e 100644 --- a/src/packages/frontend/jupyter/logo.tsx +++ b/src/packages/frontend/jupyter/logo.tsx @@ -113,7 +113,7 @@ async function getLogo({ return cache[key]; } const api = client.conat_client.projectApi({ project_id }); - const { filename, base64 } = await api.editor.jupyterKernelLogo(kernel, { + const { filename, base64 } = await api.jupyter.kernelLogo(kernel, { noCache, }); if (!filename || !base64) { diff --git a/src/packages/frontend/project/websocket/api.ts b/src/packages/frontend/project/websocket/api.ts index 31dced4922..4fad962002 100644 --- a/src/packages/frontend/project/websocket/api.ts +++ b/src/packages/frontend/project/websocket/api.ts @@ -306,7 +306,7 @@ export class API { compute_server_id ?? this.getComputeServerId(opts.args[0]), timeout: (opts.timeout ?? 60) * 1000 + 5000, }); - return await api.editor.jupyterNbconvert(opts); + return await api.jupyter.nbconvert(opts); }; // Get contents of an ipynb file, but with output and attachments removed (to save space) @@ -318,7 +318,7 @@ export class API { compute_server_id: compute_server_id ?? this.getComputeServerId(ipynb_path), }); - return await api.editor.jupyterStripNotebook(ipynb_path); + return await api.jupyter.stripNotebook(ipynb_path); }; // Run the notebook filling in the output of all cells, then return the @@ -338,7 +338,7 @@ export class API { compute_server_id, timeout: 60 + 2 * max_total_time_ms, }); - return await api.editor.jupyterRunNotebook(opts); + return await api.jupyter.runNotebook(opts); }; // Get the x11 *channel* for the given '.x11' path. diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts index 1020c94ec7..bd80e795b4 100644 --- a/src/packages/jupyter/control.ts +++ b/src/packages/jupyter/control.ts @@ -14,7 +14,7 @@ const logger = getLogger("jupyter:control"); const sessions: { [path: string]: { syncdb: SyncDB; actions; store } } = {}; let project_id: string = ""; -export function jupyterStart({ +export function start({ path, client, project_id: project_id0, @@ -27,10 +27,10 @@ export function jupyterStart({ }) { project_id = project_id0; if (sessions[path] != null) { - logger.debug("jupyterStart: ", path, " - already running"); + logger.debug("start: ", path, " - already running"); return; } - logger.debug("jupyterStart: ", path, " - starting it"); + logger.debug("start: ", path, " - starting it"); const syncdb = new SyncDB({ ...SYNCDB_OPTIONS, project_id, @@ -41,22 +41,22 @@ export function jupyterStart({ // [ ] TODO: some way to convey this to clients (?) syncdb.on("error", (err) => { logger.debug(`syncdb error -- ${err}`, path); - jupyterStop({ path }); + stop({ path }); }); syncdb.on("close", () => { - jupyterStop({ path }); + stop({ path }); }); const { actions, store } = initJupyterRedux(syncdb, client); sessions[path] = { syncdb, actions, store }; } -export function jupyterStop({ path }: { path: string }) { +export function stop({ path }: { path: string }) { const session = sessions[path]; if (session == null) { - logger.debug("jupyterStop: ", path, " - not running"); + logger.debug("stop: ", path, " - not running"); } else { const { syncdb } = session; - logger.debug("jupyterStop: ", path, " - stopping it"); + logger.debug("stop: ", path, " - stopping it"); syncdb.close(); delete sessions[path]; const path_ipynb = original_path(path); @@ -65,8 +65,8 @@ export function jupyterStop({ path }: { path: string }) { } // Returns async iterator over outputs -export async function jupyterRun({ path, cells, noHalt }: RunOptions) { - logger.debug("jupyterRun", { path, noHalt }); +export async function run({ path, cells, noHalt }: RunOptions) { + logger.debug("run:", { path, noHalt }); const session = sessions[path]; if (session == null) { diff --git a/src/packages/project/conat/api/editor.ts b/src/packages/project/conat/api/editor.ts index c858c7d124..b03f0c4ea1 100644 --- a/src/packages/project/conat/api/editor.ts +++ b/src/packages/project/conat/api/editor.ts @@ -1,9 +1,4 @@ -export { jupyter_strip_notebook as jupyterStripNotebook } from "@cocalc/jupyter/nbgrader/jupyter-parse"; -export { jupyter_run_notebook as jupyterRunNotebook } from "@cocalc/jupyter/nbgrader/jupyter-run"; -export { nbconvert as jupyterNbconvert } from "../../jupyter/convert"; export { formatString } from "../../formatters"; -export { logo as jupyterKernelLogo } from "@cocalc/jupyter/kernel/logo"; -export { get_kernel_data as jupyterKernels } from "@cocalc/jupyter/kernel/kernel-data"; export { newFile } from "@cocalc/backend/misc/new-file"; import { printSageWS as printSageWS0 } from "@cocalc/project/print_to_pdf"; @@ -34,30 +29,3 @@ export async function printSageWS(opts): Promise { } export { createTerminalService } from "@cocalc/project/conat/terminal"; - -import { getClient } from "@cocalc/project/client"; -import { project_id } from "@cocalc/project/data"; -import * as control from "@cocalc/jupyter/control"; -import { SandboxedFilesystem } from "@cocalc/backend/files/sandbox"; - -export async function jupyterStart(path: string) { - const fs = new SandboxedFilesystem(process.env.HOME ?? "/tmp", { - unsafeMode: true, - }); - await control.jupyterStart({ project_id, path, client: getClient(), fs }); -} - -// IMPORTANT: jupyterRun is NOT used directly by the API, but instead by packages/project/conat/jupyter.ts -// It is convenient to have it here so it can call jupyterStart above, etc. The reason is because -// this returns an async iterator managed using a dedicated socket, and the api is request/response.. -export async function jupyterRun(opts: { - path: string; - cells: { id: string; input: string }[]; -}) { - await jupyterStart(opts.path); - return await control.jupyterRun(opts); -} - -export async function jupyterStop(path: string) { - await control.jupyterStop({ path }); -} diff --git a/src/packages/project/conat/api/index.ts b/src/packages/project/conat/api/index.ts index ef45cb1eb8..7133583609 100644 --- a/src/packages/project/conat/api/index.ts +++ b/src/packages/project/conat/api/index.ts @@ -152,11 +152,13 @@ async function handleApiRequest(request, mesg) { import * as system from "./system"; import * as editor from "./editor"; +import * as jupyter from "./jupyter"; import * as sync from "./sync"; export const projectApi: ProjectApi = { system, editor, + jupyter, sync, }; diff --git a/src/packages/project/conat/jupyter.ts b/src/packages/project/conat/jupyter.ts index 80f8482acc..82a04312d3 100644 --- a/src/packages/project/conat/jupyter.ts +++ b/src/packages/project/conat/jupyter.ts @@ -16,8 +16,7 @@ Then run this code in nodejs: */ - -import { jupyterRun } from "@cocalc/project/conat/api/editor"; +import { run } from "@cocalc/project/conat/api/jupyter"; import { outputHandler } from "@cocalc/jupyter/control"; import { jupyterServer } from "@cocalc/conat/project/jupyter/run-code"; import { connectToConat } from "@cocalc/project/conat/connection"; @@ -34,7 +33,7 @@ export function init() { client, project_id, compute_server_id, - jupyterRun, + run, outputHandler, }); } From d3c1d2dc8d5d3b96b6c37032b80e82c7c4a2fe98 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 1 Aug 2025 20:01:45 +0000 Subject: [PATCH 157/798] jupyter: rewriting to move more control/state to frontend --- src/packages/conat/project/api/jupyter.ts | 13 + .../frontend/jupyter/browser-actions.ts | 360 ++++++++++++++++-- src/packages/jupyter/control.ts | 98 +++-- src/packages/jupyter/kernel/kernel.ts | 21 +- src/packages/jupyter/pool/pool.ts | 4 +- src/packages/jupyter/redux/actions.ts | 311 +-------------- src/packages/jupyter/redux/project-actions.ts | 8 +- src/packages/jupyter/redux/run-all-loop.ts | 63 --- .../jupyter/types/project-interface.ts | 4 +- src/packages/project/conat/api/jupyter.ts | 50 +++ src/packages/util/jupyter/names.ts | 23 +- 11 files changed, 501 insertions(+), 454 deletions(-) delete mode 100644 src/packages/jupyter/redux/run-all-loop.ts create mode 100644 src/packages/project/conat/api/jupyter.ts diff --git a/src/packages/conat/project/api/jupyter.ts b/src/packages/conat/project/api/jupyter.ts index 20ec07b4bf..3defeb5b57 100644 --- a/src/packages/conat/project/api/jupyter.ts +++ b/src/packages/conat/project/api/jupyter.ts @@ -10,8 +10,12 @@ export const jupyter = { runNotebook: true, kernelLogo: true, kernels: true, + introspect: true, + signal: true, }; +// In the functions below path can be either the .ipynb or the .sage-jupyter2 path, and +// the correct backend kernel will get found/created automatically. export interface Jupyter { stripNotebook: (path_ipynb: string) => Promise; @@ -29,4 +33,13 @@ export interface Jupyter { ) => Promise<{ filename: string; base64: string }>; kernels: (opts?: { noCache?: boolean }) => Promise; + + introspect: (opts: { + path: string; + code: string; + cursor_pos: number; + detail_level: 0 | 1; + }) => Promise; + + signal: (opts: { path: string; signal: string }) => Promise; } diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index a2341cf7f6..7601684e60 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -18,7 +18,6 @@ import { get_local_storage, set_local_storage, } from "@cocalc/frontend/misc/local-storage"; -import track from "@cocalc/frontend/user-tracking"; import { webapp_client } from "@cocalc/frontend/webapp-client"; import { JupyterActions as JupyterActions0 } from "@cocalc/jupyter/redux/actions"; import { CellToolbarName } from "@cocalc/jupyter/types"; @@ -27,6 +26,7 @@ import { base64ToBuffer, bufferToBase64 } from "@cocalc/util/base64"; import { Config as FormatterConfig, Syntax } from "@cocalc/util/code-formatter"; import { closest_kernel_match, + cmp, field_cmp, from_json, history_path, @@ -64,6 +64,11 @@ import { } from "@cocalc/conat/project/jupyter/run-code"; import { OutputHandler } from "@cocalc/jupyter/execute/output-handler"; import { throttle } from "lodash"; +import { + char_idx_to_js_idx, + codemirror_to_jupyter_pos, + js_idx_to_char_idx, +} from "@cocalc/jupyter/util/misc"; const OUTPUT_FPS = 29; @@ -77,6 +82,8 @@ export class JupyterActions extends JupyterActions0 { private account_change_editor_settings: any; private update_keyboard_shortcuts: any; public syncdbPath: string; + private last_cursor_move_time: Date = new Date(0); + private _introspect_request?: any; protected init2(): void { this.syncdbPath = syncdbPath(this.path); @@ -125,15 +132,15 @@ export class JupyterActions extends JupyterActions0 { this.fetch_jupyter_kernels(); // Load kernel (once ipynb file loads). - (async () => { - await this.set_kernel_after_load(); - if (!this.store) return; - track("jupyter", { - kernel: this.store.get("kernel"), - project_id: this.project_id, - path: this.path, - }); - })(); + // (async () => { + // await this.set_kernel_after_load(); + // if (!this.store) return; + // track("jupyter", { + // kernel: this.store.get("kernel"), + // project_id: this.project_id, + // path: this.path, + // }); + // })(); // nbgrader support this.nbgrader_actions = new NBGraderActions(this, this.redux); @@ -1132,22 +1139,22 @@ export class JupyterActions extends JupyterActions0 { }); }; - private set_kernel_after_load = async (): Promise => { - // Browser Client: Wait until the .ipynb file has actually been parsed into - // the (hidden, e.g. .a.ipynb.sage-jupyter2) syncdb file, - // then set the kernel, if necessary. - try { - await this.syncdb.wait((s) => !!s.get_one({ type: "file" }), 600); - } catch (err) { - if (this._state != "ready") { - // Probably user just closed the notebook before it finished - // loading, so we don't need to set the kernel. - return; - } - throw Error("error waiting for ipynb file to load"); - } - this._syncdb_init_kernel(); - }; + // private set_kernel_after_load = async (): Promise => { + // // Browser Client: Wait until the .ipynb file has actually been parsed into + // // the (hidden, e.g. .a.ipynb.sage-jupyter2) syncdb file, + // // then set the kernel, if necessary. + // try { + // await this.syncdb.wait((s) => !!s.get_one({ type: "file" }), 600); + // } catch (err) { + // if (this._state != "ready") { + // // Probably user just closed the notebook before it finished + // // loading, so we don't need to set the kernel. + // return; + // } + // throw Error("error waiting for ipynb file to load"); + // } + // this._syncdb_init_kernel(); + // }; private _syncdb_init_kernel = (): void => { // console.log("jupyter::_syncdb_init_kernel", this.store.get("kernel")); @@ -1534,7 +1541,7 @@ export class JupyterActions extends JupyterActions0 { private jupyterClient?; private runQueue: any[] = []; private runningNow = false; - async runCells(ids: string[], opts: { noHalt?: boolean } = {}) { + runCells = async (ids: string[], opts: { noHalt?: boolean } = {}) => { if (this.store?.get("read_only")) { return; } @@ -1552,6 +1559,7 @@ export class JupyterActions extends JupyterActions0 { // [ ] **TODO: Must invalidate this when compute server changes!!!!!** // and const compute_server_id = await this.getComputeServerId(); + if (this.isClosed()) return; this.jupyterClient = jupyterClient({ path: this.syncdbPath, client: webapp_client.conat_client.conat(), @@ -1604,9 +1612,11 @@ export class JupyterActions extends JupyterActions0 { cells.sort(field_cmp("pos")); const runner = await client.run(cells, opts); + if (this.isClosed()) return; let handler: null | OutputHandler = null; let id: null | string = null; for await (const mesgs of runner) { + if (this.isClosed()) return; for (const mesg of mesgs) { if (!opts.noHalt && mesg.msg_type == "error") { this.clearRunQueue(); @@ -1630,15 +1640,309 @@ export class JupyterActions extends JupyterActions0 { } } handler?.done(); - this.save_asap(); + this.syncdb.save(); + setTimeout(() => { + if (!this.isClosed()) { + this.syncdb.save(); + } + }, 1000); } catch (err) { console.warn("runCells", err); } finally { + if (this.isClosed()) return; this.runningNow = false; if (this.runQueue.length > 0) { const [ids, opts] = this.runQueue.shift(); this.runCells(ids, opts); } } + }; + + is_introspecting(): boolean { + const actions = this.getFrameActions(); + return actions?.store?.get("introspect") != null; + } + + introspect_close = () => { + if (this.is_introspecting()) { + this.getFrameActions()?.setState({ introspect: undefined }); + } + }; + + introspect_at_pos = async ( + code: string, + detail_level: 0 | 1 = 0, + pos: { ch: number; line: number }, + ): Promise => { + if (code === "") return; // no-op if there is no code (should never happen) + await this.introspect( + code, + detail_level, + codemirror_to_jupyter_pos(code, pos), + ); + }; + + introspect = async ( + code: string, + detail_level: 0 | 1, + cursor_pos?: number, + ): Promise | undefined> => { + const req = (this._introspect_request = + (this._introspect_request != null ? this._introspect_request : 0) + 1); + + if (cursor_pos == null) { + cursor_pos = code.length; + } + cursor_pos = js_idx_to_char_idx(cursor_pos, code); + + let introspect; + try { + const api = await this.conatApi(); + introspect = await api.jupyter.introspect({ + path: this.path, + code, + cursor_pos, + detail_level, + }); + if (introspect.status !== "ok") { + introspect = { error: "completion failed" }; + } + delete introspect.status; + } catch (err) { + introspect = { error: err }; + } + if (this._introspect_request > req) return; + const i = fromJS(introspect); + this.getFrameActions()?.setState({ + introspect: i, + }); + return introspect; // convenient / useful, e.g., for use by whiteboard. + }; + + clear_introspect = (): void => { + this._introspect_request = + (this._introspect_request != null ? this._introspect_request : 0) + 1; + this.getFrameActions()?.setState({ introspect: undefined }); + }; + + // Attempt to fetch completions for give code and cursor_pos + // If successful, the completions are put in store.get('completions') and looks like + // this (as an immutable map): + // cursor_end : 2 + // cursor_start : 0 + // matches : ['the', 'completions', ...] + // status : "ok" + // code : code + // cursor_pos : cursor_pos + // + // If not successful, result is: + // status : "error" + // code : code + // cursor_pos : cursor_pos + // error : 'an error message' + // + // Only the most recent fetch has any impact, and calling + // clear_complete() ensures any fetch made before that + // is ignored. + + // Returns true if a dialog with options appears, and false otherwise. + complete = async ( + code: string, + pos?: { line: number; ch: number } | number, + id?: string, + offset?: any, + ): Promise => { + let cursor_pos; + const req = (this._complete_request = + (this._complete_request != null ? this._complete_request : 0) + 1); + + this.setState({ complete: undefined }); + + // pos can be either a {line:?, ch:?} object as in codemirror, + // or a number. + if (pos == null || typeof pos == "number") { + cursor_pos = pos; + } else { + cursor_pos = codemirror_to_jupyter_pos(code, pos); + } + cursor_pos = js_idx_to_char_idx(cursor_pos, code); + + const start = new Date(); + let complete; + try { + complete = await this.api().complete({ + code, + cursor_pos, + }); + } catch (err) { + if (this._complete_request > req) return false; + this.setState({ complete: { error: err } }); + // no op for now... + throw Error(`ignore -- ${err}`); + //return false; + } + + if (this.last_cursor_move_time >= start) { + // see https://github.com/sagemathinc/cocalc/issues/3611 + throw Error("ignore"); + //return false; + } + if (this._complete_request > req) { + // future completion or clear happened; so ignore this result. + throw Error("ignore"); + //return false; + } + + if (complete.status !== "ok") { + this.setState({ + complete: { + error: complete.error ? complete.error : "completion failed", + }, + }); + return false; + } + + if (complete.matches == 0) { + return false; + } + + delete complete.status; + complete.base = code; + complete.code = code; + complete.pos = char_idx_to_js_idx(cursor_pos, code); + complete.cursor_start = char_idx_to_js_idx(complete.cursor_start, code); + complete.cursor_end = char_idx_to_js_idx(complete.cursor_end, code); + complete.id = id; + // Set the result so the UI can then react to the change. + if (offset != null) { + complete.offset = offset; + } + // For some reason, sometimes complete.matches are not unique, which is annoying/confusing, + // and breaks an assumption in our react code too. + // I think the reason is e.g., a filename and a variable could be the same. We're not + // worrying about that now. + complete.matches = Array.from(new Set(complete.matches)); + // sort in a way that matches how JupyterLab sorts completions, which + // is case insensitive with % magics at the bottom + complete.matches.sort((x, y) => { + const c = cmp(getCompletionGroup(x), getCompletionGroup(y)); + if (c) { + return c; + } + return cmp(x.toLowerCase(), y.toLowerCase()); + }); + const i_complete = fromJS(complete); + if (complete.matches && complete.matches.length === 1 && id != null) { + // special case -- a unique completion and we know id of cell in which completing is given. + this.select_complete(id, complete.matches[0], i_complete); + return false; + } else { + this.setState({ complete: i_complete }); + return true; + } + }; + + clear_complete = (): void => { + this._complete_request = + (this._complete_request != null ? this._complete_request : 0) + 1; + this.setState({ complete: undefined }); + }; + + public select_complete( + id: string, + item: string, + complete?: Map, + ): void { + if (complete == null) { + complete = this.store.get("complete"); + } + this.clear_complete(); + if (complete == null) { + return; + } + const input = complete.get("code"); + if (input != null && complete.get("error") == null) { + const starting = input.slice(0, complete.get("cursor_start")); + const ending = input.slice(complete.get("cursor_end")); + const new_input = starting + item + ending; + const base = complete.get("base"); + this.complete_cell(id, base, new_input); + } + } + + complete_cell(id: string, base: string, new_input: string): void { + this.merge_cell_input(id, base, new_input); + } + + public set_cursor_locs(locs: any[] = [], side_effect: boolean = false): void { + this.last_cursor_move_time = new Date(); + if (this.syncdb == null) { + // syncdb not always set -- https://github.com/sagemathinc/cocalc/issues/2107 + return; + } + if (locs.length === 0) { + // don't remove on blur -- cursor will fade out just fine + return; + } + this._cursor_locs = locs; // remember our own cursors for splitting cell + this.syncdb.set_cursor_locs(locs, side_effect); + } + + async signal(signal = "SIGINT"): Promise { + const api = await this.conatApi(); + try { + await api.jupyter.signal({ path: this.path, signal }); + } catch (err) { + this.set_error(err); + } + } + + // Kill the running kernel and does NOT start it up again. + halt = reuseInFlight(async (): Promise => { + if (this.restartKernelOnClose != null && this.jupyter_kernel != null) { + this.jupyter_kernel.removeListener("closed", this.restartKernelOnClose); + delete this.restartKernelOnClose; + } + this.clear_all_cell_run_state(); + await this.signal("SIGKILL"); + // Wait a little, since SIGKILL has to really happen on backend, + // and server has to respond and change state. + const not_running = (s): boolean => { + if (this._state === "closed") return true; + const t = s.get_one({ type: "settings" }); + return t != null && t.get("backend_state") != "running"; + }; + try { + await this.syncdb.wait(not_running, 30); + // worked -- and also no need to show "kernel got killed" message since this was intentional. + this.set_error(""); + } catch (err) { + // failed + this.set_error(err); + } + }); + + restart = reuseInFlight(async (): Promise => { + await this.halt(); + if (this.is_closed()) return; + this.clear_all_cell_run_state(); + }); + + shutdown = reuseInFlight(async (): Promise => { + if (this.is_closed()) return; + await this.signal("SIGKILL"); + if (this.is_closed()) return; + this.clear_all_cell_run_state(); + }); +} + +function getCompletionGroup(x: string): number { + switch (x[0]) { + case "_": + return 1; + case "%": + return 2; + default: + return 0; } } diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts index bd80e795b4..8efeddd0ac 100644 --- a/src/packages/jupyter/control.ts +++ b/src/packages/jupyter/control.ts @@ -3,21 +3,27 @@ import { SYNCDB_OPTIONS } from "@cocalc/jupyter/redux/sync"; import { type Filesystem } from "@cocalc/conat/files/fs"; import { getLogger } from "@cocalc/backend/logger"; import { initJupyterRedux, removeJupyterRedux } from "@cocalc/jupyter/kernel"; -import { original_path } from "@cocalc/util/misc"; +import { syncdbPath, ipynbPath } from "@cocalc/util/jupyter/names"; import { once } from "@cocalc/util/async-utils"; import { OutputHandler } from "@cocalc/jupyter/execute/output-handler"; import { throttle } from "lodash"; import { type RunOptions } from "@cocalc/conat/project/jupyter/run-code"; +import { type JupyterActions } from "@cocalc/jupyter/redux/project-actions"; const logger = getLogger("jupyter:control"); -const sessions: { [path: string]: { syncdb: SyncDB; actions; store } } = {}; +const jupyterActions: { [ipynbPath: string]: JupyterActions } = {}; + +export function isRunning(path): boolean { + return jupyterActions[ipynbPath(path)] != null; +} + let project_id: string = ""; export function start({ path, - client, project_id: project_id0, + client, fs, }: { path: string; @@ -25,42 +31,40 @@ export function start({ project_id: string; fs: Filesystem; }) { - project_id = project_id0; - if (sessions[path] != null) { - logger.debug("start: ", path, " - already running"); + if (isRunning(path)) { return; } + project_id = project_id0; logger.debug("start: ", path, " - starting it"); const syncdb = new SyncDB({ ...SYNCDB_OPTIONS, project_id, - path, + path: syncdbPath(path), client, fs, }); - // [ ] TODO: some way to convey this to clients (?) syncdb.on("error", (err) => { + // [ ] TODO: some way to convey this to clients (?) logger.debug(`syncdb error -- ${err}`, path); stop({ path }); }); - syncdb.on("close", () => { + syncdb.once("closed", () => { stop({ path }); }); - const { actions, store } = initJupyterRedux(syncdb, client); - sessions[path] = { syncdb, actions, store }; + const { actions } = initJupyterRedux(syncdb, client); + jupyterActions[ipynbPath(path)] = actions; } export function stop({ path }: { path: string }) { - const session = sessions[path]; - if (session == null) { + const actions = jupyterActions[ipynbPath(path)]; + if (actions == null) { logger.debug("stop: ", path, " - not running"); } else { - const { syncdb } = session; + delete jupyterActions[ipynbPath(path)]; + const { syncdb } = actions; logger.debug("stop: ", path, " - stopping it"); syncdb.close(); - delete sessions[path]; - const path_ipynb = original_path(path); - removeJupyterRedux(path_ipynb, project_id); + removeJupyterRedux(ipynbPath(path), project_id); } } @@ -68,42 +72,42 @@ export function stop({ path }: { path: string }) { export async function run({ path, cells, noHalt }: RunOptions) { logger.debug("run:", { path, noHalt }); - const session = sessions[path]; - if (session == null) { - throw Error(`${path} not running`); + const actions = jupyterActions[ipynbPath(path)]; + if (actions == null) { + throw Error(`${ipynbPath(path)} not running`); } - const { syncdb, actions } = session; - if (syncdb.isClosed()) { + if (actions.syncdb.isClosed()) { // shouldn't be possible throw Error("syncdb is closed"); } - if (!syncdb.isReady()) { + if (!actions.syncdb.isReady()) { logger.debug("jupyterRun: waiting until ready"); - await once(syncdb, "ready"); + await once(actions.syncdb, "ready"); } logger.debug("jupyterRun: running"); - async function* run() { + async function* runCells() { for (const cell of cells) { - actions.initKernel(); - const output = actions.jupyter_kernel.execute_code({ + actions.ensureKernelIsReady(); + const kernel = actions.jupyter_kernel!; + const output = kernel.execute_code({ halt_on_error: !noHalt, code: cell.input, }); - for await (const mesg of output.iter()) { - mesg.id = cell.id; + for await (const mesg0 of output.iter()) { + const mesg = { ...mesg0, id: cell.id }; yield mesg; if (!noHalt && mesg.msg_type == "error") { // done running code because there was an error. return; } } - if (actions.jupyter_kernel.failedError) { + if (kernel.failedError) { // kernel failed during call - throw Error(actions.jupyter_kernel.failedError); + throw Error(kernel.failedError); } } } - return await run(); + return await runCells(); } class MulticellOutputHandler { @@ -156,9 +160,33 @@ class MulticellOutputHandler { const BACKEND_OUTPUT_FPS = 8; export function outputHandler({ path, cells }: RunOptions) { - if (sessions[path] == null) { - throw Error(`session '${path}' not available`); + if (jupyterActions[ipynbPath(path)] == null) { + throw Error(`session '${ipynbPath(path)}' not available`); } - const { actions } = sessions[path]; + const actions = jupyterActions[ipynbPath(path)]; return new MulticellOutputHandler(cells, actions); } + +function getKernel(path: string) { + const actions = jupyterActions[ipynbPath(path)]; + if (actions == null) { + throw Error(`${ipynbPath(path)} not running`); + } + actions.ensureKernelIsReady(); + return actions.jupyter_kernel!; +} + +export async function introspect(opts: { + path: string; + code: string; + cursor_pos: number; + detail_level: 0 | 1; +}) { + const kernel = getKernel(opts.path); + return await kernel.introspect(opts); +} + +export async function signal(opts: { path: string; signal: string }) { + const kernel = getKernel(opts.path); + await kernel.signal(opts.signal); +} diff --git a/src/packages/jupyter/kernel/kernel.ts b/src/packages/jupyter/kernel/kernel.ts index f8ef7ac1fa..d41b1253a1 100644 --- a/src/packages/jupyter/kernel/kernel.ts +++ b/src/packages/jupyter/kernel/kernel.ts @@ -250,7 +250,7 @@ export class JupyterKernel public _execute_code_queue: CodeExecutionEmitter[] = []; public sockets?: JupyterSockets; private has_ensured_running: boolean = false; - private failedError: string = ""; + public failedError: string = ""; constructor( name: string | undefined, @@ -284,6 +284,8 @@ export class JupyterKernel dbg("done"); } + isClosed = () => this._state == "closed"; + get_path = () => { return this._path; }; @@ -388,7 +390,7 @@ export class JupyterKernel } catch (err) { dbg(`ERROR spawning kernel - ${err}, ${err.stack}`); // @ts-ignore - if (this._state == "closed") { + if (this.isClosed()) { throw Error("closed"); } // console.trace(err); @@ -554,6 +556,7 @@ export class JupyterKernel // Signal should be a string like "SIGINT", "SIGKILL". // See https://nodejs.org/api/process.html#process_process_kill_pid_signal + // this does NOT raise an error. signal = (signal: string): void => { const dbg = this.dbg("signal"); const pid = this.pid(); @@ -564,9 +567,7 @@ export class JupyterKernel try { process.kill(-pid, signal); // negative to signal the process group this.clear_execute_code_queue(); - } catch (err) { - dbg(`error: ${err}`); - } + } catch {} }; close = (): void => { @@ -628,7 +629,7 @@ export class JupyterKernel ensure_running = reuseInFlight(async (): Promise => { const dbg = this.dbg("ensure_running"); dbg(this._state); - if (this._state == "closed") { + if (this.isClosed()) { throw Error("closed so not possible to ensure running"); } if (this._state == "running") { @@ -659,7 +660,7 @@ export class JupyterKernel opts.halt_on_error = true; } if (this._state === "closed") { - throw Error("closed -- kernel -- execute_code"); + throw Error("execute_code: jupyter kernel is closed"); } const code = new CodeExecutionEmitter(this, opts); if (skipToFront) { @@ -740,7 +741,7 @@ export class JupyterKernel const dbg = this.dbg("_clear_execute_code_queue"); // ensure no future queued up evaluation occurs (currently running // one will complete and new executions could happen) - if (this._state === "closed") { + if (this.isClosed()) { dbg("no op since state is closed"); return; } @@ -762,7 +763,7 @@ export class JupyterKernel // the terminal and nbgrader and the stateless api. execute_code_now = async (opts: ExecOpts): Promise => { this.dbg("execute_code_now")(); - if (this._state == "closed") { + if (this.isClosed()) { throw Error("closed"); } if (this.failedError) { @@ -854,7 +855,7 @@ export class JupyterKernel await this.ensure_running(); } // Do a paranoid double check anyways... - if (this.sockets == null || this._state == "closed") { + if (this.sockets == null || this.isClosed()) { throw Error("not running, so can't call"); } diff --git a/src/packages/jupyter/pool/pool.ts b/src/packages/jupyter/pool/pool.ts index 8e74263a2c..da5fc3091c 100644 --- a/src/packages/jupyter/pool/pool.ts +++ b/src/packages/jupyter/pool/pool.ts @@ -267,8 +267,8 @@ export async function killKernel(kernel: SpawnedKernel) { log.debug("killKernel pid=", kernel.spawn.pid); try { process.kill(-kernel.spawn.pid, "SIGTERM"); - } catch (error) { - log.error("Failed to send SIGTERM to Jupyter kernel", error); + } catch { + //log.error("Failed to send SIGTERM to Jupyter kernel", error); } } kernel.spawn?.close?.(); diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index 7c3c9f85ba..872f472826 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -39,11 +39,6 @@ import { JupyterStore, JupyterStoreState } from "@cocalc/jupyter/redux/store"; import { Cell, KernelInfo } from "@cocalc/jupyter/types"; import { IPynbImporter } from "@cocalc/jupyter/ipynb/import-from-ipynb"; import type { JupyterKernelInterface } from "@cocalc/jupyter/types/project-interface"; -import { - char_idx_to_js_idx, - codemirror_to_jupyter_pos, - js_idx_to_char_idx, -} from "@cocalc/jupyter/util/misc"; import { SyncDB } from "@cocalc/sync/editor/db/sync"; import type { Client } from "@cocalc/sync/client/types"; import latexEnvs from "@cocalc/util/latex-envs"; @@ -70,9 +65,7 @@ export class JupyterActions extends Actions { readonly path: string; readonly project_id: string; public jupyter_kernel?: JupyterKernelInterface; - private last_cursor_move_time: Date = new Date(0); - private _cursor_locs?: any; - private _introspect_request?: any; + public _cursor_locs?: any; protected set_save_status: any; protected _client: Client; protected _file_watcher: any; @@ -95,7 +88,6 @@ export class JupyterActions extends Actions { store: any, client: Client, ): void { - console.log("jupyter actions: _init", { path }); this._client = client; const dbg = this.dbg("_init"); dbg("Initializing Jupyter Actions"); @@ -209,11 +201,10 @@ export class JupyterActions extends Actions { // an account_change listener. } - public is_closed(): boolean { - return (this._state ?? "closed") === "closed"; - } + isClosed = () => (this._state ?? "closed") == "closed"; + is_closed = () => (this._state ?? "closed") == "closed"; - public close() { + close() { if (this.is_closed()) { return; } @@ -1031,20 +1022,6 @@ export class JupyterActions extends Actions { this.runCells(v.slice(i)); } - public set_cursor_locs(locs: any[] = [], side_effect: boolean = false): void { - this.last_cursor_move_time = new Date(); - if (this.syncdb == null) { - // syncdb not always set -- https://github.com/sagemathinc/cocalc/issues/2107 - return; - } - if (locs.length === 0) { - // don't remove on blur -- cursor will fade out just fine - return; - } - this._cursor_locs = locs; // remember our own cursors for splitting cell - this.syncdb.set_cursor_locs(locs, side_effect); - } - public split_cell(id: string, cursor: { line: number; ch: number }): void { if (this.check_edit_protection(id, "splitting cell")) { return; @@ -1410,155 +1387,6 @@ export class JupyterActions extends Actions { return this.store.getIn(["cells", id, "input"], ""); } - // Attempt to fetch completions for give code and cursor_pos - // If successful, the completions are put in store.get('completions') and looks like - // this (as an immutable map): - // cursor_end : 2 - // cursor_start : 0 - // matches : ['the', 'completions', ...] - // status : "ok" - // code : code - // cursor_pos : cursor_pos - // - // If not successful, result is: - // status : "error" - // code : code - // cursor_pos : cursor_pos - // error : 'an error message' - // - // Only the most recent fetch has any impact, and calling - // clear_complete() ensures any fetch made before that - // is ignored. - - // Returns true if a dialog with options appears, and false otherwise. - public async complete( - code: string, - pos?: { line: number; ch: number } | number, - id?: string, - offset?: any, - ): Promise { - let cursor_pos; - const req = (this._complete_request = - (this._complete_request != null ? this._complete_request : 0) + 1); - - this.setState({ complete: undefined }); - - // pos can be either a {line:?, ch:?} object as in codemirror, - // or a number. - if (pos == null || typeof pos == "number") { - cursor_pos = pos; - } else { - cursor_pos = codemirror_to_jupyter_pos(code, pos); - } - cursor_pos = js_idx_to_char_idx(cursor_pos, code); - - const start = new Date(); - let complete; - try { - complete = await this.api().complete({ - code, - cursor_pos, - }); - } catch (err) { - if (this._complete_request > req) return false; - this.setState({ complete: { error: err } }); - // no op for now... - throw Error(`ignore -- ${err}`); - //return false; - } - - if (this.last_cursor_move_time >= start) { - // see https://github.com/sagemathinc/cocalc/issues/3611 - throw Error("ignore"); - //return false; - } - if (this._complete_request > req) { - // future completion or clear happened; so ignore this result. - throw Error("ignore"); - //return false; - } - - if (complete.status !== "ok") { - this.setState({ - complete: { - error: complete.error ? complete.error : "completion failed", - }, - }); - return false; - } - - if (complete.matches == 0) { - return false; - } - - delete complete.status; - complete.base = code; - complete.code = code; - complete.pos = char_idx_to_js_idx(cursor_pos, code); - complete.cursor_start = char_idx_to_js_idx(complete.cursor_start, code); - complete.cursor_end = char_idx_to_js_idx(complete.cursor_end, code); - complete.id = id; - // Set the result so the UI can then react to the change. - if (offset != null) { - complete.offset = offset; - } - // For some reason, sometimes complete.matches are not unique, which is annoying/confusing, - // and breaks an assumption in our react code too. - // I think the reason is e.g., a filename and a variable could be the same. We're not - // worrying about that now. - complete.matches = Array.from(new Set(complete.matches)); - // sort in a way that matches how JupyterLab sorts completions, which - // is case insensitive with % magics at the bottom - complete.matches.sort((x, y) => { - const c = misc.cmp(getCompletionGroup(x), getCompletionGroup(y)); - if (c) { - return c; - } - return misc.cmp(x.toLowerCase(), y.toLowerCase()); - }); - const i_complete = immutable.fromJS(complete); - if (complete.matches && complete.matches.length === 1 && id != null) { - // special case -- a unique completion and we know id of cell in which completing is given. - this.select_complete(id, complete.matches[0], i_complete); - return false; - } else { - this.setState({ complete: i_complete }); - return true; - } - } - - clear_complete = (): void => { - this._complete_request = - (this._complete_request != null ? this._complete_request : 0) + 1; - this.setState({ complete: undefined }); - }; - - public select_complete( - id: string, - item: string, - complete?: immutable.Map, - ): void { - if (complete == null) { - complete = this.store.get("complete"); - } - this.clear_complete(); - if (complete == null) { - return; - } - const input = complete.get("code"); - if (input != null && complete.get("error") == null) { - const starting = input.slice(0, complete.get("cursor_start")); - const ending = input.slice(complete.get("cursor_end")); - const new_input = starting + item + ending; - const base = complete.get("base"); - this.complete_cell(id, base, new_input); - } - } - - complete_cell(id: string, base: string, new_input: string): void { - this.merge_cell_input(id, base, new_input); - } - merge_cell_input( id: string, base: string, @@ -1577,126 +1405,6 @@ export class JupyterActions extends Actions { this.set_cell_input(id, new_input, save); } - is_introspecting(): boolean { - const actions = this.getFrameActions() as any; - return actions?.store?.get("introspect") != null; - } - - introspect_close = () => { - if (this.is_introspecting()) { - this.getFrameActions()?.setState({ introspect: undefined }); - } - }; - - introspect_at_pos = async ( - code: string, - level: 0 | 1 = 0, - pos: { ch: number; line: number }, - ): Promise => { - if (code === "") return; // no-op if there is no code (should never happen) - await this.introspect(code, level, codemirror_to_jupyter_pos(code, pos)); - }; - - introspect = async ( - code: string, - level: 0 | 1, - cursor_pos?: number, - ): Promise | undefined> => { - const req = (this._introspect_request = - (this._introspect_request != null ? this._introspect_request : 0) + 1); - - if (cursor_pos == null) { - cursor_pos = code.length; - } - cursor_pos = js_idx_to_char_idx(cursor_pos, code); - - let introspect; - try { - introspect = await this.api().introspect({ - code, - cursor_pos, - level, - }); - if (introspect.status !== "ok") { - introspect = { error: "completion failed" }; - } - delete introspect.status; - } catch (err) { - introspect = { error: err }; - } - if (this._introspect_request > req) return; - const i = immutable.fromJS(introspect); - this.getFrameActions()?.setState({ - introspect: i, - }); - return i; // convenient / useful, e.g., for use by whiteboard. - }; - - clear_introspect = (): void => { - this._introspect_request = - (this._introspect_request != null ? this._introspect_request : 0) + 1; - this.getFrameActions()?.setState({ introspect: undefined }); - }; - - public async signal(signal = "SIGINT"): Promise { - const api = this.api({ timeout: 5000 }); - try { - await api.signal(signal); - } catch (err) { - this.set_error(err); - } - } - - // Kill the running kernel and does NOT start it up again. - halt = reuseInFlight(async (): Promise => { - if (this.restartKernelOnClose != null && this.jupyter_kernel != null) { - this.jupyter_kernel.removeListener("closed", this.restartKernelOnClose); - delete this.restartKernelOnClose; - } - this.clear_all_cell_run_state(); - await this.signal("SIGKILL"); - // Wait a little, since SIGKILL has to really happen on backend, - // and server has to respond and change state. - const not_running = (s): boolean => { - if (this._state === "closed") return true; - const t = s.get_one({ type: "settings" }); - return t != null && t.get("backend_state") != "running"; - }; - try { - await this.syncdb.wait(not_running, 30); - // worked -- and also no need to show "kernel got killed" message since this was intentional. - this.set_error(""); - } catch (err) { - // failed - this.set_error(err); - } - }); - - restart = reuseInFlight(async (): Promise => { - await this.halt(); - if (this._state === "closed") return; - this.clear_all_cell_run_state(); - // Actually start it running again (rather than waiting for - // user to do something), since this is called "restart". - try { - await this.set_backend_kernel_info(); // causes kernel to start - } catch (err) { - this.set_error(err); - } - }); - - public shutdown = reuseInFlight(async (): Promise => { - if (this._state === ("closed" as State)) { - return; - } - await this.signal("SIGKILL"); - if (this._state === ("closed" as State)) { - return; - } - this.clear_all_cell_run_state(); - await this.save_asap(); - }); - set_backend_kernel_info = async (): Promise => { if (this._state === "closed" || this.syncdb.is_read_only()) { return; @@ -2550,14 +2258,3 @@ function bounded_integer(n: any, min: any, max: any, def: any) { } return n; } - -function getCompletionGroup(x: string): number { - switch (x[0]) { - case "_": - return 1; - case "%": - return 2; - default: - return 0; - } -} diff --git a/src/packages/jupyter/redux/project-actions.ts b/src/packages/jupyter/redux/project-actions.ts index 8befaacec2..01f1e0d537 100644 --- a/src/packages/jupyter/redux/project-actions.ts +++ b/src/packages/jupyter/redux/project-actions.ts @@ -26,9 +26,13 @@ export class JupyterActions extends JupyterActions0 { capture_output_message = (_opts) => {}; process_comm_message_from_kernel = (_mesg) => {}; - initKernel = () => { + ensureKernelIsReady = () => { if (this.jupyter_kernel != null) { - return; + if (this.jupyter_kernel.isClosed()) { + delete this.jupyter_kernel; + } else { + return; + } } const kernel = this.store.get("kernel"); console.log("initKernel", { kernel, path: this.path }); diff --git a/src/packages/jupyter/redux/run-all-loop.ts b/src/packages/jupyter/redux/run-all-loop.ts deleted file mode 100644 index 472aa42ab3..0000000000 --- a/src/packages/jupyter/redux/run-all-loop.ts +++ /dev/null @@ -1,63 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { delay } from "awaiting"; - -import { close } from "@cocalc/util/misc"; -import { JupyterActions } from "./project-actions"; - -export class RunAllLoop { - private actions: JupyterActions; - public interval_s: number; - private closed: boolean = false; - private dbg: Function; - - constructor(actions, interval_s) { - this.actions = actions; - this.interval_s = interval_s; - this.dbg = actions.dbg("RunAllLoop"); - this.dbg(`interval_s=${interval_s}`); - this.loop(); - } - - public set_interval(interval_s: number): void { - if (this.closed) { - throw Error("should not call set_interval if RunAllLoop is closed"); - } - if (this.interval_s == interval_s) return; - this.dbg(`.set_interval: interval_s=${interval_s}`); - this.interval_s = interval_s; - } - - private async loop(): Promise { - this.dbg("starting loop..."); - while (true) { - if (this.closed) break; - try { - this.dbg("loop: restart"); - await this.actions.restart(); - } catch (err) { - this.dbg(`restart failed (will try run-all anyways) - ${err}`); - } - if (this.closed) break; - try { - this.dbg("loop: run_all_cells"); - await this.actions.run_all_cells(true); - } catch (err) { - this.dbg(`run_all_cells failed - ${err}`); - } - if (this.closed) break; - this.dbg(`loop: waiting ${this.interval_s} seconds`); - await delay(this.interval_s * 1000); - } - this.dbg("terminating loop..."); - } - - public close() { - this.dbg("close"); - close(this); - this.closed = true; - } -} diff --git a/src/packages/jupyter/types/project-interface.ts b/src/packages/jupyter/types/project-interface.ts index 009f4b5226..df1898833f 100644 --- a/src/packages/jupyter/types/project-interface.ts +++ b/src/packages/jupyter/types/project-interface.ts @@ -76,7 +76,7 @@ export interface ExecOpts { timeout_ms?: number; } -export type OutputMessage = object; // todo +export type OutputMessage = any; // todo export interface CodeExecutionEmitterInterface extends EventEmitterInterface { emit_output(result: OutputMessage): void; @@ -97,7 +97,9 @@ export interface JupyterKernelInterface extends EventEmitterInterface { name: string | undefined; // name = undefined implies it is not spawnable. It's a notebook with no actual jupyter kernel process. store: any; readonly identity: string; + failedError: string; + isClosed(): boolean; get_state(): string; signal(signal: string): void; close(): void; diff --git a/src/packages/project/conat/api/jupyter.ts b/src/packages/project/conat/api/jupyter.ts new file mode 100644 index 0000000000..8d6260ed71 --- /dev/null +++ b/src/packages/project/conat/api/jupyter.ts @@ -0,0 +1,50 @@ +export { jupyter_strip_notebook as stripNotebook } from "@cocalc/jupyter/nbgrader/jupyter-parse"; +export { jupyter_run_notebook as runNotebook } from "@cocalc/jupyter/nbgrader/jupyter-run"; +export { nbconvert } from "../../jupyter/convert"; +export { formatString } from "../../formatters"; +export { logo as kernelLogo } from "@cocalc/jupyter/kernel/logo"; +export { get_kernel_data as kernels } from "@cocalc/jupyter/kernel/kernel-data"; +export { newFile } from "@cocalc/backend/misc/new-file"; +import { getClient } from "@cocalc/project/client"; +import { project_id } from "@cocalc/project/data"; +import * as control from "@cocalc/jupyter/control"; +import { SandboxedFilesystem } from "@cocalc/backend/files/sandbox"; + +let fs: SandboxedFilesystem | null = null; +export async function start(path: string) { + if (control.isRunning(path)) { + return; + } + fs ??= new SandboxedFilesystem(process.env.HOME ?? "/tmp", { + unsafeMode: true, + }); + await control.start({ project_id, path, client: getClient(), fs }); +} + +// IMPORTANT: run is NOT used directly by the API, but instead by packages/project/conat/jupyter.ts +// It is convenient to have it here so it can call start above, etc. The reason is because +// this returns an async iterator managed using a dedicated socket, and the api is request/response, +// so it can't just be part of the normal api. +export async function run(opts: { + path: string; + cells: { id: string; input: string }[]; +}) { + await start(opts.path); + return await control.run(opts); +} + +export async function stop(path: string) { + await control.stop({ path }); +} + +export async function introspect(opts) { + await start(opts.path); + return await control.introspect(opts); +} + +export async function signal(opts) { + if (!control.isRunning(opts.path)) { + return; + } + await control.signal(opts); +} diff --git a/src/packages/util/jupyter/names.ts b/src/packages/util/jupyter/names.ts index e7d578494d..f16106a862 100644 --- a/src/packages/util/jupyter/names.ts +++ b/src/packages/util/jupyter/names.ts @@ -1,12 +1,23 @@ -import { meta_file } from "@cocalc/util/misc"; +import { meta_file, original_path } from "@cocalc/util/misc"; export const JUPYTER_POSTFIX = "jupyter2"; export const JUPYTER_SYNCDB_EXTENSIONS = `sage-${JUPYTER_POSTFIX}`; -// a.ipynb --> ".a.ipynb.sage-jupyter2" -export function syncdbPath(ipynbPath: string) { - if (!ipynbPath.endsWith(".ipynb")) { - throw Error(`ipynbPath must end with .ipynb but it is "${ipynbPath}"`); +// a.ipynb or .a.ipynb.sage-jupyter2 --> .a.ipynb.sage-jupyter2 +export function syncdbPath(path: string) { + if (path.endsWith(JUPYTER_POSTFIX)) { + return path; } - return meta_file(ipynbPath, JUPYTER_POSTFIX); + if (!path.endsWith(".ipynb")) { + throw Error(`must end with .ipynb but it is "${ipynbPath}"`); + } + return meta_file(path, JUPYTER_POSTFIX); +} + +// a.ipynb or .a.ipynb.sage-jupyter2 --> a.ipynb +export function ipynbPath(path: string) { + if (path.endsWith(".ipynb")) { + return path; + } + return original_path(path); } From 9dc18999af073ff7001486cfe0de4a1a9846d67e Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 2 Aug 2025 01:17:36 +0000 Subject: [PATCH 158/798] frontend terminal -- noticed some cases where it might try to do something while closed, so cleaned that up --- .../terminal-editor/connected-terminal.ts | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index 34e73fb720..eaf068aac6 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -50,7 +50,6 @@ const MAX_DELAY = 15000; const ENABLE_WEBGL = false; - // ephemeral = faster, less load on servers, but if project and browser all // close, the history is gone... which may be good and less confusing. const EPHEMERAL = true; @@ -60,8 +59,10 @@ interface Path { directory?: string; } +type State = "ready" | "closed"; + export class Terminal { - private state: string = "ready"; + private state: State = "ready"; private actions: Actions | ConnectedTerminalInterface; private account_store: any; private project_actions: ProjectActions; @@ -196,6 +197,8 @@ export class Terminal { // this.terminal_resize = debounce(this.terminal_resize, 2000); } + isClosed = () => (this.state ?? "closed") === "closed"; + private get_xtermjs_options = (): any => { const rendererType = this.rendererType; const settings = this.account_store.get("terminal"); @@ -221,13 +224,13 @@ export class Terminal { }; private assert_not_closed = (): void => { - if (this.state === "closed") { + if (this.isClosed()) { throw Error("BUG -- Terminal is closed."); } }; close = (): void => { - if (this.state === "closed") { + if (this.isClosed()) { return; } this.set_connection_status("disconnected"); @@ -398,10 +401,9 @@ export class Terminal { }; private render = async (data: string): Promise => { - if (data == null) { + if (data == null || this.isClosed()) { return; } - this.assert_not_closed(); this.history += data; if (this.history.length > MAX_HISTORY_LENGTH) { this.history = this.history.slice( @@ -423,7 +425,7 @@ export class Terminal { await delay(0); this.ignoreData--; } - if (this.state == "done") return; + if (this.isClosed()) return; // tell anyone who waited for output coming back about this while (this.render_done.length > 0) { this.render_done.pop()?.(); @@ -453,7 +455,7 @@ export class Terminal { }; touch = async () => { - if (this.state === "closed") return; + if (this.isClosed()) return; if (Date.now() - this.last_active < 70000) { if (this.project_actions.isTabClosed()) { return; @@ -467,7 +469,7 @@ export class Terminal { }; init_keyhandler = (): void => { - if (this.state === "closed") { + if (this.isClosed()) { return; } if (this.keyhandler_initialized) { @@ -578,7 +580,7 @@ export class Terminal { // Stop ignoring terminal data... but ONLY once // the render buffer is also empty. no_ignore = async (): Promise => { - if (this.state === "closed") { + if (this.isClosed()) { return; } const g = (cb) => { @@ -590,7 +592,7 @@ export class Terminal { } // cause render to actually appear now. await delay(0); - if (this.state === "closed") { + if (this.isClosed()) { return; } try { @@ -725,12 +727,14 @@ export class Terminal { update_cwd = debounce( async () => { + if (this.isClosed()) return; let cwd; try { cwd = await this.conn?.api.cwd(); } catch { return; } + if (this.isClosed()) return; if (cwd != null) { this.actions.set_terminal_cwd(this.id, cwd); } @@ -775,14 +779,14 @@ export class Terminal { } focus(): void { - if (this.state === "closed") { + if (this.isClosed()) { return; } this.terminal.focus(); } refresh(): void { - if (this.state === "closed") { + if (this.isClosed()) { return; } this.terminal.refresh(0, this.terminal.rows - 1); @@ -792,7 +796,7 @@ export class Terminal { try { await open_init_file(this.actions._get_project_actions(), this.termPath); } catch (err) { - if (this.state === "closed") { + if (this.isClosed()) { return; } this.actions.set_error(`Problem opening init file -- ${err}`); From 73a9e318a8321b2950f1e7a93c0d9aa68bef4d7f Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 2 Aug 2025 02:10:43 +0000 Subject: [PATCH 159/798] jupyter: fix situation where errors during execution not properly reported to browser --- src/packages/conat/project/jupyter/run-code.ts | 17 +++++++++++------ .../frontend/jupyter/browser-actions.ts | 2 ++ src/packages/jupyter/execute/execute-code.ts | 8 +++++++- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/packages/conat/project/jupyter/run-code.ts b/src/packages/conat/project/jupyter/run-code.ts index ef78773bf9..3a0dc32919 100644 --- a/src/packages/conat/project/jupyter/run-code.ts +++ b/src/packages/conat/project/jupyter/run-code.ts @@ -120,13 +120,16 @@ export function jupyterServer({ noHalt, }); } catch (err) { - //console.log(err); - logger.debug("server: failed response -- ", err); + logger.debug("server: failed to handle execute request -- ", err); if (socket.state != "closed") { try { - socket.write(null, { headers: { error: `${err}` } }); - } catch { + logger.debug("sending to client: ", { + headers: { error: `${err}` }, + }); + socket.write(null, { headers: { foo: "bar", error: `${err}` } }); + } catch (err) { // an error trying to report an error shouldn't crash everything + logger.debug("WARNING: unable to send error to client", err); } } } @@ -195,12 +198,14 @@ async function handleRequest({ throttle.write(mesg); } } + // no errors happened, so close up and flush and + // remaining data immediately: handler?.done(); - } finally { - if (socket.state != "closed" && !unhandledClientWriteError) { + if (socket.state != "closed") { throttle.flush(); socket.write(null); } + } finally { throttle.close(); } } diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 7601684e60..4c5dd69d10 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -1648,6 +1648,8 @@ export class JupyterActions extends JupyterActions0 { }, 1000); } catch (err) { console.warn("runCells", err); + this.clearRunQueue(); + this.set_error(err); } finally { if (this.isClosed()) return; this.runningNow = false; diff --git a/src/packages/jupyter/execute/execute-code.ts b/src/packages/jupyter/execute/execute-code.ts index 73744b34a7..df396ebbe8 100644 --- a/src/packages/jupyter/execute/execute-code.ts +++ b/src/packages/jupyter/execute/execute-code.ts @@ -150,7 +150,13 @@ export class CodeExecutionEmitter }; throw_error = (err): void => { - this.emit("error", err); + if (this._iter != null) { + // using the iter, so we can use that to report the error + this._iter.throw(err); + } else { + // no iter so make error known via error event + this.emit("error", err); + } this.close(); }; From 3f029952400632860c399d6ee132af9560c0ee32 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 2 Aug 2025 02:43:44 +0000 Subject: [PATCH 160/798] jupyter: tab completion --- src/packages/conat/project/api/jupyter.ts | 7 + src/packages/frontend/client/project.ts | 3 +- .../frontend/jupyter/browser-actions.ts | 124 +++++++++--------- src/packages/jupyter/control.ts | 9 ++ src/packages/jupyter/redux/actions.ts | 1 - src/packages/project/conat/api/index.ts | 4 +- src/packages/project/conat/api/jupyter.ts | 5 + 7 files changed, 87 insertions(+), 66 deletions(-) diff --git a/src/packages/conat/project/api/jupyter.ts b/src/packages/conat/project/api/jupyter.ts index 3defeb5b57..1ac6b39ed3 100644 --- a/src/packages/conat/project/api/jupyter.ts +++ b/src/packages/conat/project/api/jupyter.ts @@ -11,6 +11,7 @@ export const jupyter = { kernelLogo: true, kernels: true, introspect: true, + complete: true, signal: true, }; @@ -41,5 +42,11 @@ export interface Jupyter { detail_level: 0 | 1; }) => Promise; + complete: (opts: { + path: string; + code: string; + cursor_pos: number; + }) => Promise; + signal: (opts: { path: string; signal: string }) => Promise; } diff --git a/src/packages/frontend/client/project.ts b/src/packages/frontend/client/project.ts index 8938d29aa4..55ee2b9d80 100644 --- a/src/packages/frontend/client/project.ts +++ b/src/packages/frontend/client/project.ts @@ -41,6 +41,7 @@ import { WebappClient } from "./client"; import { throttle } from "lodash"; import { writeFile, type WriteFileOptions } from "@cocalc/conat/files/write"; import { readFile, type ReadFileOptions } from "@cocalc/conat/files/read"; +import { type ProjectApi } from "@cocalc/conat/project/api"; export class ProjectClient { private client: WebappClient; @@ -50,7 +51,7 @@ export class ProjectClient { this.client = client; } - conatApi = (project_id: string, compute_server_id = 0) => { + conatApi = (project_id: string, compute_server_id = 0): ProjectApi => { return this.client.conat_client.projectApi({ project_id, compute_server_id, diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 4c5dd69d10..9f4af24b10 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -82,8 +82,7 @@ export class JupyterActions extends JupyterActions0 { private account_change_editor_settings: any; private update_keyboard_shortcuts: any; public syncdbPath: string; - private last_cursor_move_time: Date = new Date(0); - private _introspect_request?: any; + private lastCursorMoveTime: number = 0; protected init2(): void { this.syncdbPath = syncdbPath(this.path); @@ -1454,13 +1453,13 @@ export class JupyterActions extends JupyterActions0 { // tells them to open this jupyter notebook, so it can provide the compute // functionality. - conatApi = async () => { + private jupyterApi = async () => { const compute_server_id = await this.getComputeServerId(); const api = webapp_client.project_client.conatApi( this.project_id, compute_server_id, ); - return api; + return api.jupyter; }; initBackend = async () => { @@ -1470,8 +1469,8 @@ export class JupyterActions extends JupyterActions0 { return true; } try { - const api = await this.conatApi(); - await api.jupyter.start(this.syncdbPath); + const api = await this.jupyterApi(); + await api.start(this.syncdbPath); return true; } catch (err) { console.log("failed to initialize ", this.path, err); @@ -1483,8 +1482,8 @@ export class JupyterActions extends JupyterActions0 { }; stopBackend = async () => { - const api = await this.conatApi(); - await api.jupyter.stop(this.syncdbPath); + const api = await this.jupyterApi(); + await api.stop(this.syncdbPath); }; getOutputHandler = (cell) => { @@ -1684,14 +1683,14 @@ export class JupyterActions extends JupyterActions0 { ); }; + private introspectRequest: number = 0; introspect = async ( code: string, detail_level: 0 | 1, cursor_pos?: number, ): Promise | undefined> => { - const req = (this._introspect_request = - (this._introspect_request != null ? this._introspect_request : 0) + 1); - + this.introspectRequest++; + const req = this.introspectRequest; if (cursor_pos == null) { cursor_pos = code.length; } @@ -1699,8 +1698,8 @@ export class JupyterActions extends JupyterActions0 { let introspect; try { - const api = await this.conatApi(); - introspect = await api.jupyter.introspect({ + const api = await this.jupyterApi(); + introspect = await api.introspect({ path: this.path, code, cursor_pos, @@ -1713,55 +1712,56 @@ export class JupyterActions extends JupyterActions0 { } catch (err) { introspect = { error: err }; } - if (this._introspect_request > req) return; - const i = fromJS(introspect); - this.getFrameActions()?.setState({ - introspect: i, - }); + if (this.introspectRequest > req) return; + this.getFrameActions()?.setState({ introspect }); return introspect; // convenient / useful, e.g., for use by whiteboard. }; clear_introspect = (): void => { - this._introspect_request = - (this._introspect_request != null ? this._introspect_request : 0) + 1; + this.introspectRequest = + (this.introspectRequest != null ? this.introspectRequest : 0) + 1; this.getFrameActions()?.setState({ introspect: undefined }); }; - // Attempt to fetch completions for give code and cursor_pos - // If successful, the completions are put in store.get('completions') and looks like - // this (as an immutable map): - // cursor_end : 2 - // cursor_start : 0 - // matches : ['the', 'completions', ...] - // status : "ok" - // code : code - // cursor_pos : cursor_pos - // - // If not successful, result is: - // status : "error" - // code : code - // cursor_pos : cursor_pos - // error : 'an error message' - // - // Only the most recent fetch has any impact, and calling - // clear_complete() ensures any fetch made before that - // is ignored. + /* + complete: + + Attempt to fetch completions for give code and cursor_pos + If successful, the completions are put in store.get('completions') and looks + like this (as an immutable map): + cursor_end : 2 + cursor_start : 0 + matches : ['the', 'completions', ...] + status : "ok" + code : code + cursor_pos : cursor_pos + + If not successful, result is: + status : "error" + code : code + cursor_pos : cursor_pos + error : 'an error message' + + Only the most recent fetch has any impact, and calling + clear_complete() ensures any fetch made before that + is ignored. // Returns true if a dialog with options appears, and false otherwise. + */ + private completeRequest = 0; complete = async ( code: string, pos?: { line: number; ch: number } | number, id?: string, offset?: any, ): Promise => { - let cursor_pos; - const req = (this._complete_request = - (this._complete_request != null ? this._complete_request : 0) + 1); - + this.completeRequest++; + const req = this.completeRequest; this.setState({ complete: undefined }); // pos can be either a {line:?, ch:?} object as in codemirror, // or a number. + let cursor_pos; if (pos == null || typeof pos == "number") { cursor_pos = pos; } else { @@ -1769,30 +1769,28 @@ export class JupyterActions extends JupyterActions0 { } cursor_pos = js_idx_to_char_idx(cursor_pos, code); - const start = new Date(); + const start = Date.now(); let complete; try { - complete = await this.api().complete({ + const api = await this.jupyterApi(); + complete = await api.complete({ + path: this.path, code, cursor_pos, }); } catch (err) { - if (this._complete_request > req) return false; + if (this.completeRequest > req) return false; this.setState({ complete: { error: err } }); - // no op for now... throw Error(`ignore -- ${err}`); - //return false; } - if (this.last_cursor_move_time >= start) { + if (this.lastCursorMoveTime >= start) { // see https://github.com/sagemathinc/cocalc/issues/3611 throw Error("ignore"); - //return false; } - if (this._complete_request > req) { + if (this.completeRequest > req) { // future completion or clear happened; so ignore this result. throw Error("ignore"); - //return false; } if (complete.status !== "ok") { @@ -1845,8 +1843,8 @@ export class JupyterActions extends JupyterActions0 { }; clear_complete = (): void => { - this._complete_request = - (this._complete_request != null ? this._complete_request : 0) + 1; + this.completeRequest = + (this.completeRequest != null ? this.completeRequest : 0) + 1; this.setState({ complete: undefined }); }; @@ -1872,12 +1870,12 @@ export class JupyterActions extends JupyterActions0 { } } - complete_cell(id: string, base: string, new_input: string): void { + complete_cell = (id: string, base: string, new_input: string): void => { this.merge_cell_input(id, base, new_input); - } + }; - public set_cursor_locs(locs: any[] = [], side_effect: boolean = false): void { - this.last_cursor_move_time = new Date(); + set_cursor_locs = (locs: any[] = [], side_effect: boolean = false): void => { + this.lastCursorMoveTime = Date.now(); if (this.syncdb == null) { // syncdb not always set -- https://github.com/sagemathinc/cocalc/issues/2107 return; @@ -1888,16 +1886,16 @@ export class JupyterActions extends JupyterActions0 { } this._cursor_locs = locs; // remember our own cursors for splitting cell this.syncdb.set_cursor_locs(locs, side_effect); - } + }; - async signal(signal = "SIGINT"): Promise { - const api = await this.conatApi(); + signal = async (signal = "SIGINT"): Promise => { + const api = await this.jupyterApi(); try { - await api.jupyter.signal({ path: this.path, signal }); + await api.signal({ path: this.path, signal }); } catch (err) { this.set_error(err); } - } + }; // Kill the running kernel and does NOT start it up again. halt = reuseInFlight(async (): Promise => { diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts index 8efeddd0ac..78674b6edc 100644 --- a/src/packages/jupyter/control.ts +++ b/src/packages/jupyter/control.ts @@ -186,6 +186,15 @@ export async function introspect(opts: { return await kernel.introspect(opts); } +export async function complete(opts: { + path: string; + code: string; + cursor_pos: number; +}) { + const kernel = getKernel(opts.path); + return await kernel.complete(opts); +} + export async function signal(opts: { path: string; signal: string }) { const kernel = getKernel(opts.path); await kernel.signal(opts.signal); diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index 872f472826..2022d935eb 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -73,7 +73,6 @@ export class JupyterActions extends Actions { protected restartKernelOnClose?: (...args: any[]) => void; public asyncBlobStore: AKV; - public _complete_request?: number; public store: JupyterStore; public syncdb: SyncDB; private labels?: { diff --git a/src/packages/project/conat/api/index.ts b/src/packages/project/conat/api/index.ts index 7133583609..a6b32a86ce 100644 --- a/src/packages/project/conat/api/index.ts +++ b/src/packages/project/conat/api/index.ts @@ -166,7 +166,9 @@ async function getResponse({ name, args }) { const [group, functionName] = name.split("."); const f = projectApi[group]?.[functionName]; if (f == null) { - throw Error(`unknown function '${name}'`); + throw Error( + `unknown function '${name}' -- available functions are ${JSON.stringify(Object.keys(projectApi[group]))}`, + ); } return await f(...args); } diff --git a/src/packages/project/conat/api/jupyter.ts b/src/packages/project/conat/api/jupyter.ts index 8d6260ed71..0afe0a9a93 100644 --- a/src/packages/project/conat/api/jupyter.ts +++ b/src/packages/project/conat/api/jupyter.ts @@ -42,6 +42,11 @@ export async function introspect(opts) { return await control.introspect(opts); } +export async function complete(opts) { + await start(opts.path); + return await control.complete(opts); +} + export async function signal(opts) { if (!control.isRunning(opts.path)) { return; From 9e7e214b7778c944c038c0ff872b12346ec3f414 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 2 Aug 2025 03:03:12 +0000 Subject: [PATCH 161/798] add jupyter api getConnectionFile and use that instead of redux/sync --- src/packages/conat/project/api/jupyter.ts | 3 +++ .../frame-editors/jupyter-editor/actions.ts | 24 ++++--------------- .../frontend/jupyter/browser-actions.ts | 9 +++++++ src/packages/jupyter/control.ts | 10 ++++++++ src/packages/jupyter/kernel/kernel.ts | 2 +- src/packages/jupyter/redux/actions.ts | 1 - src/packages/jupyter/redux/store.ts | 1 - .../jupyter/types/project-interface.ts | 2 +- src/packages/project/conat/api/jupyter.ts | 5 ++++ 9 files changed, 33 insertions(+), 24 deletions(-) diff --git a/src/packages/conat/project/api/jupyter.ts b/src/packages/conat/project/api/jupyter.ts index 1ac6b39ed3..22f7e46211 100644 --- a/src/packages/conat/project/api/jupyter.ts +++ b/src/packages/conat/project/api/jupyter.ts @@ -13,6 +13,7 @@ export const jupyter = { introspect: true, complete: true, signal: true, + getConnectionFile: true, }; // In the functions below path can be either the .ipynb or the .sage-jupyter2 path, and @@ -48,5 +49,7 @@ export interface Jupyter { cursor_pos: number; }) => Promise; + getConnectionFile: (opts: { path: string }) => Promise; + signal: (opts: { path: string; signal: string }) => Promise; } diff --git a/src/packages/frontend/frame-editors/jupyter-editor/actions.ts b/src/packages/frontend/frame-editors/jupyter-editor/actions.ts index e15e797f68..ed8f115c65 100644 --- a/src/packages/frontend/frame-editors/jupyter-editor/actions.ts +++ b/src/packages/frontend/frame-editors/jupyter-editor/actions.ts @@ -117,26 +117,12 @@ export class JupyterEditorActions extends BaseActions { private watchJupyterStore = (): void => { const store = this.jupyter_actions.store; - let connection_file = store.get("connection_file"); store.on("change", () => { // sync read only state -- source of true is jupyter_actions.store.get('read_only') const read_only = store.get("read_only"); if (read_only != this.store.get("read_only")) { this.setState({ read_only }); } - // sync connection file - const c = store.get("connection_file"); - if (c == connection_file) { - return; - } - connection_file = c; - const id = this._get_most_recent_shell_id("jupyter"); - if (id == null) { - // There is no Jupyter console open right now... - return; - } - // This will update the connection file - this.shell(id, true); }); }; @@ -295,14 +281,12 @@ export class JupyterEditorActions extends BaseActions { } protected async get_shell_spec( - id: string, - ): Promise { - id = id; // not used - const connection_file = this.jupyter_actions.store.get("connection_file"); - if (connection_file == null) return; + _id: string, + ): Promise<{ command: string; args: string[] }> { + const connectionFile = await this.jupyter_actions.getConnectionFile(); return { command: "jupyter", - args: ["console", "--existing", connection_file], + args: ["console", "--existing", connectionFile], }; } diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 9f4af24b10..01ddede80b 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -1336,6 +1336,7 @@ export class JupyterActions extends JupyterActions0 { }; private saveIpynb = async () => { + if(this.isClosed()) return; const ipynb = await this.toIpynb(); const serialize = JSON.stringify(ipynb, undefined, 2); this.syncdb.fs.writeFile(this.path, serialize); @@ -1639,6 +1640,9 @@ export class JupyterActions extends JupyterActions0 { } } handler?.done(); + if (this.isClosed()) { + return; + } this.syncdb.save(); setTimeout(() => { if (!this.isClosed()) { @@ -1934,6 +1938,11 @@ export class JupyterActions extends JupyterActions0 { if (this.is_closed()) return; this.clear_all_cell_run_state(); }); + + getConnectionFile = async (): Promise => { + const api = await this.jupyterApi(); + return await api.getConnectionFile({ path: this.path }); + }; } function getCompletionGroup(x: string): number { diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts index 78674b6edc..2c7bc900cf 100644 --- a/src/packages/jupyter/control.ts +++ b/src/packages/jupyter/control.ts @@ -195,6 +195,16 @@ export async function complete(opts: { return await kernel.complete(opts); } +export async function getConnectionFile(opts: { path }) { + const kernel = getKernel(opts.path); + await kernel.ensure_running(); + const c = kernel.getConnectionFile(); + if (c == null) { + throw Error("unable to start kernel"); + } + return c; +} + export async function signal(opts: { path: string; signal: string }) { const kernel = getKernel(opts.path); await kernel.signal(opts.signal); diff --git a/src/packages/jupyter/kernel/kernel.ts b/src/packages/jupyter/kernel/kernel.ts index d41b1253a1..b83594c80e 100644 --- a/src/packages/jupyter/kernel/kernel.ts +++ b/src/packages/jupyter/kernel/kernel.ts @@ -404,7 +404,7 @@ export class JupyterKernel return this._kernel; }; - get_connection_file = (): string | undefined => { + getConnectionFile = (): string | undefined => { return this._kernel?.connectionFile; }; diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index 2022d935eb..3cbc2c992d 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -641,7 +641,6 @@ export class JupyterActions extends Actions { kernel_state: record.get("kernel_state"), kernel_error: record.get("kernel_error"), metadata: record.get("metadata"), // extra custom user-specified metadata - connection_file: record.get("connection_file") ?? "", max_output_length: bounded_integer( record.get("max_output_length"), 100, diff --git a/src/packages/jupyter/redux/store.ts b/src/packages/jupyter/redux/store.ts index a334be7de2..1109054ecd 100644 --- a/src/packages/jupyter/redux/store.ts +++ b/src/packages/jupyter/redux/store.ts @@ -60,7 +60,6 @@ export interface JupyterStoreState { cm_options: any; complete: any; confirm_dialog: any; - connection_file?: string; contents?: List>; // optional global contents info (about sections, problems, etc.) default_kernel?: string; directory: string; diff --git a/src/packages/jupyter/types/project-interface.ts b/src/packages/jupyter/types/project-interface.ts index df1898833f..986d16cd28 100644 --- a/src/packages/jupyter/types/project-interface.ts +++ b/src/packages/jupyter/types/project-interface.ts @@ -127,7 +127,7 @@ export interface JupyterKernelInterface extends EventEmitterInterface { buffers?: any[]; buffers64?: any[]; }): void; - get_connection_file(): string | undefined; + getConnectionFile(): string | undefined; _execute_code_queue: CodeExecutionEmitterInterface[]; clear_execute_code_queue(): void; diff --git a/src/packages/project/conat/api/jupyter.ts b/src/packages/project/conat/api/jupyter.ts index 0afe0a9a93..849d81071f 100644 --- a/src/packages/project/conat/api/jupyter.ts +++ b/src/packages/project/conat/api/jupyter.ts @@ -47,6 +47,11 @@ export async function complete(opts) { return await control.complete(opts); } +export async function getConnectionFile(opts) { + await start(opts.path); + return await control.getConnectionFile(opts); +} + export async function signal(opts) { if (!control.isRunning(opts.path)) { return; From ca84275fb5b61e26998180accf9101357f01dc7a Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 2 Aug 2025 04:34:45 +0000 Subject: [PATCH 162/798] jupyter: ensure positions are unique --- .../frontend/jupyter/browser-actions.ts | 19 +++++++++------- src/packages/jupyter/redux/actions.ts | 22 ++++++++----------- src/packages/jupyter/util/cell-utils.ts | 14 ++++++------ 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 01ddede80b..5570db0696 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -86,7 +86,6 @@ export class JupyterActions extends JupyterActions0 { protected init2(): void { this.syncdbPath = syncdbPath(this.path); - this.update_contents = debounce(this.update_contents.bind(this), 2000); this.setState({ toolbar: !this.get_local_storage("hide_toolbar"), cell_toolbar: this.get_local_storage("cell_toolbar"), @@ -116,13 +115,16 @@ export class JupyterActions extends JupyterActions0 { this.syncdb.on("connected", this.sync_read_only); // first update - this.syncdb.once("change", this.updateContentsNow); - this.syncdb.once("change", this.updateRunProgress); + this.syncdb.once("change", () => { + this.updateContentsNow(); + this.updateRunProgress(); + this.ensurePositionsAreUnique(); + }); this.syncdb.on("change", () => { // And activity indicator this.activity(); - // Update table of contents + // Update table of contents -- this is debounced this.update_contents(); // run progress this.updateRunProgress(); @@ -319,7 +321,7 @@ export class JupyterActions extends JupyterActions0 { } public async close(): Promise { - if (this.is_closed()) return; + if (this.isClosed()) return; this.jupyterClient?.close(); await super.close(); } @@ -987,9 +989,10 @@ export class JupyterActions extends JupyterActions0 { this.setState({ contents }); }; - public update_contents(): void { + update_contents = debounce(() => { + if (this.isClosed()) return; this.updateContentsNow(); - } + }, 2000); protected __syncdb_change_post_hook(_doInit: boolean) { if (this._state === "init") { @@ -1336,7 +1339,7 @@ export class JupyterActions extends JupyterActions0 { }; private saveIpynb = async () => { - if(this.isClosed()) return; + if (this.isClosed()) return; const ipynb = await this.toIpynb(); const serialize = JSON.stringify(ipynb, undefined, 2); this.syncdb.fs.writeFile(this.path, serialize); diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index 3cbc2c992d..b66bddb166 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -1300,7 +1300,7 @@ export class JupyterActions extends Actions { cell = cell.set("pos", i); this._set(cell, false); }); - this.ensure_positions_are_unique(); + this.ensurePositionsAreUnique(); this._sync(); return; } @@ -1731,7 +1731,7 @@ export class JupyterActions extends Actions { this._state = "ready"; }; - public set_cell_slide(id: string, value: any): void { + set_cell_slide = (id: string, value: any) => { if (!value) { value = null; // delete } @@ -1743,15 +1743,11 @@ export class JupyterActions extends Actions { id, slide: value, }); - } + }; - public ensure_positions_are_unique(): void { - if (this._state != "ready" || this.store == null) { - // because of debouncing, this ensure_positions_are_unique can - // be called after jupyter actions are closed. - return; - } - const changes = cell_utils.ensure_positions_are_unique( + ensurePositionsAreUnique = () => { + if (this.isClosed()) return; + const changes = cell_utils.ensurePositionsAreUnique( this.store.get("cells"), ); if (changes != null) { @@ -1761,9 +1757,9 @@ export class JupyterActions extends Actions { } } this._sync(); - } + }; - public set_default_kernel(kernel?: string): void { + set_default_kernel = (kernel?: string) => { if (kernel == null || kernel === "") return; // doesn't make sense for project (right now at least) if (this.is_project || this.is_compute_server) return; @@ -1780,7 +1776,7 @@ export class JupyterActions extends Actions { (this.redux.getTable("account") as any).set({ editor_settings: { jupyter: cur }, }); - } + }; edit_attachments = (id: string): void => { this.setState({ edit_attachments: id }); diff --git a/src/packages/jupyter/util/cell-utils.ts b/src/packages/jupyter/util/cell-utils.ts index ecc59eef87..e6d68ac9b3 100644 --- a/src/packages/jupyter/util/cell-utils.ts +++ b/src/packages/jupyter/util/cell-utils.ts @@ -13,7 +13,7 @@ import { field_cmp, len } from "@cocalc/util/misc"; export function positions_between( before_pos: number | undefined, after_pos: number | undefined, - num: number + num: number, ) { // Return an array of num equally spaced positions starting after // before_pos and ending before after_pos, so @@ -66,22 +66,22 @@ export function sorted_cell_list(cells: Map): List { .toList(); } -export function ensure_positions_are_unique(cells?: Map) { +export function ensurePositionsAreUnique(cells?: Map) { // Verify that pos's of cells are distinct. If not // return map from id's to new unique positions. if (cells == null) { return; } - const v: any = {}; + const v = new Set(); let all_unique = true; cells.forEach((cell) => { const pos = cell.get("pos"); - if (pos == null || v[pos]) { + if (pos == null || v.has(pos)) { // dup! (or not defined) all_unique = false; return false; } - v[pos] = true; + v.add(pos); }); if (all_unique) { return; @@ -99,7 +99,7 @@ export function new_cell_pos( cells: Map, cell_list: List, cur_id: string, - delta: -1 | 1 + delta: -1 | 1, ): number { /* Returns pos for a new cell whose position @@ -145,7 +145,7 @@ export function new_cell_pos( export function move_selected_cells( v?: string[], selected?: { [id: string]: true }, - delta?: number + delta?: number, ) { /* - v = ordered js array of all cell id's From ccc1c25d64332814cad9fb27870bf85f5294041e Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 2 Aug 2025 05:12:15 +0000 Subject: [PATCH 163/798] file explorer -- fix bug when hitting enter in action box --- .../frontend/project/explorer/action-box.tsx | 25 +++++++++++-------- .../project/explorer/create-archive.tsx | 5 ++-- .../frontend/project/explorer/download.tsx | 6 ++--- .../frontend/project/explorer/explorer.tsx | 12 +++++++-- .../frontend/project/explorer/rename-file.tsx | 6 ++--- src/packages/frontend/project_actions.ts | 2 +- 6 files changed, 34 insertions(+), 22 deletions(-) diff --git a/src/packages/frontend/project/explorer/action-box.tsx b/src/packages/frontend/project/explorer/action-box.tsx index 110bdf2e5b..14aec4cfb6 100644 --- a/src/packages/frontend/project/explorer/action-box.tsx +++ b/src/packages/frontend/project/explorer/action-box.tsx @@ -84,8 +84,15 @@ export function ActionBox({ compute_server_id ?? 0, ); + function clear() { + actions.set_all_files_unchecked(); + setTimeout(() => { + actions.set_file_action(); + }, 1); + } + function cancel_action(): void { - actions.set_file_action(); + clear(); } function action_key(e): void { @@ -121,8 +128,7 @@ export function ActionBox({ actions.close_tab(path); } actions.delete_files({ paths }); - actions.set_file_action(); - actions.set_all_files_unchecked(); + clear(); } function render_delete_warning() { @@ -195,8 +201,7 @@ export function ActionBox({ src: checked_files.toArray(), dest: move_destination, }); - actions.set_file_action(); - actions.set_all_files_unchecked(); + clear(); } function valid_move_input(): boolean { @@ -340,7 +345,7 @@ export function ActionBox({ } } - actions.set_file_action(); + clear(); } function valid_copy_input(): boolean { @@ -561,17 +566,17 @@ export function ActionBox({ function render_action_box(action: FileAction) { switch (action) { case "compress": - return ; + return ; case "copy": return render_copy(); case "delete": return render_delete(); case "download": - return ; + return ; case "rename": - return ; + return ; case "duplicate": - return ; + return ; case "move": return render_move(); case "share": diff --git a/src/packages/frontend/project/explorer/create-archive.tsx b/src/packages/frontend/project/explorer/create-archive.tsx index b599de0761..1ae72fb230 100644 --- a/src/packages/frontend/project/explorer/create-archive.tsx +++ b/src/packages/frontend/project/explorer/create-archive.tsx @@ -9,7 +9,7 @@ import { useProjectContext } from "@cocalc/frontend/project/context"; import { path_split, plural } from "@cocalc/util/misc"; import CheckedFiles from "./checked-files"; -export default function CreateArchive({}) { +export default function CreateArchive({ clear }) { const intl = useIntl(); const inputRef = useRef(null); const { actions } = useProjectContext(); @@ -53,8 +53,7 @@ export default function CreateArchive({}) { setLoading(false); } - actions.set_all_files_unchecked(); - actions.set_file_action(); + clear(); }; if (actions == null) { diff --git a/src/packages/frontend/project/explorer/download.tsx b/src/packages/frontend/project/explorer/download.tsx index 0bf5c11a42..2c213e2e1f 100644 --- a/src/packages/frontend/project/explorer/download.tsx +++ b/src/packages/frontend/project/explorer/download.tsx @@ -12,7 +12,7 @@ import { path_split, path_to_file, plural } from "@cocalc/util/misc"; import { PRE_STYLE } from "./action-box"; import CheckedFiles from "./checked-files"; -export default function Download() { +export default function Download({ clear }) { const intl = useIntl(); const inputRef = useRef(null); const { actions } = useProjectContext(); @@ -98,8 +98,8 @@ export default function Download() { } finally { setLoading(false); } - actions.set_all_files_unchecked(); - actions.set_file_action(); + + clear(); }; if (actions == null) { diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index 4694e6ac5d..83b65339d0 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -160,10 +160,12 @@ export function Explorer() { }, [listing, current_path, strippedPublicPaths]); useEffect(() => { - if (listing == null) { + if (listing == null || file_action) { return; } + console.log("enabling key handler"); const handle_files_key_down = (e): void => { + console.log("key down", e.key); if (actions == null) { return; } @@ -179,6 +181,11 @@ export function Explorer() { } else if (e.key == "ArrowDown") { actions.increment_selected_file_index(); } else if (e.key == "Enter") { + console.log("Enter key", checked_files.size, file_action); + if (checked_files.size > 0 && file_action != undefined) { + // using the action box. + return; + } if (file_search.startsWith("/")) { // running a terminal command return; @@ -211,10 +218,11 @@ export function Explorer() { $(window).on("keydown", handle_files_key_down); $(window).on("keyup", handle_files_key_up); return () => { + console.log("disabling key handler"); $(window).off("keydown", handle_files_key_down); $(window).off("keyup", handle_files_key_up); }; - }, [project_id, current_path, listing]); + }, [project_id, current_path, listing, file_action]); if (listingError) { return ; diff --git a/src/packages/frontend/project/explorer/rename-file.tsx b/src/packages/frontend/project/explorer/rename-file.tsx index 327427126f..4bcc06bcbb 100644 --- a/src/packages/frontend/project/explorer/rename-file.tsx +++ b/src/packages/frontend/project/explorer/rename-file.tsx @@ -18,9 +18,10 @@ const MAX_FILENAME_LENGTH = 4095; interface Props { duplicate?: boolean; + clear: () => void; } -export default function RenameFile({ duplicate }: Props) { +export default function RenameFile({ duplicate, clear }: Props) { const intl = useIntl(); const inputRef = useRef(null); const { actions } = useProjectContext(); @@ -85,8 +86,7 @@ export default function RenameFile({ duplicate }: Props) { } finally { setLoading(false); } - actions.set_all_files_unchecked(); - actions.set_file_action(); + clear(); }; if (actions == null) { diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index d16ddbf3f6..6e18614b23 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -1722,7 +1722,6 @@ export class ProjectActions extends Actions { } this.setState({ checked_files: store.get("checked_files").clear(), - file_action: undefined, }); } @@ -1747,6 +1746,7 @@ export class ProjectActions extends Actions { }; set_file_action = (action?: FileAction): void => { + console.trace("set_file_action", action); const store = this.get_store(); if (store == null) { return; From bcc49258330aa0feb2e77e3c9ff72eb5e4a3d5d9 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 2 Aug 2025 15:01:21 +0000 Subject: [PATCH 164/798] remove debug messages --- src/packages/frontend/project/explorer/explorer.tsx | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index 83b65339d0..2c2968b0eb 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -163,9 +163,7 @@ export function Explorer() { if (listing == null || file_action) { return; } - console.log("enabling key handler"); const handle_files_key_down = (e): void => { - console.log("key down", e.key); if (actions == null) { return; } @@ -181,7 +179,6 @@ export function Explorer() { } else if (e.key == "ArrowDown") { actions.increment_selected_file_index(); } else if (e.key == "Enter") { - console.log("Enter key", checked_files.size, file_action); if (checked_files.size > 0 && file_action != undefined) { // using the action box. return; @@ -218,7 +215,6 @@ export function Explorer() { $(window).on("keydown", handle_files_key_down); $(window).on("keyup", handle_files_key_up); return () => { - console.log("disabling key handler"); $(window).off("keydown", handle_files_key_down); $(window).off("keyup", handle_files_key_up); }; From 2f8f7f074a5ccfbf41caae8c65a7f15ec1184ad4 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 2 Aug 2025 16:12:57 +0000 Subject: [PATCH 165/798] add ripgrep support to the fs module (not used by frontend yet) --- src/packages/backend/files/sandbox/index.ts | 59 ++- src/packages/backend/files/sandbox/ripgrep.ts | 344 ++++++++++++++++++ src/packages/backend/package.json | 4 +- src/packages/conat/files/fs.ts | 23 ++ src/packages/package.json | 4 +- src/packages/pnpm-lock.yaml | 39 ++ 6 files changed, 461 insertions(+), 12 deletions(-) create mode 100644 src/packages/backend/files/sandbox/ripgrep.ts diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index 52749163fc..61003ced40 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -70,12 +70,16 @@ import { replace_all } from "@cocalc/util/misc"; import { EventIterator } from "@cocalc/util/event-iterator"; import { type WatchOptions } from "@cocalc/conat/files/watch"; import find, { type FindOptions } from "./find"; +import ripgrep, { type RipgrepOptions } from "./ripgrep"; -// max time a user find request can run -- this can cause excessive +// max time a user find request can run (in safe mode) -- this can cause excessive // load on a server if there were a directory with a massive number of files, // so must be limited. const MAX_FIND_TIMEOUT = 3000; +// max time a user ripgrep can run (when in safe mode) +const MAX_RIPGREP_TIMEOUT = 3000; + interface Options { // unsafeMode -- if true, assume security model where user is running this // themself, e.g., in a project, so no security is needed at all. @@ -204,17 +208,42 @@ export class SandboxedFilesystem { printf: string, options?: FindOptions, ): Promise<{ stdout: Buffer; truncated: boolean }> => { - options = { ...options }; - if ( - !this.unsafeMode && - (!options.timeout || options.timeout > MAX_FIND_TIMEOUT) - ) { - options.timeout = MAX_FIND_TIMEOUT; - } - + options = { + ...options, + timeout: capTimeout(options?.timeout, MAX_FIND_TIMEOUT), + }; return await find(await this.safeAbsPath(path), printf, options); }; + ripgrep = async ( + path: string, + regexp: string, + options?: RipgrepOptions, + ): Promise<{ + stdout: Buffer; + stderr: Buffer; + code: number | null; + truncated: boolean; + }> => { + if (this.unsafeMode) { + // unsafeMode = slightly less locked down... + return await ripgrep(path, regexp, { + timeout: options?.timeout, + options: options?.options, + allowedBasePath: "/", + }); + } + options = { + ...options, + timeout: capTimeout(options?.timeout, MAX_RIPGREP_TIMEOUT), + }; + return await ripgrep(await this.safeAbsPath(path), regexp, { + timeout: capTimeout(options?.timeout, MAX_RIPGREP_TIMEOUT), + options: options?.options, + allowedBasePath: this.path, + }); + }; + // hard link link = async (existingPath: string, newPath: string) => { this.assertWritable(newPath); @@ -357,3 +386,15 @@ export class SandboxError extends Error { this.path = path; } } + +function capTimeout(timeout: any, max: number): number { + try { + timeout = parseFloat(timeout); + } catch { + return max; + } + if (!isFinite(timeout)) { + return max; + } + return Math.min(timeout, max); +} diff --git a/src/packages/backend/files/sandbox/ripgrep.ts b/src/packages/backend/files/sandbox/ripgrep.ts new file mode 100644 index 0000000000..ab5c04824f --- /dev/null +++ b/src/packages/backend/files/sandbox/ripgrep.ts @@ -0,0 +1,344 @@ +import { spawn } from "node:child_process"; +import { realpath } from "node:fs/promises"; +import * as path from "node:path"; +import type { RipgrepOptions } from "@cocalc/conat/files/fs"; +export type { RipgrepOptions }; + +const MAX_OUTPUT_SIZE = 10 * 1024 * 1024; // 10MB limit + +// Safely allowed options that don't pose security risks +const SAFE_OPTIONS = new Set([ + // Search behavior + "--case-sensitive", + "-s", + "--ignore-case", + "-i", + "--word-regexp", + "-w", + "--line-number", + "-n", + "--count", + "-c", + "--files-with-matches", + "-l", + "--files-without-match", + "--fixed-strings", + "-F", + "--invert-match", + "-v", + + // Output format + "--heading", + "--no-heading", + "--column", + "--pretty", + "--color", + "--no-line-number", + "-N", + + // Context lines (safe as long as we control the path) + "--context", + "-C", + "--before-context", + "-B", + "--after-context", + "-A", + + // Performance/filtering + "--max-count", + "-m", + "--max-depth", + "--max-filesize", + "--type", + "-t", + "--type-not", + "-T", + "--glob", + "-g", + "--iglob", + + // File selection + "--no-ignore", + "--hidden", + "--one-file-system", + "--null-data", + "--multiline", + "-U", + "--multiline-dotall", + "--crlf", + "--encoding", + "-E", + "--no-encoding", +]); + +// Options that take values - need special validation +const OPTIONS_WITH_VALUES = new Set([ + "--max-count", + "-m", + "--max-depth", + "--max-filesize", + "--type", + "-t", + "--type-not", + "-T", + "--glob", + "-g", + "--iglob", + "--context", + "-C", + "--before-context", + "-B", + "--after-context", + "-A", + "--encoding", + "-E", + "--color", +]); + +interface ExtendedRipgrepOptions extends RipgrepOptions { + options?: string[]; + allowedBasePath?: string; // The base path users are allowed to search within +} + +function validateGlobPattern(pattern: string): boolean { + // Reject patterns that could escape directory + if (pattern.includes("../") || pattern.includes("..\\")) { + return false; + } + // Reject absolute paths + if (path.isAbsolute(pattern)) { + return false; + } + return true; +} + +function validateNumber(value: string): boolean { + return /^\d+$/.test(value); +} + +function validateEncoding(value: string): boolean { + // Allow only safe encodings + const safeEncodings = [ + "utf-8", + "utf-16", + "utf-16le", + "utf-16be", + "ascii", + "latin-1", + ]; + return safeEncodings.includes(value.toLowerCase()); +} + +function parseAndValidateOptions(options: string[]): string[] { + const validatedOptions: string[] = []; + let i = 0; + + while (i < options.length) { + const opt = options[i]; + + // Check if this is a safe option + if (!SAFE_OPTIONS.has(opt)) { + throw new Error(`Disallowed option: ${opt}`); + } + + validatedOptions.push(opt); + + // Handle options that take values + if (OPTIONS_WITH_VALUES.has(opt)) { + i++; + if (i >= options.length) { + throw new Error(`Option ${opt} requires a value`); + } + + const value = options[i]; + + // Validate based on option type + if (opt === "--glob" || opt === "-g" || opt === "--iglob") { + if (!validateGlobPattern(value)) { + throw new Error(`Invalid glob pattern: ${value}`); + } + } else if ( + opt === "--max-count" || + opt === "-m" || + opt === "--max-depth" || + opt === "--context" || + opt === "-C" || + opt === "--before-context" || + opt === "-B" || + opt === "--after-context" || + opt === "-A" + ) { + if (!validateNumber(value)) { + throw new Error(`Invalid number for ${opt}: ${value}`); + } + } else if (opt === "--encoding" || opt === "-E") { + if (!validateEncoding(value)) { + throw new Error(`Invalid encoding: ${value}`); + } + } else if (opt === "--color") { + if (!["never", "auto", "always", "ansi"].includes(value)) { + throw new Error(`Invalid color option: ${value}`); + } + } + + validatedOptions.push(value); + } + + i++; + } + + return validatedOptions; +} + +export default async function ripgrep( + searchPath: string, + regexp: string, + { timeout = 0, options = [], allowedBasePath }: ExtendedRipgrepOptions = {}, +): Promise<{ + stdout: Buffer; + stderr: Buffer; + code: number | null; + truncated: boolean; +}> { + if (!searchPath) { + throw Error("path must be specified"); + } + if (!regexp) { + throw Error("regexp must be specified"); + } + + // Validate and normalize the search path + let normalizedPath: string; + try { + // Resolve to real path (follows symlinks to get actual path) + normalizedPath = await realpath(searchPath); + } catch (err) { + // If path doesn't exist, use normalize to check it + normalizedPath = path.normalize(searchPath); + } + + // Security check: ensure path is within allowed base path + if (allowedBasePath) { + const normalizedBase = await realpath(allowedBasePath); + const relative = path.relative(normalizedBase, normalizedPath); + + // If relative path starts with .. or is absolute, it's outside allowed path + if (relative.startsWith("..") || path.isAbsolute(relative)) { + throw new Error("Search path is outside allowed directory"); + } + } + + // Validate regexp doesn't contain null bytes (command injection protection) + if (regexp.includes("\0")) { + throw new Error("Invalid regexp: contains null bytes"); + } + + // Build arguments array with security flags first + const args = [ + "--no-follow", // Don't follow symlinks + "--no-config", // Ignore config files + "--no-ignore-global", // Don't use global gitignore + "--no-require-git", // Don't require git repo + "--no-messages", // Suppress error messages that might leak info + ]; + + // Add validated user options + if (options.length > 0) { + const validatedOptions = parseAndValidateOptions(options); + args.push(...validatedOptions); + } + + // Add the search pattern and path last + args.push("--", regexp, normalizedPath); // -- prevents regexp from being treated as option + + return new Promise((resolve, reject) => { + const stdoutChunks: Buffer[] = []; + const stderrChunks: Buffer[] = []; + let truncated = false; + let stdoutSize = 0; + let stderrSize = 0; + + const child = spawn("rg", args, { + stdio: ["ignore", "pipe", "pipe"], + env: { + // Minimal environment - only what ripgrep needs + PATH: process.env.PATH, + HOME: "/tmp", // Prevent access to user's home + RIPGREP_CONFIG_PATH: "/dev/null", // Explicitly disable config + }, + cwd: allowedBasePath || process.cwd(), // Restrict working directory + }); + + let timeoutHandle: NodeJS.Timeout | null = null; + + if (timeout > 0) { + timeoutHandle = setTimeout(() => { + truncated = true; + child.kill("SIGTERM"); + // Force kill after grace period + setTimeout(() => { + if (!child.killed) { + child.kill("SIGKILL"); + } + }, 1000); + }, timeout); + } + + child.stdout.on("data", (chunk: Buffer) => { + stdoutSize += chunk.length; + if (stdoutSize > MAX_OUTPUT_SIZE) { + truncated = true; + child.kill("SIGTERM"); + return; + } + stdoutChunks.push(chunk); + }); + + child.stderr.on("data", (chunk: Buffer) => { + stderrSize += chunk.length; + if (stderrSize > MAX_OUTPUT_SIZE) { + truncated = true; + child.kill("SIGTERM"); + return; + } + stderrChunks.push(chunk); + }); + + child.on("error", (err) => { + if (timeoutHandle) clearTimeout(timeoutHandle); + reject(err); + }); + + child.on("close", (code) => { + if (timeoutHandle) clearTimeout(timeoutHandle); + + const stdout = Buffer.concat(stdoutChunks); + const stderr = Buffer.concat(stderrChunks); + + // Truncate output if it's too large + const finalStdout = + stdout.length > MAX_OUTPUT_SIZE + ? stdout.slice(0, MAX_OUTPUT_SIZE) + : stdout; + const finalStderr = + stderr.length > MAX_OUTPUT_SIZE + ? stderr.slice(0, MAX_OUTPUT_SIZE) + : stderr; + + resolve({ + stdout: finalStdout, + stderr: finalStderr, + code, + truncated, + }); + }); + }); +} + +// Export utility functions for testing +export const _internal = { + validateGlobPattern, + validateNumber, + validateEncoding, + parseAndValidateOptions, +}; diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index 48c602b8a1..754945c3b2 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -17,7 +17,8 @@ "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf dist node_modules", - "build": "pnpm exec tsc --build", + "install-ripgrep": "echo 'require(\"@vscode/ripgrep/lib/postinstall\")' | node", + "build": "pnpm install-ripgrep && pnpm exec tsc --build", "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", "test": "pnpm exec jest --forceExit", "test-conat": " pnpm exec jest --forceExit conat", @@ -37,6 +38,7 @@ "@cocalc/backend": "workspace:*", "@cocalc/conat": "workspace:*", "@cocalc/util": "workspace:*", + "@vscode/ripgrep": "^1.15.14", "awaiting": "^3.0.0", "better-sqlite3": "^11.10.0", "chokidar": "^3.6.0", diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index ea7f567f10..284593b795 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -17,6 +17,11 @@ import { isValidUUID } from "@cocalc/util/misc"; export const DEFAULT_FILE_SERVICE = "fs"; +export interface RipgrepOptions { + timeout?: number; + options?: string[]; +} + export interface FindOptions { // timeout is very limited (e.g., 3s?) if fs is running on file // server (not in own project) @@ -76,13 +81,28 @@ export interface Filesystem { // arbitrary directory listing info, which is just not possible // with the fs API, but required in any serious application. // find -P {path} -maxdepth 1 -mindepth 1 -printf {printf} + // For security reasons, this does not support all find arguments, + // and can only use limited resources. find: ( path: string, printf: string, options?: FindOptions, ) => Promise<{ stdout: Buffer; truncated: boolean }>; + // Convenience function that uses the find and stat support to + // provide all files in a directory by using tricky options to find, + // and ensuring they are used by stat in a consistent way for updates. listing?: (path: string) => Promise; + + // We add ripgrep, as a 1-call way to very efficiently search in files + // directly on whatever is serving files. + // For security reasons, this does not support all ripgrep arguments, + // and can only use limited resources. + ripgrep: ( + path: string, + regexp: string, + options?: RipgrepOptions, + ) => Promise<{ stdout: Buffer; stderr: Buffer; truncated: boolean }>; } interface IDirent { @@ -272,6 +292,9 @@ export async function fsServer({ service, fs, client, project_id }: Options) { async rename(oldPath: string, newPath: string) { await (await fs(this.subject)).rename(oldPath, newPath); }, + async ripgrep(path: string, regexp: string, options?: RipgrepOptions) { + return await (await fs(this.subject)).ripgrep(path, regexp, options); + }, async rm(path: string, options?) { await (await fs(this.subject)).rm(path, options); }, diff --git a/src/packages/package.json b/src/packages/package.json index ec02a0d467..7dc7a7d386 100644 --- a/src/packages/package.json +++ b/src/packages/package.json @@ -31,10 +31,10 @@ "tar-fs@3.0.8": "3.0.9" }, "onlyBuiltDependencies": [ + "@vscode/ripgrep", + "better-sqlite3", "websocket-sftp", "websocketfs", - "zeromq", - "better-sqlite3", "zstd-napi" ] } diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 7628085fb0..b6d5458ae6 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -87,6 +87,9 @@ importers: '@cocalc/util': specifier: workspace:* version: link:../util + '@vscode/ripgrep': + specifier: ^1.15.14 + version: 1.15.14 awaiting: specifier: ^3.0.0 version: 3.0.0 @@ -4568,6 +4571,9 @@ packages: peerDependencies: react: '>= 16.8.0' + '@vscode/ripgrep@1.15.14': + resolution: {integrity: sha512-/G1UJPYlm+trBWQ6cMO3sv6b8D1+G16WaJH1/DSqw32JOVlzgZbLkDxRyzIpTpv30AcYGMkCf5tUqGlW6HbDWw==} + '@vscode/vscode-languagedetection@1.0.22': resolution: {integrity: sha512-rQ/BgMyLuIXSmbA0MSkIPHtcOw14QkeDbAq19sjvaS9LTRr905yij0S8lsyqN5JgOsbtIx7pAcyOxFMzPmqhZQ==} hasBin: true @@ -5137,6 +5143,9 @@ packages: bser@2.1.1: resolution: {integrity: sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==} + buffer-crc32@0.2.13: + resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==} + buffer-equal-constant-time@1.0.1: resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} @@ -6710,6 +6719,9 @@ packages: fb-watchman@2.0.2: resolution: {integrity: sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==} + fd-slicer@1.1.0: + resolution: {integrity: sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==} + fdir@6.4.6: resolution: {integrity: sha512-hiFoqpyZcfNm1yc4u8oWCf9A2c4D3QjCrks3zmoVKVxpQRzmPNar1hUJcBG2RQHvEVGDN+Jm81ZheVLAQMK6+w==} peerDependencies: @@ -9263,6 +9275,9 @@ packages: resolution: {integrity: sha512-/Y3fcFrXEAsMjJXeL9J8+ZG9U01LbuWaYypvDW2ycW1jL269L3js3DVBjDJ0Up9Np1uqDXsDrRihHANhZOlwdQ==} engines: {node: '>=20'} + pend@1.2.0: + resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==} + performance-now@2.1.0: resolution: {integrity: sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==} @@ -11734,6 +11749,9 @@ packages: resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==} engines: {node: '>=12'} + yauzl@2.10.0: + resolution: {integrity: sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==} + yjs@13.6.27: resolution: {integrity: sha512-OIDwaflOaq4wC6YlPBy2L6ceKeKuF7DeTxx+jPzv1FHn9tCZ0ZwSRnUBxD05E3yed46fv/FWJbvR+Ud7x0L7zw==} engines: {node: '>=16.0.0', npm: '>=8.0.0'} @@ -14904,6 +14922,14 @@ snapshots: '@use-gesture/core': 10.3.1 react: 19.1.0 + '@vscode/ripgrep@1.15.14': + dependencies: + https-proxy-agent: 7.0.6 + proxy-from-env: 1.1.0 + yauzl: 2.10.0 + transitivePeerDependencies: + - supports-color + '@vscode/vscode-languagedetection@1.0.22': {} '@webassemblyjs/ast@1.14.1': @@ -15571,6 +15597,8 @@ snapshots: dependencies: node-int64: 0.4.0 + buffer-crc32@0.2.13: {} + buffer-equal-constant-time@1.0.1: {} buffer-from@1.1.2: {} @@ -17423,6 +17451,10 @@ snapshots: dependencies: bser: 2.1.1 + fd-slicer@1.1.0: + dependencies: + pend: 1.2.0 + fdir@6.4.6(picomatch@4.0.2): optionalDependencies: picomatch: 4.0.2 @@ -20524,6 +20556,8 @@ snapshots: optionalDependencies: '@napi-rs/canvas': 0.1.73 + pend@1.2.0: {} + performance-now@2.1.0: {} pg-cloudflare@1.2.7: @@ -23457,6 +23491,11 @@ snapshots: y18n: 5.0.8 yargs-parser: 21.1.1 + yauzl@2.10.0: + dependencies: + buffer-crc32: 0.2.13 + fd-slicer: 1.1.0 + yjs@13.6.27: dependencies: lib0: 0.2.109 From ec056d89b5c97d1f8926471013ae93e2a984abbb Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 2 Aug 2025 17:19:23 +0000 Subject: [PATCH 166/798] ripgrep: automate installing it properly (not using the vscode version) --- .../backend/files/sandbox/install-ripgrep.ts | 136 ++++++++++++++++++ src/packages/backend/files/sandbox/ripgrep.ts | 13 +- src/packages/backend/package.json | 4 +- 3 files changed, 145 insertions(+), 8 deletions(-) create mode 100644 src/packages/backend/files/sandbox/install-ripgrep.ts diff --git a/src/packages/backend/files/sandbox/install-ripgrep.ts b/src/packages/backend/files/sandbox/install-ripgrep.ts new file mode 100644 index 0000000000..506dbd9dc6 --- /dev/null +++ b/src/packages/backend/files/sandbox/install-ripgrep.ts @@ -0,0 +1,136 @@ +/* +Download a ripgrep binary. + +This supports: + +- x86_64 Linux +- aarch64 Linux +- arm64 macos + +This assumes tar is installed. + +NOTE: There are several npm modules that purport to install ripgrep. We do not use +https://www.npmjs.com/package/@vscode/ripgrep because it is not properly maintained, +e.g., + - security vulnerabilities: https://github.com/microsoft/ripgrep-prebuilt/issues/48 + - not updated to a major new release without a good reason: https://github.com/microsoft/ripgrep-prebuilt/issues/38 +*/ + +import { arch, platform } from "os"; +import { exists } from "@cocalc/backend/misc/async-utils-node"; +import { execFileSync } from "child_process"; +import { writeFile, unlink, chmod } from "fs/promises"; +import { join } from "path"; + +// See https://github.com/BurntSushi/ripgrep/releases +const VERSION = "14.1.1"; +const BASE = "https://github.com/BurntSushi/ripgrep/releases/download"; + +export const rgPath = join(__dirname, "rg"); + +export async function install() { + if (await exists(rgPath)) { + return; + } + const url = getUrl(); + // - 1. Fetch the tarball from the github url (using the fetch library) + const response = await downloadFromGithub(url); + const tarballBuffer = Buffer.from(await response.arrayBuffer()); + + // - 2. Extract the file "rg" from the tarball to ${__dirname}/rg + // The tarball contains this one file "rg" at the top level, i.e., for + // ripgrep-14.1.1-x86_64-unknown-linux-musl.tar.gz + // we have "tar tvf ripgrep-14.1.1-x86_64-unknown-linux-musl.tar.gz" outputs + // ... + // ripgrep-14.1.1-x86_64-unknown-linux-musl/rg + const tmpFile = join(__dirname, `ripgrep-${VERSION}.tar.gz`); + await writeFile(tmpFile, tarballBuffer); + // sync is fine since this is run at *build time*. + execFileSync("tar", [ + "xzf", + tmpFile, + "--strip-components=1", + `-C`, + __dirname, + `ripgrep-${VERSION}-${getName()}/rg`, + ]); + await unlink(tmpFile); + + // - 3. Make the file rg executable + await chmod(rgPath, 0o755); +} + +// Download from github, but aware of rate limits, the retry-after header, etc. +async function downloadFromGithub(url: string) { + const maxRetries = 10; + const baseDelay = 1000; // 1 second + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + const res = await fetch(url); + + if (res.status === 429) { + // Rate limit error + if (attempt === maxRetries) { + throw new Error("Rate limit exceeded after max retries"); + } + + const retryAfter = res.headers.get("retry-after"); + const delay = retryAfter + ? parseInt(retryAfter) * 1000 + : baseDelay * Math.pow(2, attempt - 1); // Exponential backoff + + console.log( + `Rate limited. Retrying in ${delay}ms (attempt ${attempt}/${maxRetries})`, + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + continue; + } + + if (!res.ok) { + throw new Error(`HTTP ${res.status}: ${res.statusText}`); + } + return res; + } catch (error) { + if (attempt === maxRetries) { + throw error; + } + + const delay = baseDelay * Math.pow(2, attempt - 1); + console.log( + `Fetch failed. Retrying in ${delay}ms (attempt ${attempt}/${maxRetries})`, + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + } + } + throw new Error("Should not reach here"); +} + +function getUrl() { + return `${BASE}/${VERSION}/ripgrep-${VERSION}-${getName()}.tar.gz`; +} + +function getName() { + switch (platform()) { + case "linux": + switch (arch()) { + case "x64": + return "x86_64-unknown-linux-musl"; + case "arm64": + return "aarch64-unknown-linux-gnu"; + default: + throw Error(`unsupported arch '${arch()}'`); + } + case "darwin": + switch (arch()) { + case "x64": + return "x86_64-apple-darwin"; + case "arm64": + return "aarch64-apple-darwin"; + default: + throw Error(`unsupported arch '${arch()}'`); + } + default: + throw Error(`unsupported platform '${platform()}'`); + } +} diff --git a/src/packages/backend/files/sandbox/ripgrep.ts b/src/packages/backend/files/sandbox/ripgrep.ts index ab5c04824f..5e587a2c2f 100644 --- a/src/packages/backend/files/sandbox/ripgrep.ts +++ b/src/packages/backend/files/sandbox/ripgrep.ts @@ -3,6 +3,7 @@ import { realpath } from "node:fs/promises"; import * as path from "node:path"; import type { RipgrepOptions } from "@cocalc/conat/files/fs"; export type { RipgrepOptions }; +import { rgPath } from "./install-ripgrep"; const MAX_OUTPUT_SIZE = 10 * 1024 * 1024; // 10MB limit @@ -69,6 +70,9 @@ const SAFE_OPTIONS = new Set([ "--encoding", "-E", "--no-encoding", + + // basic info + "--version", ]); // Options that take values - need special validation @@ -180,13 +184,10 @@ function parseAndValidateOptions(options: string[]): string[] { throw new Error(`Invalid color option: ${value}`); } } - validatedOptions.push(value); } - i++; } - return validatedOptions; } @@ -200,10 +201,10 @@ export default async function ripgrep( code: number | null; truncated: boolean; }> { - if (!searchPath) { + if (searchPath == null) { throw Error("path must be specified"); } - if (!regexp) { + if (regexp == null) { throw Error("regexp must be specified"); } @@ -258,7 +259,7 @@ export default async function ripgrep( let stdoutSize = 0; let stderrSize = 0; - const child = spawn("rg", args, { + const child = spawn(rgPath, args, { stdio: ["ignore", "pipe", "pipe"], env: { // Minimal environment - only what ripgrep needs diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index 754945c3b2..3cf22ef11b 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -17,8 +17,8 @@ "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf dist node_modules", - "install-ripgrep": "echo 'require(\"@vscode/ripgrep/lib/postinstall\")' | node", - "build": "pnpm install-ripgrep && pnpm exec tsc --build", + "install-ripgrep": "echo 'require(\"@cocalc/backend/files/sandbox/install-ripgrep\").install()' | node", + "build": "pnpm exec tsc --build && pnpm install-ripgrep", "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", "test": "pnpm exec jest --forceExit", "test-conat": " pnpm exec jest --forceExit conat", From bbd52a0fec380f00f339337aa4b107b032824804 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 2 Aug 2025 10:38:43 -0700 Subject: [PATCH 167/798] update better-sqlite3 and zstd-napi (both broke building on macos) --- src/packages/backend/package.json | 14 +++++++++++--- src/packages/pnpm-lock.yaml | 11 ++++++----- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index 3cf22ef11b..861ad94254 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -13,7 +13,10 @@ "./auth/*": "./dist/auth/*.js", "./auth/tokens/*": "./dist/auth/tokens/*.js" }, - "keywords": ["utilities", "cocalc"], + "keywords": [ + "utilities", + "cocalc" + ], "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf dist node_modules", @@ -31,7 +34,12 @@ "conat-persist": "DEBUG=cocalc:* node ./bin/conat-persist.cjs", "conat-test-server": "node ./bin/conat-test-server.cjs" }, - "files": ["dist/**", "bin/**", "README.md", "package.json"], + "files": [ + "dist/**", + "bin/**", + "README.md", + "package.json" + ], "author": "SageMath, Inc.", "license": "SEE LICENSE.md", "dependencies": { @@ -40,7 +48,7 @@ "@cocalc/util": "workspace:*", "@vscode/ripgrep": "^1.15.14", "awaiting": "^3.0.0", - "better-sqlite3": "^11.10.0", + "better-sqlite3": "^12.2.0", "chokidar": "^3.6.0", "debug": "^4.4.0", "fs-extra": "^11.2.0", diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index b6d5458ae6..8781b7180e 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -94,8 +94,8 @@ importers: specifier: ^3.0.0 version: 3.0.0 better-sqlite3: - specifier: ^11.10.0 - version: 11.10.0 + specifier: ^12.2.0 + version: 12.2.0 chokidar: specifier: ^3.6.0 version: 3.6.0 @@ -5056,8 +5056,9 @@ packages: batch@0.6.1: resolution: {integrity: sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==} - better-sqlite3@11.10.0: - resolution: {integrity: sha512-EwhOpyXiOEL/lKzHz9AW1msWFNzGc/z+LzeB3/jnFJpxu+th2yqvzsSWas1v9jgs9+xiXJcD5A8CJxAG2TaghQ==} + better-sqlite3@12.2.0: + resolution: {integrity: sha512-eGbYq2CT+tos1fBwLQ/tkBt9J5M3JEHjku4hbvQUePCckkvVf14xWj+1m7dGoK81M/fOjFT7yM9UMeKT/+vFLQ==} + engines: {node: 20.x || 22.x || 23.x || 24.x} big.js@3.2.0: resolution: {integrity: sha512-+hN/Zh2D08Mx65pZ/4g5bsmNiZUuChDiQfTUQ7qJr4/kuopCr88xZsAXv6mBoZEsUI4OuGHlX59qE94K2mMW8Q==} @@ -15497,7 +15498,7 @@ snapshots: batch@0.6.1: {} - better-sqlite3@11.10.0: + better-sqlite3@12.2.0: dependencies: bindings: 1.5.0 prebuild-install: 7.1.3 From 144f647dd59f7e3654592469d7df623ae5d6a2cb Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 2 Aug 2025 17:42:03 +0000 Subject: [PATCH 168/798] ripgrep: whitelist searching binary files --- src/packages/backend/files/sandbox/ripgrep.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/packages/backend/files/sandbox/ripgrep.ts b/src/packages/backend/files/sandbox/ripgrep.ts index 5e587a2c2f..5c7859f824 100644 --- a/src/packages/backend/files/sandbox/ripgrep.ts +++ b/src/packages/backend/files/sandbox/ripgrep.ts @@ -73,6 +73,11 @@ const SAFE_OPTIONS = new Set([ // basic info "--version", + + // allow searching in binary files (files that contain NUL) + // this should be safe, since we're capturing output to buffers. + "--text", + "-a", ]); // Options that take values - need special validation From 79dcc3c823900420e92d90ca4119ff0b8b562a51 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 2 Aug 2025 22:25:10 +0000 Subject: [PATCH 169/798] files sandbox: create generic exec command for running command with various sandboxing dimensions, but keeping api as similar to upstream as possible; use for the fd command. --- src/packages/backend/files/sandbox/exec.ts | 155 ++++++++++++++++++ src/packages/backend/files/sandbox/fd.ts | 134 +++++++++++++++ src/packages/backend/files/sandbox/find.ts | 11 ++ src/packages/backend/files/sandbox/index.ts | 36 ++-- src/packages/backend/files/sandbox/ripgrep.ts | 7 + src/packages/conat/files/fs.ts | 25 +++ 6 files changed, 355 insertions(+), 13 deletions(-) create mode 100644 src/packages/backend/files/sandbox/exec.ts create mode 100644 src/packages/backend/files/sandbox/fd.ts diff --git a/src/packages/backend/files/sandbox/exec.ts b/src/packages/backend/files/sandbox/exec.ts new file mode 100644 index 0000000000..8a829a1771 --- /dev/null +++ b/src/packages/backend/files/sandbox/exec.ts @@ -0,0 +1,155 @@ +import { spawn } from "node:child_process"; +import { arch } from "node:os"; +import { type ExecOutput } from "@cocalc/conat/files/fs"; +export { type ExecOutput }; + +const DEFAULT_TIMEOUT = 3_000; +const DEFAULT_MAX_SIZE = 10_000_000; + +export interface Options { + // the path to the command + cmd: string; + // positional arguments; these are not checked in any way, so are given after '--' for safety + positionalArgs?: string[]; + // whitelisted args flags; these are checked according to the whitelist specified below + options?: string[]; + // if given, use these options when os.arch()=='darwin' (i.e., macOS) + darwin?: string[]; + // if given, use these options when os.arch()=='linux' + linux?: string[]; + // when total size of stdout and stderr hits this amount, command is terminated, and + // truncated is set. The total amount of output may thus be slightly larger than maxOutput + maxSize?: number; + // command is terminated after this many ms + timeout?: number; + // each command line option that is explicitly whitelisted + // should be a key in the following whitelist map. + // The value can be either: + // - true: in which case the option does not take a argument, or + // - a function: in which the option takes exactly one argument; the function should validate that argument + // and throw an error if the argument is not allowed. + whitelist?: { [option: string]: true | ValidateFunction }; + // where to launch command + cwd?: string; +} + +type ValidateFunction = (value: string) => void; + +export default async function exec({ + cmd, + positionalArgs = [], + options = [], + linux = [], + darwin = [], + maxSize = DEFAULT_MAX_SIZE, + timeout = DEFAULT_TIMEOUT, + whitelist = {}, + cwd, +}: Options): Promise { + if (arch() == "darwin") { + options = options.concat(darwin); + } else if (arch() == "linux") { + options = options.concat(linux); + } + options = parseAndValidateOptions(options, whitelist); + + return new Promise((resolve, reject) => { + const stdoutChunks: Buffer[] = []; + const stderrChunks: Buffer[] = []; + let truncated = false; + let stdoutSize = 0; + let stderrSize = 0; + + const args = options.concat(["--"]).concat(positionalArgs); + const child = spawn(cmd, args, { + stdio: ["ignore", "pipe", "pipe"], + env: {}, + cwd, + }); + + let timeoutHandle: NodeJS.Timeout | null = null; + + if (timeout > 0) { + timeoutHandle = setTimeout(() => { + truncated = true; + child.kill("SIGTERM"); + // Force kill after grace period + setTimeout(() => { + if (!child.killed) { + child.kill("SIGKILL"); + } + }, 1000); + }, timeout); + } + + child.stdout.on("data", (chunk: Buffer) => { + stdoutSize += chunk.length; + if (stdoutSize + stderrSize >= maxSize) { + truncated = true; + child.kill("SIGTERM"); + return; + } + stdoutChunks.push(chunk); + }); + + child.stderr.on("data", (chunk: Buffer) => { + stderrSize += chunk.length; + if (stdoutSize + stderrSize > maxSize) { + truncated = true; + child.kill("SIGTERM"); + return; + } + stderrChunks.push(chunk); + }); + + child.on("error", (err) => { + if (timeoutHandle) { + clearTimeout(timeoutHandle); + } + reject(err); + }); + + child.once("close", (code) => { + if (timeoutHandle) { + clearTimeout(timeoutHandle); + } + + resolve({ + stdout: Buffer.concat(stdoutChunks), + stderr: Buffer.concat(stderrChunks), + code, + truncated, + }); + }); + }); +} + +function parseAndValidateOptions(options: string[], whitelist): string[] { + const validatedOptions: string[] = []; + let i = 0; + + while (i < options.length) { + const opt = options[i]; + + // Check if this is a safe option + const validate = whitelist[opt]; + if (!validate) { + throw new Error(`Disallowed option: ${opt}`); + } + validatedOptions.push(opt); + + // Handle options that take values + if (validate !== true) { + i++; + if (i >= options.length) { + throw new Error(`Option ${opt} requires a value`); + } + const value = options[i]; + validate(value); + // didn't throw, so good to go + validatedOptions.push(value); + } + i++; + } + return validatedOptions; +} diff --git a/src/packages/backend/files/sandbox/fd.ts b/src/packages/backend/files/sandbox/fd.ts new file mode 100644 index 0000000000..14551a9091 --- /dev/null +++ b/src/packages/backend/files/sandbox/fd.ts @@ -0,0 +1,134 @@ +import exec, { type ExecOutput } from "./exec"; +import { type FdOptions } from "@cocalc/conat/files/fs"; +export { type FdOptions }; + +export default async function fd( + path: string, + { options, darwin, linux, pattern, timeout, maxSize }: FdOptions, +): Promise { + if (path == null) { + throw Error("path must be specified"); + } + + return await exec({ + cmd: "/projects/6b851643-360e-435e-b87e-f9a6ab64a8b1/bin/fd", + cwd: path, + positionalArgs: pattern ? [pattern] : [], + options, + darwin, + linux, + maxSize, + timeout, + whitelist, + }); +} + +const whitelist = { + "-H": true, + "--hidden": true, + + "-I": true, + "--no-ignore": true, + + "-u": true, + "--unrestricted": true, + + "--no-ignore-vcs": true, + "--no-require-git": true, + + "-s": true, + "--case-sensitive": true, + + "-i": true, + "--ignore-case": true, + + "-g": true, + "--glob": true, + + "--regex": true, + + "-F": true, + "--fixed-strings": true, + + "--and": anyValue, + + "-l": true, + "--list-details": true, + + "-p": true, + "--full-path": true, + + "-0": true, + "--print0": true, + + "--max-results": validateInt, + + "-1": true, + + "-q": true, + "--quite": true, + + "--show-errors": true, + + "--strip-cwd-prefix": validateEnum(["never", "always", "auto"]), + + "--one-file-system": true, + "--mount": true, + "--xdev": true, + + "-h": true, + "--help": true, + + "-V": true, + "--version": true, + + "-d": validateInt, + "--max-depth": validateInt, + + "--min-depth": validateInt, + + "--exact-depth": validateInt, + + "--prune": true, + + "--type": anyValue, + + "-e": anyValue, + "--extension": anyValue, + + "-E": anyValue, + "--exclude": anyValue, + + "--ignore-file": anyValue, + + "-c": validateEnum(["never", "always", "auto"]), + "--color": validateEnum(["never", "always", "auto"]), + + "-S": anyValue, + "--size": anyValue, + + "--changed-within": anyValue, + "--changed-before": anyValue, + + "-o": anyValue, + "--owner": anyValue, + + "--format": anyValue, +} as const; + +function anyValue() {} + +function validateEnum(allowed: string[]) { + return (value: string) => { + if (!allowed.includes(value)) { + throw Error("invalid value"); + } + }; +} + +function validateInt(value: string) { + const count = parseInt(value); + if (!isFinite(count)) { + throw Error("argument must be a number"); + } +} diff --git a/src/packages/backend/files/sandbox/find.ts b/src/packages/backend/files/sandbox/find.ts index 90cd67c9bc..2f6e614ce3 100644 --- a/src/packages/backend/files/sandbox/find.ts +++ b/src/packages/backend/files/sandbox/find.ts @@ -1,3 +1,14 @@ +/* + +NOTE: there is a program https://github.com/sharkdp/fd that is a very fast +parallel rust program for finding files matching a pattern. It is complementary +to find here though, because we mainly use find to compute directory +listing info (e.g., file size, mtime, etc.), and fd does NOT do that; it can +exec ls, but that is slower than using find. So both find and fd are useful +for different tasks -- find is *better* for directory listings and fd is better +for finding filesnames in a directory tree that match a pattern. +*/ + import { spawn } from "node:child_process"; import type { FindOptions, FindExpression } from "@cocalc/conat/files/fs"; export type { FindOptions, FindExpression }; diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index 61003ced40..cf8874cdce 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -71,6 +71,8 @@ import { EventIterator } from "@cocalc/util/event-iterator"; import { type WatchOptions } from "@cocalc/conat/files/watch"; import find, { type FindOptions } from "./find"; import ripgrep, { type RipgrepOptions } from "./ripgrep"; +import fd, { type FdOptions } from "./fd"; +import { type ExecOutput } from "./exec"; // max time a user find request can run (in safe mode) -- this can cause excessive // load on a server if there were a directory with a massive number of files, @@ -80,6 +82,8 @@ const MAX_FIND_TIMEOUT = 3000; // max time a user ripgrep can run (when in safe mode) const MAX_RIPGREP_TIMEOUT = 3000; +const MAX_FD_TIMEOUT = 3000; + interface Options { // unsafeMode -- if true, assume security model where user is running this // themself, e.g., in a project, so no security is needed at all. @@ -208,13 +212,17 @@ export class SandboxedFilesystem { printf: string, options?: FindOptions, ): Promise<{ stdout: Buffer; truncated: boolean }> => { - options = { - ...options, - timeout: capTimeout(options?.timeout, MAX_FIND_TIMEOUT), - }; + options = capTimeout(options, MAX_FIND_TIMEOUT); return await find(await this.safeAbsPath(path), printf, options); }; + fd = async (path: string, options?: FdOptions): Promise => { + return await fd( + await this.safeAbsPath(path), + capTimeout(options, MAX_FD_TIMEOUT), + ); + }; + ripgrep = async ( path: string, regexp: string, @@ -233,10 +241,7 @@ export class SandboxedFilesystem { allowedBasePath: "/", }); } - options = { - ...options, - timeout: capTimeout(options?.timeout, MAX_RIPGREP_TIMEOUT), - }; + options = capTimeout(options, MAX_RIPGREP_TIMEOUT); return await ripgrep(await this.safeAbsPath(path), regexp, { timeout: capTimeout(options?.timeout, MAX_RIPGREP_TIMEOUT), options: options?.options, @@ -387,14 +392,19 @@ export class SandboxError extends Error { } } -function capTimeout(timeout: any, max: number): number { +function capTimeout(options, max: number) { + if (options == null) { + return { timeout: max }; + } + + let timeout; try { - timeout = parseFloat(timeout); + timeout = parseFloat(options.timeout); } catch { - return max; + return { ...options, timeout: max }; } if (!isFinite(timeout)) { - return max; + return { ...options, timeout: max }; } - return Math.min(timeout, max); + return { ...options, timeout: Math.min(timeout, max) }; } diff --git a/src/packages/backend/files/sandbox/ripgrep.ts b/src/packages/backend/files/sandbox/ripgrep.ts index 5c7859f824..f2ff751661 100644 --- a/src/packages/backend/files/sandbox/ripgrep.ts +++ b/src/packages/backend/files/sandbox/ripgrep.ts @@ -78,6 +78,13 @@ const SAFE_OPTIONS = new Set([ // this should be safe, since we're capturing output to buffers. "--text", "-a", + + // this ignores gitignore, hidden, and binary restrictions, which we allow + // above, so should be safe. + "--unrestricted", + "-u", + + "--debug", ]); // Options that take values - need special validation diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index 284593b795..b3ee87786f 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -17,6 +17,14 @@ import { isValidUUID } from "@cocalc/util/misc"; export const DEFAULT_FILE_SERVICE = "fs"; +export interface ExecOutput { + stdout: Buffer; + stderr: Buffer; + code: number | null; + // true if terminated early due to output size or time + truncated?: boolean; +} + export interface RipgrepOptions { timeout?: number; options?: string[]; @@ -44,6 +52,15 @@ export type FindExpression = | { type: "or"; left: FindExpression; right: FindExpression } | { type: "not"; expr: FindExpression }; +export interface FdOptions { + pattern?: string; + options?: string[]; + darwin?: string[]; + linux?: string[]; + timeout?: number; + maxSize?: number; +} + export interface Filesystem { appendFile: (path: string, data: string | Buffer, encoding?) => Promise; chmod: (path: string, mode: string | number) => Promise; @@ -94,6 +111,11 @@ export interface Filesystem { // and ensuring they are used by stat in a consistent way for updates. listing?: (path: string) => Promise; + // fd is a rust rewrite of find that is extremely fast at finding + // files that match an expression, e.g., + // options: { type: "name", pattern:"^\.DS_Store$" } + fd: (path: string, options?: FdOptions) => Promise; + // We add ripgrep, as a 1-call way to very efficiently search in files // directly on whatever is serving files. // For security reasons, this does not support all ripgrep arguments, @@ -257,6 +279,9 @@ export async function fsServer({ service, fs, client, project_id }: Options) { async exists(path: string): Promise { return await (await fs(this.subject)).exists(path); }, + async fd(path: string, options?: FdOptions) { + return await (await fs(this.subject)).fd(path, options); + }, async find(path: string, printf: string, options?: FindOptions) { return await (await fs(this.subject)).find(path, printf, options); }, From 284f91dd547a8bce8a23e6371cf146b23d27307c Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 3 Aug 2025 04:23:58 +0000 Subject: [PATCH 170/798] automate install of fd and ripgrep in a more robust way --- src/packages/backend/files/sandbox/exec.ts | 52 +- src/packages/backend/files/sandbox/fd.test.ts | 27 + src/packages/backend/files/sandbox/fd.ts | 69 +-- .../backend/files/sandbox/find.test.ts | 36 +- src/packages/backend/files/sandbox/find.ts | 257 ++++---- src/packages/backend/files/sandbox/index.ts | 40 +- .../{install-ripgrep.ts => install.ts} | 117 ++-- .../backend/files/sandbox/ripgrep.test.ts | 27 + src/packages/backend/files/sandbox/ripgrep.ts | 548 +++++++----------- src/packages/backend/package.json | 2 +- src/packages/conat/files/fs.ts | 46 +- src/packages/conat/files/listing.ts | 16 +- src/packages/frontend/project/search/body.tsx | 17 +- 13 files changed, 593 insertions(+), 661 deletions(-) create mode 100644 src/packages/backend/files/sandbox/fd.test.ts rename src/packages/backend/files/sandbox/{install-ripgrep.ts => install.ts} (57%) create mode 100644 src/packages/backend/files/sandbox/ripgrep.test.ts diff --git a/src/packages/backend/files/sandbox/exec.ts b/src/packages/backend/files/sandbox/exec.ts index 8a829a1771..b0d477b622 100644 --- a/src/packages/backend/files/sandbox/exec.ts +++ b/src/packages/backend/files/sandbox/exec.ts @@ -2,6 +2,9 @@ import { spawn } from "node:child_process"; import { arch } from "node:os"; import { type ExecOutput } from "@cocalc/conat/files/fs"; export { type ExecOutput }; +import getLogger from "@cocalc/backend/logger"; + +const logger = getLogger("files:sandbox:exec"); const DEFAULT_TIMEOUT = 3_000; const DEFAULT_MAX_SIZE = 10_000_000; @@ -9,13 +12,15 @@ const DEFAULT_MAX_SIZE = 10_000_000; export interface Options { // the path to the command cmd: string; - // positional arguments; these are not checked in any way, so are given after '--' for safety + // position args *before* any options; these are not sanitized + prefixArgs?: string[]; + // positional arguments; these are not sanitized, but are given after '--' for safety positionalArgs?: string[]; // whitelisted args flags; these are checked according to the whitelist specified below options?: string[]; - // if given, use these options when os.arch()=='darwin' (i.e., macOS) + // if given, use these options when os.arch()=='darwin' (i.e., macOS); these must match whitelist darwin?: string[]; - // if given, use these options when os.arch()=='linux' + // if given, use these options when os.arch()=='linux'; these must match whitelist linux?: string[]; // when total size of stdout and stderr hits this amount, command is terminated, and // truncated is set. The total amount of output may thus be slightly larger than maxOutput @@ -31,6 +36,9 @@ export interface Options { whitelist?: { [option: string]: true | ValidateFunction }; // where to launch command cwd?: string; + + // options that are always included first for safety and need NOT match whitelist + safety?: string[]; } type ValidateFunction = (value: string) => void; @@ -38,9 +46,11 @@ type ValidateFunction = (value: string) => void; export default async function exec({ cmd, positionalArgs = [], + prefixArgs = [], options = [], linux = [], darwin = [], + safety = [], maxSize = DEFAULT_MAX_SIZE, timeout = DEFAULT_TIMEOUT, whitelist = {}, @@ -51,7 +61,7 @@ export default async function exec({ } else if (arch() == "linux") { options = options.concat(linux); } - options = parseAndValidateOptions(options, whitelist); + options = safety.concat(parseAndValidateOptions(options, whitelist)); return new Promise((resolve, reject) => { const stdoutChunks: Buffer[] = []; @@ -60,7 +70,13 @@ export default async function exec({ let stdoutSize = 0; let stderrSize = 0; - const args = options.concat(["--"]).concat(positionalArgs); + const args = prefixArgs.concat(options); + if (positionalArgs.length > 0) { + args.push("--", ...positionalArgs); + } + + // console.log(`${cmd} ${args.join(" ")}`); + logger.debug({ cmd, args }); const child = spawn(cmd, args, { stdio: ["ignore", "pipe", "pipe"], env: {}, @@ -144,7 +160,7 @@ function parseAndValidateOptions(options: string[], whitelist): string[] { if (i >= options.length) { throw new Error(`Option ${opt} requires a value`); } - const value = options[i]; + const value = String(options[i]); validate(value); // didn't throw, so good to go validatedOptions.push(value); @@ -153,3 +169,27 @@ function parseAndValidateOptions(options: string[], whitelist): string[] { } return validatedOptions; } + +export const validate = { + str: () => {}, + set: (allowed) => { + allowed = new Set(allowed); + return (value: string) => { + if (!allowed.includes(value)) { + throw Error("invalid value"); + } + }; + }, + int: (value: string) => { + const x = parseInt(value); + if (!isFinite(x)) { + throw Error("argument must be a number"); + } + }, + float: (value: string) => { + const x = parseFloat(value); + if (!isFinite(x)) { + throw Error("argument must be a number"); + } + }, +}; diff --git a/src/packages/backend/files/sandbox/fd.test.ts b/src/packages/backend/files/sandbox/fd.test.ts new file mode 100644 index 0000000000..f17132d972 --- /dev/null +++ b/src/packages/backend/files/sandbox/fd.test.ts @@ -0,0 +1,27 @@ +import fd from "./fd"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let tempDir; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); +}); +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); + +describe("fd files", () => { + it("directory starts empty", async () => { + const { stdout, truncated } = await fd(tempDir); + expect(stdout.length).toBe(0); + expect(truncated).toBe(false); + }); + + it("create a file and see it appears via fd", async () => { + await writeFile(join(tempDir, "a.txt"), "hello"); + const { stdout, truncated } = await fd(tempDir); + expect(truncated).toBe(false); + expect(stdout.toString()).toEqual("a.txt\n"); + }); +}); diff --git a/src/packages/backend/files/sandbox/fd.ts b/src/packages/backend/files/sandbox/fd.ts index 14551a9091..dfc92e8e1e 100644 --- a/src/packages/backend/files/sandbox/fd.ts +++ b/src/packages/backend/files/sandbox/fd.ts @@ -1,22 +1,24 @@ -import exec, { type ExecOutput } from "./exec"; +import exec, { type ExecOutput, validate } from "./exec"; import { type FdOptions } from "@cocalc/conat/files/fs"; export { type FdOptions }; +import { fd as fdPath } from "./install"; export default async function fd( path: string, - { options, darwin, linux, pattern, timeout, maxSize }: FdOptions, + { options, darwin, linux, pattern, timeout, maxSize }: FdOptions = {}, ): Promise { if (path == null) { throw Error("path must be specified"); } return await exec({ - cmd: "/projects/6b851643-360e-435e-b87e-f9a6ab64a8b1/bin/fd", + cmd: fdPath, cwd: path, positionalArgs: pattern ? [pattern] : [], options, darwin, linux, + safety: ["--no-follow"], maxSize, timeout, whitelist, @@ -50,7 +52,7 @@ const whitelist = { "-F": true, "--fixed-strings": true, - "--and": anyValue, + "--and": validate.str, "-l": true, "--list-details": true, @@ -61,7 +63,7 @@ const whitelist = { "-0": true, "--print0": true, - "--max-results": validateInt, + "--max-results": validate.int, "-1": true, @@ -70,7 +72,7 @@ const whitelist = { "--show-errors": true, - "--strip-cwd-prefix": validateEnum(["never", "always", "auto"]), + "--strip-cwd-prefix": validate.set(["never", "always", "auto"]), "--one-file-system": true, "--mount": true, @@ -82,53 +84,36 @@ const whitelist = { "-V": true, "--version": true, - "-d": validateInt, - "--max-depth": validateInt, + "-d": validate.int, + "--max-depth": validate.int, - "--min-depth": validateInt, + "--min-depth": validate.int, - "--exact-depth": validateInt, + "--exact-depth": validate.int, "--prune": true, - "--type": anyValue, + "--type": validate.str, - "-e": anyValue, - "--extension": anyValue, + "-e": validate.str, + "--extension": validate.str, - "-E": anyValue, - "--exclude": anyValue, + "-E": validate.str, + "--exclude": validate.str, - "--ignore-file": anyValue, + "--ignore-file": validate.str, - "-c": validateEnum(["never", "always", "auto"]), - "--color": validateEnum(["never", "always", "auto"]), + "-c": validate.set(["never", "always", "auto"]), + "--color": validate.set(["never", "always", "auto"]), - "-S": anyValue, - "--size": anyValue, + "-S": validate.str, + "--size": validate.str, - "--changed-within": anyValue, - "--changed-before": anyValue, + "--changed-within": validate.str, + "--changed-before": validate.str, - "-o": anyValue, - "--owner": anyValue, + "-o": validate.str, + "--owner": validate.str, - "--format": anyValue, + "--format": validate.str, } as const; - -function anyValue() {} - -function validateEnum(allowed: string[]) { - return (value: string) => { - if (!allowed.includes(value)) { - throw Error("invalid value"); - } - }; -} - -function validateInt(value: string) { - const count = parseInt(value); - if (!isFinite(count)) { - throw Error("argument must be a number"); - } -} diff --git a/src/packages/backend/files/sandbox/find.test.ts b/src/packages/backend/files/sandbox/find.test.ts index f57ff4cd48..66574f5cac 100644 --- a/src/packages/backend/files/sandbox/find.test.ts +++ b/src/packages/backend/files/sandbox/find.test.ts @@ -13,14 +13,18 @@ afterAll(async () => { describe("find files", () => { it("directory starts empty", async () => { - const { stdout, truncated } = await find(tempDir, "%f\n"); + const { stdout, truncated } = await find(tempDir, { + options: ["-maxdepth", "1", "-mindepth", "1", "-printf", "%f\n"], + }); expect(stdout.length).toBe(0); expect(truncated).toBe(false); }); it("create a file and see it appears in find", async () => { await writeFile(join(tempDir, "a.txt"), "hello"); - const { stdout, truncated } = await find(tempDir, "%f\n"); + const { stdout, truncated } = await find(tempDir, { + options: ["-maxdepth", "1", "-mindepth", "1", "-printf", "%f\n"], + }); expect(truncated).toBe(false); expect(stdout.toString()).toEqual("a.txt\n"); }); @@ -29,17 +33,25 @@ describe("find files", () => { await writeFile(join(tempDir, "pattern"), ""); await mkdir(join(tempDir, "blue")); await writeFile(join(tempDir, "blue", "Patton"), ""); - const { stdout } = await find(tempDir, "%f\n", { - expression: { type: "iname", pattern: "patt*" }, + const { stdout } = await find(tempDir, { + options: [ + "-maxdepth", + "1", + "-mindepth", + "1", + "-iname", + "patt*", + "-printf", + "%f\n", + ], }); const v = stdout.toString().trim().split("\n"); expect(new Set(v)).toEqual(new Set(["pattern"])); }); it("find file in a subdirectory too", async () => { - const { stdout } = await find(tempDir, "%P\n", { - recursive: true, - expression: { type: "iname", pattern: "patt*" }, + const { stdout } = await find(tempDir, { + options: ["-iname", "patt*", "-printf", "%P\n"], }); const w = stdout.toString().trim().split("\n"); expect(new Set(w)).toEqual(new Set(["pattern", "blue/Patton"])); @@ -52,11 +64,17 @@ describe("find files", () => { await writeFile(join(tempDir, `${i}`), ""); } const t = Date.now(); - const { stdout, truncated } = await find(tempDir, "%f\n", { timeout: 0.1 }); + const { stdout, truncated } = await find(tempDir, { + options: ["-printf", "%f\n"], + timeout: 0.1, + }); + expect(truncated).toBe(true); expect(Date.now() - t).toBeGreaterThan(1); - const { stdout: stdout2 } = await find(tempDir, "%f\n"); + const { stdout: stdout2 } = await find(tempDir, { + options: ["-maxdepth", "1", "-mindepth", "1", "-printf", "%f\n"], + }); expect(stdout2.length).toBeGreaterThan(stdout.length); }); }); diff --git a/src/packages/backend/files/sandbox/find.ts b/src/packages/backend/files/sandbox/find.ts index 2f6e614ce3..8c1929ce70 100644 --- a/src/packages/backend/files/sandbox/find.ts +++ b/src/packages/backend/files/sandbox/find.ts @@ -1,172 +1,117 @@ /* -NOTE: there is a program https://github.com/sharkdp/fd that is a very fast -parallel rust program for finding files matching a pattern. It is complementary -to find here though, because we mainly use find to compute directory +NOTE: fd is a very fast parallel rust program for finding files matching +a pattern. It is complementary to find here though, because we mainly +use find to compute directory listing info (e.g., file size, mtime, etc.), and fd does NOT do that; it can exec ls, but that is slower than using find. So both find and fd are useful for different tasks -- find is *better* for directory listings and fd is better for finding filesnames in a directory tree that match a pattern. */ -import { spawn } from "node:child_process"; -import type { FindOptions, FindExpression } from "@cocalc/conat/files/fs"; -export type { FindOptions, FindExpression }; +import type { FindOptions } from "@cocalc/conat/files/fs"; +export type { FindOptions }; +import exec, { type ExecOutput, validate } from "./exec"; export default async function find( path: string, - printf: string, - { timeout = 0, recursive, expression }: FindOptions = {}, -): Promise<{ - // the output as a Buffer (not utf8, since it could have arbitrary file names!) - stdout: Buffer; - // truncated is true if the timeout gets hit. - truncated: boolean; -}> { - if (!path) { + { options, darwin, linux, timeout, maxSize }: FindOptions, +): Promise { + if (path == null) { throw Error("path must be specified"); } - if (!printf) { - throw Error("printf must be specified"); - } - return new Promise((resolve, reject) => { - const chunks: Buffer[] = []; - let truncated = false; - - const args = [ - "-P", // Never follow symlinks (security) - path, // Search path - "-mindepth", - "1", - ]; - if (!recursive) { - args.push("-maxdepth", "1"); - } - - // Add expression if provided - if (expression) { - try { - args.push(...buildFindArgs(expression)); - } catch (error) { - reject(error); - return; - } - } - args.push("-printf", printf); - - //console.log(`find ${args.join(" ")}`); - - // Spawn find with minimal, fixed arguments - const child = spawn("find", args, { - stdio: ["ignore", "pipe", "pipe"], - env: {}, // Empty environment (security) - shell: false, // No shell interpretation (security) - }); - - let timer; - if (timeout) { - timer = setTimeout(() => { - if (!truncated) { - truncated = true; - child.kill("SIGTERM"); - } - }, timeout); - } else { - timer = null; - } - - child.stdout.on("data", (chunk: Buffer) => { - chunks.push(chunk); - }); - - let stderr = ""; - child.stderr.on("data", (chunk: Buffer) => { - stderr += chunk.toString(); - }); - - // Handle completion - child.on("error", (error) => { - if (timer) { - clearTimeout(timer); - } - reject(error); - }); - - child.on("exit", (code) => { - if (timer) { - clearTimeout(timer); - } - - if (code !== 0 && !truncated) { - reject(new Error(`find exited with code ${code}: ${stderr}`)); - return; - } - - resolve({ stdout: Buffer.concat(chunks), truncated }); - }); + return await exec({ + cmd: "find", + cwd: path, + prefixArgs: [path ? path : "."], + options, + darwin, + linux, + maxSize, + timeout, + whitelist, + safety: [], }); } -function buildFindArgs(expr: FindExpression): string[] { - switch (expr.type) { - case "name": - // Validate pattern has no path separators - if (expr.pattern.includes("/")) { - throw new Error("Path separators not allowed in name patterns"); - } - return ["-name", expr.pattern]; - - case "iname": - if (expr.pattern.includes("/")) { - throw new Error("Path separators not allowed in name patterns"); - } - return ["-iname", expr.pattern]; - - case "type": - return ["-type", expr.value]; - - case "size": - // Validate size format (e.g., "10M", "1G", "500k") - if (!/^[0-9]+[kMGTP]?$/.test(expr.value)) { - throw new Error("Invalid size format"); - } - return ["-size", expr.operator + expr.value]; - - case "mtime": - if (!Number.isInteger(expr.days) || expr.days < 0) { - throw new Error("Invalid mtime days"); - } - return ["-mtime", expr.operator + expr.days]; - - case "newer": - // This is risky - would need to validate file path is within sandbox - if (expr.file.includes("..") || expr.file.startsWith("/")) { - throw new Error("Invalid reference file path"); - } - return ["-newer", expr.file]; - - case "and": - return [ - "(", - ...buildFindArgs(expr.left), - "-a", - ...buildFindArgs(expr.right), - ")", - ]; - - case "or": - return [ - "(", - ...buildFindArgs(expr.left), - "-o", - ...buildFindArgs(expr.right), - ")", - ]; - - case "not": - return ["!", ...buildFindArgs(expr.expr)]; - - default: - throw new Error("Unsupported expression type"); - } -} +const whitelist = { + // POSITIONAL OPTIONS + "-daystart": true, + "-regextype": validate.str, + "-warn": true, + "-nowarn": true, + + // GLOBAL OPTIONS + "-d": true, + "-depth": true, + "--help": true, + "-ignore_readdir_race": true, + "-maxdepth": validate.int, + "-mindepth": validate.int, + "-mount": true, + "-noignore_readdir_race": true, + "--version": true, + "-xdev": true, + + // TESTS + "-amin": validate.float, + "-anewer": validate.str, + "-atime": validate.float, + "-cmin": validate.float, + "-cnewer": validate.str, + "-ctime": validate.float, + "-empty": true, + "-executable": true, + "-fstype": validate.str, + "-gid": validate.int, + "-group": validate.str, + "-ilname": validate.str, + "-iname": validate.str, + "-inum": validate.int, + "-ipath": validate.str, + "-iregex": validate.str, + "-iwholename": validate.str, + "-links": validate.int, + "-lname": validate.str, + "-mmin": validate.int, + "-mtime": validate.int, + "-name": validate.str, + "-newer": validate.str, + "-newerXY": validate.str, + "-nogroup": true, + "-nouser": true, + "-path": validate.str, + "-perm": validate.str, + "-readable": true, + "-regex": validate.str, + "-samefile": validate.str, + "-size": validate.str, + "-true": true, + "-type": validate.str, + "-uid": validate.int, + "-used": validate.float, + "-user": validate.str, + "-wholename": validate.str, + "-writable": true, + "-xtype": validate.str, + "-context": validate.str, + + // ACTIONS: obviously many are not whitelisted! + "-ls": true, + "-print": true, + "-print0": true, + "-printf": validate.str, + "-prune": true, + "-quit": true, + + // OPERATORS + "(": true, + ")": true, + "!": true, + "-not": true, + "-a": true, + "-and": true, + "-o": true, + "-or": true, + ",": true, +} as const; diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index cf8874cdce..7bb111943a 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -207,13 +207,11 @@ export class SandboxedFilesystem { return await exists(await this.safeAbsPath(path)); }; - find = async ( - path: string, - printf: string, - options?: FindOptions, - ): Promise<{ stdout: Buffer; truncated: boolean }> => { - options = capTimeout(options, MAX_FIND_TIMEOUT); - return await find(await this.safeAbsPath(path), printf, options); + find = async (path: string, options?: FindOptions): Promise => { + return await find( + await this.safeAbsPath(path), + capTimeout(options, MAX_FIND_TIMEOUT), + ); }; fd = async (path: string, options?: FdOptions): Promise => { @@ -225,28 +223,14 @@ export class SandboxedFilesystem { ripgrep = async ( path: string, - regexp: string, + pattern: string, options?: RipgrepOptions, - ): Promise<{ - stdout: Buffer; - stderr: Buffer; - code: number | null; - truncated: boolean; - }> => { - if (this.unsafeMode) { - // unsafeMode = slightly less locked down... - return await ripgrep(path, regexp, { - timeout: options?.timeout, - options: options?.options, - allowedBasePath: "/", - }); - } - options = capTimeout(options, MAX_RIPGREP_TIMEOUT); - return await ripgrep(await this.safeAbsPath(path), regexp, { - timeout: capTimeout(options?.timeout, MAX_RIPGREP_TIMEOUT), - options: options?.options, - allowedBasePath: this.path, - }); + ): Promise => { + return await ripgrep( + await this.safeAbsPath(path), + pattern, + capTimeout(options, MAX_RIPGREP_TIMEOUT), + ); }; // hard link diff --git a/src/packages/backend/files/sandbox/install-ripgrep.ts b/src/packages/backend/files/sandbox/install.ts similarity index 57% rename from src/packages/backend/files/sandbox/install-ripgrep.ts rename to src/packages/backend/files/sandbox/install.ts index 506dbd9dc6..7f5fe1030b 100644 --- a/src/packages/backend/files/sandbox/install-ripgrep.ts +++ b/src/packages/backend/files/sandbox/install.ts @@ -1,11 +1,7 @@ /* -Download a ripgrep binary. +Download a ripgrep or fd binary for the operating system -This supports: - -- x86_64 Linux -- aarch64 Linux -- arm64 macos +This supports x86_64/arm64 linux & macos This assumes tar is installed. @@ -17,22 +13,61 @@ e.g., */ import { arch, platform } from "os"; -import { exists } from "@cocalc/backend/misc/async-utils-node"; import { execFileSync } from "child_process"; -import { writeFile, unlink, chmod } from "fs/promises"; +import { writeFile, stat, unlink, mkdir, chmod } from "fs/promises"; import { join } from "path"; -// See https://github.com/BurntSushi/ripgrep/releases -const VERSION = "14.1.1"; -const BASE = "https://github.com/BurntSushi/ripgrep/releases/download"; - -export const rgPath = join(__dirname, "rg"); +const i = __dirname.lastIndexOf("packages/backend"); +const binPath = join( + __dirname.slice(0, i + "packages/backend".length), + "node_modules/.bin", +); +export const ripgrep = join(binPath, "rg"); +export const fd = join(binPath, "fd"); + +const SPEC = { + ripgrep: { + // See https://github.com/BurntSushi/ripgrep/releases + VERSION: "14.1.1", + BASE: "https://github.com/BurntSushi/ripgrep/releases/download", + binary: "rg", + path: join(binPath, "rg"), + }, + fd: { + // See https://github.com/sharkdp/fd/releases + VERSION: "v10.2.0", + BASE: "https://github.com/sharkdp/fd/releases/download", + binary: "fd", + path: join(binPath, "fd"), + }, +} as const; + +type App = keyof typeof SPEC; + +// https://github.com/sharkdp/fd/releases/download/v10.2.0/fd-v10.2.0-x86_64-unknown-linux-musl.tar.gz +// https://github.com/BurntSushi/ripgrep/releases/download/14.1.1/ripgrep-14.1.1-x86_64-unknown-linux-musl.tar.gz + +async function exists(path: string) { + try { + await stat(path); + return true; + } catch { + return false; + } +} -export async function install() { - if (await exists(rgPath)) { +export async function install(app?: App) { + if (app == null) { + await Promise.all([install("ripgrep"), install("fd")]); + return; + } + if (app == "ripgrep" && (await exists(ripgrep))) { + return; + } + if (app == "fd" && (await exists(fd))) { return; } - const url = getUrl(); + const url = getUrl(app); // - 1. Fetch the tarball from the github url (using the fetch library) const response = await downloadFromGithub(url); const tarballBuffer = Buffer.from(await response.arrayBuffer()); @@ -43,21 +78,34 @@ export async function install() { // we have "tar tvf ripgrep-14.1.1-x86_64-unknown-linux-musl.tar.gz" outputs // ... // ripgrep-14.1.1-x86_64-unknown-linux-musl/rg - const tmpFile = join(__dirname, `ripgrep-${VERSION}.tar.gz`); - await writeFile(tmpFile, tarballBuffer); - // sync is fine since this is run at *build time*. - execFileSync("tar", [ - "xzf", - tmpFile, - "--strip-components=1", - `-C`, - __dirname, - `ripgrep-${VERSION}-${getName()}/rg`, - ]); - await unlink(tmpFile); - - // - 3. Make the file rg executable - await chmod(rgPath, 0o755); + + const { VERSION, binary, path } = SPEC[app]; + + const tmpFile = join(__dirname, `${app}-${VERSION}.tar.gz`); + try { + try { + if (!(await exists(binPath))) { + await mkdir(binPath); + } + } catch {} + await writeFile(tmpFile, tarballBuffer); + // sync is fine since this is run at *build time*. + execFileSync("tar", [ + "xzf", + tmpFile, + "--strip-components=1", + `-C`, + binPath, + `${app}-${VERSION}-${getOS()}/${binary}`, + ]); + + // - 3. Make the file rg executable + await chmod(path, 0o755); + } finally { + try { + await unlink(tmpFile); + } catch {} + } } // Download from github, but aware of rate limits, the retry-after header, etc. @@ -106,11 +154,12 @@ async function downloadFromGithub(url: string) { throw new Error("Should not reach here"); } -function getUrl() { - return `${BASE}/${VERSION}/ripgrep-${VERSION}-${getName()}.tar.gz`; +function getUrl(app: App) { + const { BASE, VERSION } = SPEC[app]; + return `${BASE}/${VERSION}/${app}-${VERSION}-${getOS()}.tar.gz`; } -function getName() { +function getOS() { switch (platform()) { case "linux": switch (arch()) { diff --git a/src/packages/backend/files/sandbox/ripgrep.test.ts b/src/packages/backend/files/sandbox/ripgrep.test.ts new file mode 100644 index 0000000000..34ba5e0d67 --- /dev/null +++ b/src/packages/backend/files/sandbox/ripgrep.test.ts @@ -0,0 +1,27 @@ +import ripgrep from "./ripgrep"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let tempDir; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); +}); +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); + +describe("ripgrep files", () => { + it("directory starts empty - no results", async () => { + const { stdout, truncated } = await ripgrep(tempDir, ""); + expect(stdout.length).toBe(0); + expect(truncated).toBe(false); + }); + + it("create a file and see it appears in the rigrep result", async () => { + await writeFile(join(tempDir, "a.txt"), "hello"); + const { stdout, truncated } = await ripgrep(tempDir, "he"); + expect(truncated).toBe(false); + expect(stdout.toString()).toEqual("a.txt:hello\n"); + }); +}); diff --git a/src/packages/backend/files/sandbox/ripgrep.ts b/src/packages/backend/files/sandbox/ripgrep.ts index f2ff751661..9fbb362816 100644 --- a/src/packages/backend/files/sandbox/ripgrep.ts +++ b/src/packages/backend/files/sandbox/ripgrep.ts @@ -1,357 +1,237 @@ -import { spawn } from "node:child_process"; -import { realpath } from "node:fs/promises"; -import * as path from "node:path"; +import exec, { type ExecOutput, validate } from "./exec"; import type { RipgrepOptions } from "@cocalc/conat/files/fs"; export type { RipgrepOptions }; -import { rgPath } from "./install-ripgrep"; - -const MAX_OUTPUT_SIZE = 10 * 1024 * 1024; // 10MB limit - -// Safely allowed options that don't pose security risks -const SAFE_OPTIONS = new Set([ - // Search behavior - "--case-sensitive", - "-s", - "--ignore-case", - "-i", - "--word-regexp", - "-w", - "--line-number", - "-n", - "--count", - "-c", - "--files-with-matches", - "-l", - "--files-without-match", - "--fixed-strings", - "-F", - "--invert-match", - "-v", - - // Output format - "--heading", - "--no-heading", - "--column", - "--pretty", - "--color", - "--no-line-number", - "-N", - - // Context lines (safe as long as we control the path) - "--context", - "-C", - "--before-context", - "-B", - "--after-context", - "-A", - - // Performance/filtering - "--max-count", - "-m", - "--max-depth", - "--max-filesize", - "--type", - "-t", - "--type-not", - "-T", - "--glob", - "-g", - "--iglob", - - // File selection - "--no-ignore", - "--hidden", - "--one-file-system", - "--null-data", - "--multiline", - "-U", - "--multiline-dotall", - "--crlf", - "--encoding", - "-E", - "--no-encoding", - - // basic info - "--version", - - // allow searching in binary files (files that contain NUL) - // this should be safe, since we're capturing output to buffers. - "--text", - "-a", - - // this ignores gitignore, hidden, and binary restrictions, which we allow - // above, so should be safe. - "--unrestricted", - "-u", - - "--debug", -]); - -// Options that take values - need special validation -const OPTIONS_WITH_VALUES = new Set([ - "--max-count", - "-m", - "--max-depth", - "--max-filesize", - "--type", - "-t", - "--type-not", - "-T", - "--glob", - "-g", - "--iglob", - "--context", - "-C", - "--before-context", - "-B", - "--after-context", - "-A", - "--encoding", - "-E", - "--color", -]); - -interface ExtendedRipgrepOptions extends RipgrepOptions { - options?: string[]; - allowedBasePath?: string; // The base path users are allowed to search within -} +import { ripgrep as ripgrepPath } from "./install"; -function validateGlobPattern(pattern: string): boolean { - // Reject patterns that could escape directory - if (pattern.includes("../") || pattern.includes("..\\")) { - return false; +export default async function ripgrep( + path: string, + pattern: string, + { options, darwin, linux, timeout, maxSize }: RipgrepOptions = {}, +): Promise { + if (path == null) { + throw Error("path must be specified"); } - // Reject absolute paths - if (path.isAbsolute(pattern)) { - return false; + if (pattern == null) { + throw Error("pattern must be specified"); } - return true; -} -function validateNumber(value: string): boolean { - return /^\d+$/.test(value); + return await exec({ + cmd: ripgrepPath, + cwd: path, + positionalArgs: [pattern], + options, + darwin, + linux, + maxSize, + timeout, + whitelist, + // if large memory usage is an issue, it might be caused by parallel interleaving; using + // -j1 below will prevent that, but will make ripgrep much slower (since not in parallel). + // See the ripgrep man page. + safety: ["--no-follow", "--block-buffered", "--no-config" /* "-j1"*/], + }); } -function validateEncoding(value: string): boolean { - // Allow only safe encodings - const safeEncodings = [ +const whitelist = { + "-e": validate.str, + + "-s": true, + "--case-sensitive": true, + + "--crlf": true, + + "-E": validate.set([ "utf-8", "utf-16", "utf-16le", "utf-16be", "ascii", "latin-1", - ]; - return safeEncodings.includes(value.toLowerCase()); -} + ]), + "--encoding": validate.set([ + "utf-8", + "utf-16", + "utf-16le", + "utf-16be", + "ascii", + "latin-1", + ]), -function parseAndValidateOptions(options: string[]): string[] { - const validatedOptions: string[] = []; - let i = 0; - - while (i < options.length) { - const opt = options[i]; - - // Check if this is a safe option - if (!SAFE_OPTIONS.has(opt)) { - throw new Error(`Disallowed option: ${opt}`); - } - - validatedOptions.push(opt); - - // Handle options that take values - if (OPTIONS_WITH_VALUES.has(opt)) { - i++; - if (i >= options.length) { - throw new Error(`Option ${opt} requires a value`); - } - - const value = options[i]; - - // Validate based on option type - if (opt === "--glob" || opt === "-g" || opt === "--iglob") { - if (!validateGlobPattern(value)) { - throw new Error(`Invalid glob pattern: ${value}`); - } - } else if ( - opt === "--max-count" || - opt === "-m" || - opt === "--max-depth" || - opt === "--context" || - opt === "-C" || - opt === "--before-context" || - opt === "-B" || - opt === "--after-context" || - opt === "-A" - ) { - if (!validateNumber(value)) { - throw new Error(`Invalid number for ${opt}: ${value}`); - } - } else if (opt === "--encoding" || opt === "-E") { - if (!validateEncoding(value)) { - throw new Error(`Invalid encoding: ${value}`); - } - } else if (opt === "--color") { - if (!["never", "auto", "always", "ansi"].includes(value)) { - throw new Error(`Invalid color option: ${value}`); - } - } - validatedOptions.push(value); - } - i++; - } - return validatedOptions; -} + "--engine": validate.set(["default", "pcre2", "auto"]), -export default async function ripgrep( - searchPath: string, - regexp: string, - { timeout = 0, options = [], allowedBasePath }: ExtendedRipgrepOptions = {}, -): Promise<{ - stdout: Buffer; - stderr: Buffer; - code: number | null; - truncated: boolean; -}> { - if (searchPath == null) { - throw Error("path must be specified"); - } - if (regexp == null) { - throw Error("regexp must be specified"); - } + "-F": true, + "--fixed-strings": true, - // Validate and normalize the search path - let normalizedPath: string; - try { - // Resolve to real path (follows symlinks to get actual path) - normalizedPath = await realpath(searchPath); - } catch (err) { - // If path doesn't exist, use normalize to check it - normalizedPath = path.normalize(searchPath); - } + "-i": true, + "--ignore-case": true, - // Security check: ensure path is within allowed base path - if (allowedBasePath) { - const normalizedBase = await realpath(allowedBasePath); - const relative = path.relative(normalizedBase, normalizedPath); + "-v": true, + "--invert-match": true, - // If relative path starts with .. or is absolute, it's outside allowed path - if (relative.startsWith("..") || path.isAbsolute(relative)) { - throw new Error("Search path is outside allowed directory"); - } - } + "-x": true, + "--line-regexp": true, - // Validate regexp doesn't contain null bytes (command injection protection) - if (regexp.includes("\0")) { - throw new Error("Invalid regexp: contains null bytes"); - } + "-m": validate.int, + "--max-count": validate.int, - // Build arguments array with security flags first - const args = [ - "--no-follow", // Don't follow symlinks - "--no-config", // Ignore config files - "--no-ignore-global", // Don't use global gitignore - "--no-require-git", // Don't require git repo - "--no-messages", // Suppress error messages that might leak info - ]; - - // Add validated user options - if (options.length > 0) { - const validatedOptions = parseAndValidateOptions(options); - args.push(...validatedOptions); - } + "-U": true, + "--multiline": true, - // Add the search pattern and path last - args.push("--", regexp, normalizedPath); // -- prevents regexp from being treated as option - - return new Promise((resolve, reject) => { - const stdoutChunks: Buffer[] = []; - const stderrChunks: Buffer[] = []; - let truncated = false; - let stdoutSize = 0; - let stderrSize = 0; - - const child = spawn(rgPath, args, { - stdio: ["ignore", "pipe", "pipe"], - env: { - // Minimal environment - only what ripgrep needs - PATH: process.env.PATH, - HOME: "/tmp", // Prevent access to user's home - RIPGREP_CONFIG_PATH: "/dev/null", // Explicitly disable config - }, - cwd: allowedBasePath || process.cwd(), // Restrict working directory - }); - - let timeoutHandle: NodeJS.Timeout | null = null; - - if (timeout > 0) { - timeoutHandle = setTimeout(() => { - truncated = true; - child.kill("SIGTERM"); - // Force kill after grace period - setTimeout(() => { - if (!child.killed) { - child.kill("SIGKILL"); - } - }, 1000); - }, timeout); - } - - child.stdout.on("data", (chunk: Buffer) => { - stdoutSize += chunk.length; - if (stdoutSize > MAX_OUTPUT_SIZE) { - truncated = true; - child.kill("SIGTERM"); - return; - } - stdoutChunks.push(chunk); - }); - - child.stderr.on("data", (chunk: Buffer) => { - stderrSize += chunk.length; - if (stderrSize > MAX_OUTPUT_SIZE) { - truncated = true; - child.kill("SIGTERM"); - return; - } - stderrChunks.push(chunk); - }); - - child.on("error", (err) => { - if (timeoutHandle) clearTimeout(timeoutHandle); - reject(err); - }); - - child.on("close", (code) => { - if (timeoutHandle) clearTimeout(timeoutHandle); - - const stdout = Buffer.concat(stdoutChunks); - const stderr = Buffer.concat(stderrChunks); - - // Truncate output if it's too large - const finalStdout = - stdout.length > MAX_OUTPUT_SIZE - ? stdout.slice(0, MAX_OUTPUT_SIZE) - : stdout; - const finalStderr = - stderr.length > MAX_OUTPUT_SIZE - ? stderr.slice(0, MAX_OUTPUT_SIZE) - : stderr; - - resolve({ - stdout: finalStdout, - stderr: finalStderr, - code, - truncated, - }); - }); - }); -} + "--multiline-dotall": true, + + "--no-unicode": true, + + "--null-data": true, + + "-P": true, + "--pcre2": true, + + "-S": true, + "--smart-case": true, + + "--stop-on-nonmatch": true, + + // this allows searching in binary files -- there is some danger of this + // using a lot more memory. Hence we do not allow it. + // "-a": true, + // "--text": true, + + "-w": true, + "--word-regexp": true, + + "--binary": true, + + "-g": validate.str, + "--glob": validate.str, + "--glob-case-insensitive": true, + + "-.": true, + "--hidden": true, + + "--iglob": validate.str, + + "--ignore-file-case-insensitive": true, + + "-d": validate.int, + "--max-depth": validate.int, + + "--max-filesize": validate.str, + + "--no-ignore": true, + "--no-ignore-dot": true, + "--no-ignore-exclude": true, + "--no-ignore-files": true, + "--no-ignore-global": true, + "--no-ignore-parent": true, + "--no-ignore-vcs": true, + "--no-require-git": true, + "--one-file-system": true, + + "-t": validate.str, + "--type": validate.str, + "-T": validate.str, + "--type-not": validate.str, + "--type-add": validate.str, + "--type-list": true, + "--type-clear": validate.str, + + "-u": true, + "--unrestricted": true, + + "-A": validate.int, + "--after-context": validate.int, + "-B": validate.int, + "--before-context": validate.int, + + "-b": true, + "--byte-offset": true, + + "--color": validate.set(["never", "auto", "always", "ansi"]), + "--colors": validate.str, + + "--column": true, + "-C": validate.int, + "--context": validate.int, + + "--context-separator": validate.str, + "--field-context-separator": validate.str, + "--field-match-separator": validate.str, + + "--heading": true, + "--no-heading": true, + + "-h": true, + "--help": true, + + "--include-zero": true, + + "-n": true, + "--line-number": true, + "-N": true, + "--no-line-number": true, + + "-M": validate.int, + "--max-columns": validate.int, + + "--max-columns-preview": validate.int, + + "-O": true, + "--null": true, + + "--passthru": true, + + "-p": true, + "--pretty": true, + + "-q": true, + "--quiet": true, + + // From the docs: "Neither this flag nor any other ripgrep flag will modify your files." + "-r": validate.str, + "--replace": validate.str, + + "--sort": validate.set(["none", "path", "modified", "accessed", "created"]), + "--sortr": validate.set(["none", "path", "modified", "accessed", "created"]), + + "--trim": true, + "--no-trim": true, + + "--vimgrep": true, + + "-H": true, + "--with-filename": true, + + "-I": true, + "--no-filename": true, + + "-c": true, + "--count": true, + + "--count-matches": true, + "-l": true, + "--files-with-matches": true, + "--files-without-match": true, + "--json": true, + + "--debug": true, + "--no-ignore-messages": true, + "--no-messages": true, + + "--stats": true, + + "--trace": true, + + "--files": true, + + "--generate": validate.set([ + "man", + "complete-bash", + "complete-zsh", + "complete-fish", + "complete-powershell", + ]), -// Export utility functions for testing -export const _internal = { - validateGlobPattern, - validateNumber, - validateEncoding, - parseAndValidateOptions, -}; + "--pcre2-version": true, + "-V": true, + "--version": true, +} as const; diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index 861ad94254..3be77288a7 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -20,7 +20,7 @@ "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf dist node_modules", - "install-ripgrep": "echo 'require(\"@cocalc/backend/files/sandbox/install-ripgrep\").install()' | node", + "install-ripgrep": "echo 'require(\"@cocalc/backend/files/sandbox/install\").install()' | node", "build": "pnpm exec tsc --build && pnpm install-ripgrep", "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", "test": "pnpm exec jest --forceExit", diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index b3ee87786f..90bb566836 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -26,32 +26,22 @@ export interface ExecOutput { } export interface RipgrepOptions { - timeout?: number; options?: string[]; + darwin?: string[]; + linux?: string[]; + timeout?: number; + maxSize?: number; } export interface FindOptions { - // timeout is very limited (e.g., 3s?) if fs is running on file - // server (not in own project) timeout?: number; - // recursive is false by default (unlike actual find command) - recursive?: boolean; - // see typing below -- we can't just pass arbitrary args since - // that would not be secure. - expression?: FindExpression; + // all safe whitelisted options to the find command + options?: string[]; + darwin?: string[]; + linux?: string[]; + maxSize?: number; } -export type FindExpression = - | { type: "name"; pattern: string } - | { type: "iname"; pattern: string } - | { type: "type"; value: "f" | "d" | "l" } - | { type: "size"; operator: "+" | "-"; value: string } - | { type: "mtime"; operator: "+" | "-"; days: number } - | { type: "newer"; file: string } - | { type: "and"; left: FindExpression; right: FindExpression } - | { type: "or"; left: FindExpression; right: FindExpression } - | { type: "not"; expr: FindExpression }; - export interface FdOptions { pattern?: string; options?: string[]; @@ -100,11 +90,7 @@ export interface Filesystem { // find -P {path} -maxdepth 1 -mindepth 1 -printf {printf} // For security reasons, this does not support all find arguments, // and can only use limited resources. - find: ( - path: string, - printf: string, - options?: FindOptions, - ) => Promise<{ stdout: Buffer; truncated: boolean }>; + find: (path: string, options?: FindOptions) => Promise; // Convenience function that uses the find and stat support to // provide all files in a directory by using tricky options to find, @@ -122,9 +108,9 @@ export interface Filesystem { // and can only use limited resources. ripgrep: ( path: string, - regexp: string, + pattern: string, options?: RipgrepOptions, - ) => Promise<{ stdout: Buffer; stderr: Buffer; truncated: boolean }>; + ) => Promise; } interface IDirent { @@ -282,8 +268,8 @@ export async function fsServer({ service, fs, client, project_id }: Options) { async fd(path: string, options?: FdOptions) { return await (await fs(this.subject)).fd(path, options); }, - async find(path: string, printf: string, options?: FindOptions) { - return await (await fs(this.subject)).find(path, printf, options); + async find(path: string, options?: FindOptions) { + return await (await fs(this.subject)).find(path, options); }, async link(existingPath: string, newPath: string) { await (await fs(this.subject)).link(existingPath, newPath); @@ -317,8 +303,8 @@ export async function fsServer({ service, fs, client, project_id }: Options) { async rename(oldPath: string, newPath: string) { await (await fs(this.subject)).rename(oldPath, newPath); }, - async ripgrep(path: string, regexp: string, options?: RipgrepOptions) { - return await (await fs(this.subject)).ripgrep(path, regexp, options); + async ripgrep(path: string, pattern: string, options?: RipgrepOptions) { + return await (await fs(this.subject)).ripgrep(path, pattern, options); }, async rm(path: string, options?) { await (await fs(this.subject)).rm(path, options); diff --git a/src/packages/conat/files/listing.ts b/src/packages/conat/files/listing.ts index e1836510d3..ddfd133346 100644 --- a/src/packages/conat/files/listing.ts +++ b/src/packages/conat/files/listing.ts @@ -146,11 +146,17 @@ export class Listing extends EventEmitter { async function getListing( fs: FilesystemClient, path: string, -): Promise<{ files: Files; truncated: boolean }> { - const { stdout, truncated } = await fs.find( - path, - "%f\\0%T@\\0%s\\0%y\\0%l\n", - ); +): Promise<{ files: Files; truncated?: boolean }> { + const { stdout, truncated } = await fs.find(path, { + options: [ + "-maxdepth", + "1", + "-mindepth", + "1", + "-printf", + "%f\\0%T@\\0%s\\0%y\\0%l\n", + ], + }); const buf = Buffer.from(stdout); const files: Files = {}; // todo -- what about non-utf8...? diff --git a/src/packages/frontend/project/search/body.tsx b/src/packages/frontend/project/search/body.tsx index dbf142f546..7b98eb0c7f 100644 --- a/src/packages/frontend/project/search/body.tsx +++ b/src/packages/frontend/project/search/body.tsx @@ -298,22 +298,7 @@ function ProjectSearchInput({ type="primary" onClick={() => actions?.search()} > - {neural ? ( - <> - - {small ? "" : " Neural Search"} - - ) : git ? ( - <> - - {small ? "" : " Git Grep Search"} - - ) : ( - <> - - {small ? "" : " Grep Search"} - - )} + Search } /> From 348f005276223718a3e3aafee9b541310263f345 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 3 Aug 2025 05:06:38 +0000 Subject: [PATCH 171/798] refactor search action function -- first step before rewriting it to use ripgrep, etc. --- src/packages/frontend/project/search/body.tsx | 21 +- src/packages/frontend/project/search/run.ts | 186 +++++++++++++++++ src/packages/frontend/project_actions.ts | 193 +++--------------- 3 files changed, 220 insertions(+), 180 deletions(-) create mode 100644 src/packages/frontend/project/search/run.ts diff --git a/src/packages/frontend/project/search/body.tsx b/src/packages/frontend/project/search/body.tsx index 7b98eb0c7f..ef3a2b66e4 100644 --- a/src/packages/frontend/project/search/body.tsx +++ b/src/packages/frontend/project/search/body.tsx @@ -84,11 +84,7 @@ export const ProjectSearchBody: React.FC<{ return ( - + {mode != "flyout" ? ( ) : undefined} @@ -151,12 +147,7 @@ export const ProjectSearchBody: React.FC<{ function renderHeaderFlyout() { return (
- + actions?.setState({ user_input: value })} on_submit={() => actions?.search()} on_clear={() => diff --git a/src/packages/frontend/project/search/run.ts b/src/packages/frontend/project/search/run.ts new file mode 100644 index 0000000000..18eef83742 --- /dev/null +++ b/src/packages/frontend/project/search/run.ts @@ -0,0 +1,186 @@ +import { type FilesystemClient } from "@cocalc/conat/files/fs"; +import { MARKERS } from "@cocalc/util/sagews"; +import { webapp_client } from "@cocalc/frontend/webapp-client"; + +export async function search({ + query, + path, + setState, + fs: _fs, + options = {}, + project_id, + compute_server_id, +}: { + query: string; + path: string; + setState: (any) => void; + fs: FilesystemClient; + options: { + case_sensitive?: boolean; + git_grep?: boolean; + subdirectories?: boolean; + hidden_files?: boolean; + }; + project_id: string; + compute_server_id: number; +}) { + if (!query) { + return; + } + + query = query.trim().replace(/"/g, '\\"'); + if (query === "") { + return; + } + const search_query = `"${query}"`; + setState({ + search_results: undefined, + search_error: undefined, + most_recent_search: query, + most_recent_path: path, + too_many_results: false, + }); + + // generate the grep command for the given query with the given flags + let cmd, ins; + if (options.case_sensitive) { + ins = ""; + } else { + ins = " -i "; + } + + if (options.git_grep) { + let max_depth; + if (options.subdirectories) { + max_depth = ""; + } else { + max_depth = "--max-depth=0"; + } + // The || true is so that if git rev-parse has exit code 0, + // but "git grep" finds nothing (hence has exit code 1), we don't + // fall back to normal git (the other side of the ||). See + // https://github.com/sagemathinc/cocalc/issues/4276 + cmd = `git rev-parse --is-inside-work-tree && (git grep -n -I -H ${ins} ${max_depth} ${search_query} || true) || `; + } else { + cmd = ""; + } + if (options.subdirectories) { + if (options.hidden_files) { + cmd += `rgrep -n -I -H --exclude-dir=.smc --exclude-dir=.snapshots ${ins} ${search_query} -- *`; + } else { + cmd += `rgrep -n -I -H --exclude-dir='.*' --exclude='.*' ${ins} ${search_query} -- *`; + } + } else { + if (options.hidden_files) { + cmd += `grep -n -I -H ${ins} ${search_query} -- .* *`; + } else { + cmd += `grep -n -I -H ${ins} ${search_query} -- *`; + } + } + + cmd += ` | grep -v ${MARKERS.cell}`; + const max_results = 1000; + const max_output = 110 * max_results; // just in case + + setState({ + command: cmd, + }); + + let output; + try { + output = await webapp_client.exec({ + project_id, + command: cmd + " | cut -c 1-256", // truncate horizontal line length (imagine a binary file that is one very long line) + timeout: 20, // how long grep runs on client + max_output, + bash: true, + err_on_exit: true, + compute_server_id, + filesystem: true, + path, + }); + } catch (err) { + processResults({ err, setState }); + return; + } + processResults({ + output, + max_results, + max_output, + setState, + }); +} + +function processResults({ + err, + output, + max_results, + max_output, + setState, +}: { + err?; + output?; + max_results?; + max_output?; + setState; +}) { + if (err) { + err = `${err}`; + } + if ((err && output == null) || (output != null && output.stdout == null)) { + setState({ search_error: err }); + return; + } + + const results = output.stdout.split("\n"); + const too_many_results = !!( + output.stdout.length >= max_output || + results.length > max_results || + err + ); + let num_results = 0; + const search_results: {}[] = []; + for (const line of results) { + if (line.trim() === "") { + continue; + } + let i = line.indexOf(":"); + num_results += 1; + if (i !== -1) { + // all valid lines have a ':', the last line may have been truncated too early + let filename = line.slice(0, i); + if (filename.slice(0, 2) === "./") { + filename = filename.slice(2); + } + let context = line.slice(i + 1); + // strip codes in worksheet output + if (context.length > 0 && context[0] === MARKERS.output) { + i = context.slice(1).indexOf(MARKERS.output); + context = context.slice(i + 2, context.length - 1); + } + + const m = /^(\d+):/.exec(context); + let line_number: number | undefined; + if (m != null) { + try { + line_number = parseInt(m[1]); + } catch (e) {} + } + + search_results.push({ + filename, + description: context, + line_number, + filter: `${filename.toLowerCase()} ${context.toLowerCase()}`, + }); + } + if (num_results >= max_results) { + break; + } + } + + setState({ + too_many_results, + search_results, + }); +} diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index 6e18614b23..abe12a9d31 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -104,7 +104,6 @@ import { DEFAULT_NEW_FILENAMES, NEW_FILENAMES } from "@cocalc/util/db-schema"; import * as misc from "@cocalc/util/misc"; import { reduxNameToProjectId } from "@cocalc/util/redux/name"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { MARKERS } from "@cocalc/util/sagews"; import { client_db } from "@cocalc/util/schema"; import { get_editor } from "./editors/react-wrapper"; import { type FilesystemClient } from "@cocalc/conat/files/fs"; @@ -114,6 +113,7 @@ import { type Files, } from "@cocalc/frontend/project/listing/use-files"; import { map as awaitMap } from "awaiting"; +import { search } from "@cocalc/frontend/project/search/run"; const { defaults, required } = misc; @@ -3074,167 +3074,6 @@ export class ProjectActions extends Actions { redux.getActions("account")?.set_other_settings("find_git_grep", git_grep); } - process_search_results(err, output, max_results, max_output, cmd) { - const store = this.get_store(); - if (store == undefined) { - return; - } - if (err) { - err = misc.to_user_string(err); - } - if ((err && output == null) || (output != null && output.stdout == null)) { - this.setState({ search_error: err }); - return; - } - - const results = output.stdout.split("\n"); - const too_many_results = !!( - output.stdout.length >= max_output || - results.length > max_results || - err - ); - let num_results = 0; - const search_results: {}[] = []; - for (const line of results) { - if (line.trim() === "") { - continue; - } - let i = line.indexOf(":"); - num_results += 1; - if (i !== -1) { - // all valid lines have a ':', the last line may have been truncated too early - let filename = line.slice(0, i); - if (filename.slice(0, 2) === "./") { - filename = filename.slice(2); - } - let context = line.slice(i + 1); - // strip codes in worksheet output - if (context.length > 0 && context[0] === MARKERS.output) { - i = context.slice(1).indexOf(MARKERS.output); - context = context.slice(i + 2, context.length - 1); - } - - const m = /^(\d+):/.exec(context); - let line_number: number | undefined; - if (m != null) { - try { - line_number = parseInt(m[1]); - } catch (e) {} - } - - search_results.push({ - filename, - description: context, - line_number, - filter: `${filename.toLowerCase()} ${context.toLowerCase()}`, - }); - } - if (num_results >= max_results) { - break; - } - } - - if (store.get("command") === cmd) { - // only update the state if the results are from the most recent command - this.setState({ - too_many_results, - search_results, - }); - } - } - - search = () => { - let cmd, ins; - const store = this.get_store(); - if (store == undefined) { - return; - } - - const query = store.get("user_input").trim().replace(/"/g, '\\"'); - if (query === "") { - return; - } - const search_query = `"${query}"`; - this.setState({ - search_results: undefined, - search_error: undefined, - most_recent_search: query, - most_recent_path: store.get("current_path"), - too_many_results: false, - }); - const path = store.get("current_path"); - - track("search", { - project_id: this.project_id, - path, - query, - neural_search: store.get("neural_search"), - subdirectories: store.get("subdirectories"), - hidden_files: store.get("hidden_files"), - git_grep: store.get("git_grep"), - }); - - // generate the grep command for the given query with the given flags - if (store.get("case_sensitive")) { - ins = ""; - } else { - ins = " -i "; - } - - if (store.get("git_grep")) { - let max_depth; - if (store.get("subdirectories")) { - max_depth = ""; - } else { - max_depth = "--max-depth=0"; - } - // The || true is so that if git rev-parse has exit code 0, - // but "git grep" finds nothing (hence has exit code 1), we don't - // fall back to normal git (the other side of the ||). See - // https://github.com/sagemathinc/cocalc/issues/4276 - cmd = `git rev-parse --is-inside-work-tree && (git grep -n -I -H ${ins} ${max_depth} ${search_query} || true) || `; - } else { - cmd = ""; - } - if (store.get("subdirectories")) { - if (store.get("hidden_files")) { - cmd += `rgrep -n -I -H --exclude-dir=.smc --exclude-dir=.snapshots ${ins} ${search_query} -- *`; - } else { - cmd += `rgrep -n -I -H --exclude-dir='.*' --exclude='.*' ${ins} ${search_query} -- *`; - } - } else { - if (store.get("hidden_files")) { - cmd += `grep -n -I -H ${ins} ${search_query} -- .* *`; - } else { - cmd += `grep -n -I -H ${ins} ${search_query} -- *`; - } - } - - cmd += ` | grep -v ${MARKERS.cell}`; - const max_results = 1000; - const max_output = 110 * max_results; // just in case - - this.setState({ - command: cmd, - }); - - const compute_server_id = this.getComputeServerId(); - webapp_client.exec({ - project_id: this.project_id, - command: cmd + " | cut -c 1-256", // truncate horizontal line length (imagine a binary file that is one very long line) - timeout: 20, // how long grep runs on client - max_output, - bash: true, - err_on_exit: true, - compute_server_id, - filesystem: true, - path: store.get("current_path"), - cb: (err, output) => { - this.process_search_results(err, output, max_results, max_output, cmd); - }, - }); - }; - set_file_listing_scroll(scroll_top) { this.setState({ file_listing_scroll_top: scroll_top }); } @@ -3703,4 +3542,34 @@ export class ProjectActions extends Actions { project_id: this.project_id, }); }; + + private searchId = 0; + search = async () => { + const store = this.get_store(); + if (!store) { + return; + } + const searchId = ++this.searchId; + const setState = (x) => { + if (this.searchId != searchId) { + // there's a newer search + return; + } + this.setState(x); + }; + await search({ + setState, + fs: this.fs(), + query: store.get("user_input").trim(), + path: store.get("current_path"), + project_id: this.project_id, + compute_server_id: this.getComputeServerId(), + options: { + case_sensitive: store.get("case_sensitive"), + git_grep: store.get("git_grep"), + subdirectories: store.get("subdirectories"), + hidden_files: store.get("hidden_files"), + }, + }); + }; } From 5cbff75ab6624215d4431a0f799590c154259568 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 3 Aug 2025 05:19:53 +0000 Subject: [PATCH 172/798] add dust -- still need to add whitelisted commands... --- src/packages/backend/files/sandbox/dust.ts | 29 +++++++++++++++++++ src/packages/backend/files/sandbox/index.ts | 26 +++++++++-------- src/packages/backend/files/sandbox/install.ts | 25 +++++++++++----- src/packages/conat/files/fs.ts | 14 +++++++++ 4 files changed, 75 insertions(+), 19 deletions(-) create mode 100644 src/packages/backend/files/sandbox/dust.ts diff --git a/src/packages/backend/files/sandbox/dust.ts b/src/packages/backend/files/sandbox/dust.ts new file mode 100644 index 0000000000..e5be132bd2 --- /dev/null +++ b/src/packages/backend/files/sandbox/dust.ts @@ -0,0 +1,29 @@ +import exec, { type ExecOutput /* validate */ } from "./exec"; +import { type DustOptions } from "@cocalc/conat/files/fs"; +export { type DustOptions }; +import { dust as dustPath } from "./install"; + +export default async function dust( + path: string, + { options, darwin, linux, timeout, maxSize }: DustOptions = {}, +): Promise { + if (path == null) { + throw Error("path must be specified"); + } + + return await exec({ + cmd: dustPath, + cwd: path, + positionalArgs: [path], + options, + darwin, + linux, + maxSize, + timeout, + whitelist, + }); +} + +const whitelist = { + "-j": true, +} as const; diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index 7bb111943a..2788f29017 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -72,17 +72,12 @@ import { type WatchOptions } from "@cocalc/conat/files/watch"; import find, { type FindOptions } from "./find"; import ripgrep, { type RipgrepOptions } from "./ripgrep"; import fd, { type FdOptions } from "./fd"; +import dust, { type DustOptions } from "./dust"; import { type ExecOutput } from "./exec"; -// max time a user find request can run (in safe mode) -- this can cause excessive -// load on a server if there were a directory with a massive number of files, -// so must be limited. -const MAX_FIND_TIMEOUT = 3000; - -// max time a user ripgrep can run (when in safe mode) -const MAX_RIPGREP_TIMEOUT = 3000; - -const MAX_FD_TIMEOUT = 3000; +// max time code can run (in safe mode), e.g., for find, +// ripgrep, fd, and dust. +const MAX_TIMEOUT = 5000; interface Options { // unsafeMode -- if true, assume security model where user is running this @@ -210,14 +205,21 @@ export class SandboxedFilesystem { find = async (path: string, options?: FindOptions): Promise => { return await find( await this.safeAbsPath(path), - capTimeout(options, MAX_FIND_TIMEOUT), + capTimeout(options, MAX_TIMEOUT), ); }; fd = async (path: string, options?: FdOptions): Promise => { return await fd( await this.safeAbsPath(path), - capTimeout(options, MAX_FD_TIMEOUT), + capTimeout(options, MAX_TIMEOUT), + ); + }; + + dust = async (path: string, options?: DustOptions): Promise => { + return await dust( + await this.safeAbsPath(path), + capTimeout(options, MAX_TIMEOUT), ); }; @@ -229,7 +231,7 @@ export class SandboxedFilesystem { return await ripgrep( await this.safeAbsPath(path), pattern, - capTimeout(options, MAX_RIPGREP_TIMEOUT), + capTimeout(options, MAX_TIMEOUT), ); }; diff --git a/src/packages/backend/files/sandbox/install.ts b/src/packages/backend/files/sandbox/install.ts index 7f5fe1030b..dfd396c46d 100644 --- a/src/packages/backend/files/sandbox/install.ts +++ b/src/packages/backend/files/sandbox/install.ts @@ -22,8 +22,6 @@ const binPath = join( __dirname.slice(0, i + "packages/backend".length), "node_modules/.bin", ); -export const ripgrep = join(binPath, "rg"); -export const fd = join(binPath, "fd"); const SPEC = { ripgrep: { @@ -40,8 +38,19 @@ const SPEC = { binary: "fd", path: join(binPath, "fd"), }, + dust: { + // See https://github.com/bootandy/dust/releases + VERSION: "v1.2.3", + BASE: "https://github.com/bootandy/dust/releases/download", + binary: "dust", + path: join(binPath, "dust"), + }, } as const; +export const ripgrep = SPEC.ripgrep.path; +export const fd = SPEC.fd.path; +export const dust = SPEC.dust.path; + type App = keyof typeof SPEC; // https://github.com/sharkdp/fd/releases/download/v10.2.0/fd-v10.2.0-x86_64-unknown-linux-musl.tar.gz @@ -56,15 +65,17 @@ async function exists(path: string) { } } +async function alreadyInstalled(app: App) { + return await exists(SPEC[app].path); +} + export async function install(app?: App) { if (app == null) { - await Promise.all([install("ripgrep"), install("fd")]); - return; - } - if (app == "ripgrep" && (await exists(ripgrep))) { + // @ts-ignore + await Promise.all(Object.keys(SPEC).map(install)); return; } - if (app == "fd" && (await exists(fd))) { + if (await alreadyInstalled(app)) { return; } const url = getUrl(app); diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index 90bb566836..a5f9d3c4b0 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -51,6 +51,14 @@ export interface FdOptions { maxSize?: number; } +export interface DustOptions { + options?: string[]; + darwin?: string[]; + linux?: string[]; + timeout?: number; + maxSize?: number; +} + export interface Filesystem { appendFile: (path: string, data: string | Buffer, encoding?) => Promise; chmod: (path: string, mode: string | number) => Promise; @@ -102,6 +110,9 @@ export interface Filesystem { // options: { type: "name", pattern:"^\.DS_Store$" } fd: (path: string, options?: FdOptions) => Promise; + // dust is an amazing disk space tool + dust: (path: string, options?: DustOptions) => Promise; + // We add ripgrep, as a 1-call way to very efficiently search in files // directly on whatever is serving files. // For security reasons, this does not support all ripgrep arguments, @@ -262,6 +273,9 @@ export async function fsServer({ service, fs, client, project_id }: Options) { async cp(src: string, dest: string, options?) { await (await fs(this.subject)).cp(src, dest, options); }, + async dust(path: string, options?: DustOptions) { + return await (await fs(this.subject)).dust(path, options); + }, async exists(path: string): Promise { return await (await fs(this.subject)).exists(path); }, From 7283c139f66e8bece46342367b5fd19e728f876c Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 3 Aug 2025 15:54:19 +0000 Subject: [PATCH 173/798] add dust whitelist and test --- .../backend/files/sandbox/dust.test.ts | 39 ++++++++ src/packages/backend/files/sandbox/dust.ts | 94 ++++++++++++++++++- src/packages/backend/files/sandbox/index.ts | 4 +- 3 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 src/packages/backend/files/sandbox/dust.test.ts diff --git a/src/packages/backend/files/sandbox/dust.test.ts b/src/packages/backend/files/sandbox/dust.test.ts new file mode 100644 index 0000000000..cd69a56951 --- /dev/null +++ b/src/packages/backend/files/sandbox/dust.test.ts @@ -0,0 +1,39 @@ +import dust from "./dust"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let tempDir; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); +}); +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); + +describe("dust works", () => { + it("directory starts empty - no results", async () => { + const { stdout, truncated } = await dust(tempDir, { options: ["-j"] }); + const s = JSON.parse(Buffer.from(stdout).toString()); + expect(s).toEqual({ children: [], name: tempDir, size: s.size }); + expect(truncated).toBe(false); + }); + + it("create a file and see it appears in the dust result", async () => { + await writeFile(join(tempDir, "a.txt"), "hello"); + const { stdout, truncated } = await dust(tempDir, { options: ["-j"] }); + const s = JSON.parse(Buffer.from(stdout).toString()); + expect(s).toEqual({ + size: s.size, + name: tempDir, + children: [ + { + size: s.children[0].size, + name: join(tempDir, "a.txt"), + children: [], + }, + ], + }); + expect(truncated).toBe(false); + }); +}); diff --git a/src/packages/backend/files/sandbox/dust.ts b/src/packages/backend/files/sandbox/dust.ts index e5be132bd2..1e5043ccc1 100644 --- a/src/packages/backend/files/sandbox/dust.ts +++ b/src/packages/backend/files/sandbox/dust.ts @@ -1,4 +1,4 @@ -import exec, { type ExecOutput /* validate */ } from "./exec"; +import exec, { type ExecOutput, validate } from "./exec"; import { type DustOptions } from "@cocalc/conat/files/fs"; export { type DustOptions }; import { dust as dustPath } from "./install"; @@ -25,5 +25,97 @@ export default async function dust( } const whitelist = { + "-d": validate.int, + "--depth": validate.int, + + "-n": validate.int, + "--number-of-lines": validate.int, + + "-p": true, + "--full-paths": true, + + "-X": validate.str, + "--ignore-directory": validate.str, + + "-x": true, + "--limit-filesystem": true, + + "-s": true, + "--apparent-size": true, + + "-r": true, + "--reverse": true, + + "-c": true, + "--no-colors": true, + "-C": true, + "--force-colors": true, + + "-b": true, + "--no-percent-bars": true, + + "-B": true, + "--bars-on-right": true, + + "-z": validate.str, + "--min-size": validate.str, + + "-R": true, + "--screen-reader": true, + + "--skip-total": true, + + "-f": true, + "--filecount": true, + + "-i": true, + "--ignore-hidden": true, + + "-v": validate.str, + "--invert-filter": validate.str, + + "-e": validate.str, + "--filter": validate.str, + + "-t": validate.str, + "--file-types": validate.str, + + "-w": validate.int, + "--terminal-width": validate.int, + + "-P": true, + "--no-progress": true, + + "--print-errors": true, + + "-D": true, + "--only-dir": true, + + "-F": true, + "--only-file": true, + + "-o": validate.str, + "--output-format": validate.str, + "-j": true, + "--output-json": true, + + "-M": validate.str, + "--mtime": validate.str, + + "-A": validate.str, + "--atime": validate.str, + + "-y": validate.str, + "--ctime": validate.str, + + "--collapse": validate.str, + + "-m": validate.set(["a", "c", "m"]), + "--filetime": validate.set(["a", "c", "m"]), + + "-h": true, + "--help": true, + "-V": true, + "--version": true, } as const; diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index 2788f29017..b80ba3b53b 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -219,7 +219,9 @@ export class SandboxedFilesystem { dust = async (path: string, options?: DustOptions): Promise => { return await dust( await this.safeAbsPath(path), - capTimeout(options, MAX_TIMEOUT), + // dust reasonably takes longer than the other commands and is used less, + // so for now we give it more breathing room. + capTimeout(options, 4*MAX_TIMEOUT), ); }; From 17fa2ca39fdffe78813db3427fb7d00837efc17f Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 3 Aug 2025 20:21:47 +0000 Subject: [PATCH 174/798] no longer using vscode/ripgrep --- src/packages/backend/package.json | 1 - src/packages/pnpm-lock.yaml | 39 ------------------------------- 2 files changed, 40 deletions(-) diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index 3be77288a7..c74cf851ab 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -46,7 +46,6 @@ "@cocalc/backend": "workspace:*", "@cocalc/conat": "workspace:*", "@cocalc/util": "workspace:*", - "@vscode/ripgrep": "^1.15.14", "awaiting": "^3.0.0", "better-sqlite3": "^12.2.0", "chokidar": "^3.6.0", diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 8781b7180e..00ebbf5056 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -87,9 +87,6 @@ importers: '@cocalc/util': specifier: workspace:* version: link:../util - '@vscode/ripgrep': - specifier: ^1.15.14 - version: 1.15.14 awaiting: specifier: ^3.0.0 version: 3.0.0 @@ -4571,9 +4568,6 @@ packages: peerDependencies: react: '>= 16.8.0' - '@vscode/ripgrep@1.15.14': - resolution: {integrity: sha512-/G1UJPYlm+trBWQ6cMO3sv6b8D1+G16WaJH1/DSqw32JOVlzgZbLkDxRyzIpTpv30AcYGMkCf5tUqGlW6HbDWw==} - '@vscode/vscode-languagedetection@1.0.22': resolution: {integrity: sha512-rQ/BgMyLuIXSmbA0MSkIPHtcOw14QkeDbAq19sjvaS9LTRr905yij0S8lsyqN5JgOsbtIx7pAcyOxFMzPmqhZQ==} hasBin: true @@ -5144,9 +5138,6 @@ packages: bser@2.1.1: resolution: {integrity: sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==} - buffer-crc32@0.2.13: - resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==} - buffer-equal-constant-time@1.0.1: resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} @@ -6720,9 +6711,6 @@ packages: fb-watchman@2.0.2: resolution: {integrity: sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==} - fd-slicer@1.1.0: - resolution: {integrity: sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==} - fdir@6.4.6: resolution: {integrity: sha512-hiFoqpyZcfNm1yc4u8oWCf9A2c4D3QjCrks3zmoVKVxpQRzmPNar1hUJcBG2RQHvEVGDN+Jm81ZheVLAQMK6+w==} peerDependencies: @@ -9276,9 +9264,6 @@ packages: resolution: {integrity: sha512-/Y3fcFrXEAsMjJXeL9J8+ZG9U01LbuWaYypvDW2ycW1jL269L3js3DVBjDJ0Up9Np1uqDXsDrRihHANhZOlwdQ==} engines: {node: '>=20'} - pend@1.2.0: - resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==} - performance-now@2.1.0: resolution: {integrity: sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==} @@ -11750,9 +11735,6 @@ packages: resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==} engines: {node: '>=12'} - yauzl@2.10.0: - resolution: {integrity: sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==} - yjs@13.6.27: resolution: {integrity: sha512-OIDwaflOaq4wC6YlPBy2L6ceKeKuF7DeTxx+jPzv1FHn9tCZ0ZwSRnUBxD05E3yed46fv/FWJbvR+Ud7x0L7zw==} engines: {node: '>=16.0.0', npm: '>=8.0.0'} @@ -14923,14 +14905,6 @@ snapshots: '@use-gesture/core': 10.3.1 react: 19.1.0 - '@vscode/ripgrep@1.15.14': - dependencies: - https-proxy-agent: 7.0.6 - proxy-from-env: 1.1.0 - yauzl: 2.10.0 - transitivePeerDependencies: - - supports-color - '@vscode/vscode-languagedetection@1.0.22': {} '@webassemblyjs/ast@1.14.1': @@ -15598,8 +15572,6 @@ snapshots: dependencies: node-int64: 0.4.0 - buffer-crc32@0.2.13: {} - buffer-equal-constant-time@1.0.1: {} buffer-from@1.1.2: {} @@ -17452,10 +17424,6 @@ snapshots: dependencies: bser: 2.1.1 - fd-slicer@1.1.0: - dependencies: - pend: 1.2.0 - fdir@6.4.6(picomatch@4.0.2): optionalDependencies: picomatch: 4.0.2 @@ -20557,8 +20525,6 @@ snapshots: optionalDependencies: '@napi-rs/canvas': 0.1.73 - pend@1.2.0: {} - performance-now@2.1.0: {} pg-cloudflare@1.2.7: @@ -23492,11 +23458,6 @@ snapshots: y18n: 5.0.8 yargs-parser: 21.1.1 - yauzl@2.10.0: - dependencies: - buffer-crc32: 0.2.13 - fd-slicer: 1.1.0 - yjs@13.6.27: dependencies: lib0: 0.2.109 From 61eda29753a2680ccfafb770705e7658f858919a Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 3 Aug 2025 20:31:01 +0000 Subject: [PATCH 175/798] update find unit test --- src/packages/backend/conat/files/test/local-path.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/packages/backend/conat/files/test/local-path.test.ts b/src/packages/backend/conat/files/test/local-path.test.ts index b7fdfc156d..88bbeaf783 100644 --- a/src/packages/backend/conat/files/test/local-path.test.ts +++ b/src/packages/backend/conat/files/test/local-path.test.ts @@ -197,7 +197,9 @@ describe("use all the standard api functions of fs", () => { }); it("use the find command instead of readdir", async () => { - const { stdout } = await fs.find("dirtest", "%f\n"); + const { stdout } = await fs.find("dirtest", { + options: ["-maxdepth", "1", "-mindepth", "1", "-printf", "%f\n"], + }); const v = stdout.toString().trim().split("\n"); // output of find is NOT in alphabetical order: expect(new Set(v)).toEqual(new Set(["0", "1", "2", "3", "4", fire])); From e1fc9f96b1284d147b117686596c39e1d7e58edd Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 3 Aug 2025 23:03:30 +0000 Subject: [PATCH 176/798] unsafe proof of concept rustic integration --- src/packages/backend/data.ts | 2 + src/packages/backend/files/sandbox/index.ts | 15 ++++++- src/packages/backend/files/sandbox/install.ts | 33 ++++++++++++-- src/packages/backend/files/sandbox/rustic.ts | 43 +++++++++++++++++++ src/packages/conat/files/fs.ts | 5 +++ 5 files changed, 93 insertions(+), 5 deletions(-) create mode 100644 src/packages/backend/files/sandbox/rustic.ts diff --git a/src/packages/backend/data.ts b/src/packages/backend/data.ts index a95c19dd7f..507c7b9291 100644 --- a/src/packages/backend/data.ts +++ b/src/packages/backend/data.ts @@ -180,6 +180,8 @@ export const pgdatabase: string = export const projects: string = process.env.PROJECTS ?? join(data, "projects", "[project_id]"); export const secrets: string = process.env.SECRETS ?? join(data, "secrets"); +export const rusticRepo: string = + process.env.RUSTIC_REPO ?? join(data, "rustic"); // Where the sqlite database files used for sync are stored. // The idea is there is one very fast *ephemeral* directory diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index b80ba3b53b..90946e7c62 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -73,7 +73,9 @@ import find, { type FindOptions } from "./find"; import ripgrep, { type RipgrepOptions } from "./ripgrep"; import fd, { type FdOptions } from "./fd"; import dust, { type DustOptions } from "./dust"; +import rustic from "./rustic"; import { type ExecOutput } from "./exec"; +import { rusticRepo } from "@cocalc/backend/data"; // max time code can run (in safe mode), e.g., for find, // ripgrep, fd, and dust. @@ -96,11 +98,13 @@ const INTERNAL_METHODS = new Set([ "unsafeMode", "readonly", "assertWritable", + "rusticRepo", ]); export class SandboxedFilesystem { public readonly unsafeMode: boolean; public readonly readonly: boolean; + private readonly rusticRepo: string = rusticRepo; constructor( // path should be the path to a FOLDER on the filesystem (not a file) public readonly path: string, @@ -221,10 +225,19 @@ export class SandboxedFilesystem { await this.safeAbsPath(path), // dust reasonably takes longer than the other commands and is used less, // so for now we give it more breathing room. - capTimeout(options, 4*MAX_TIMEOUT), + capTimeout(options, 4 * MAX_TIMEOUT), ); }; + rustic = async (args: string[]): Promise => { + return await rustic(args, { + repo: this.rusticRepo, + safeAbsPath: this.safeAbsPath, + timeout: 120_000, + maxSize: 10_000, + }); + }; + ripgrep = async ( path: string, pattern: string, diff --git a/src/packages/backend/files/sandbox/install.ts b/src/packages/backend/files/sandbox/install.ts index dfd396c46d..7b5bd947c8 100644 --- a/src/packages/backend/files/sandbox/install.ts +++ b/src/packages/backend/files/sandbox/install.ts @@ -23,6 +23,15 @@ const binPath = join( "node_modules/.bin", ); +interface Spec { + VERSION: string; + BASE: string; + binary: string; + path: string; + stripComponents?: number; + pathInArchive?: string; +} + const SPEC = { ripgrep: { // See https://github.com/BurntSushi/ripgrep/releases @@ -45,11 +54,21 @@ const SPEC = { binary: "dust", path: join(binPath, "dust"), }, -} as const; + rustic: { + // See https://github.com/rustic-rs/rustic/releases + VERSION: "v0.9.5", + BASE: "https://github.com/rustic-rs/rustic/releases/download", + binary: "rustic", + path: join(binPath, "rustic"), + stripComponents: 0, + pathInArchive: "rustic", + }, +}; export const ripgrep = SPEC.ripgrep.path; export const fd = SPEC.fd.path; export const dust = SPEC.dust.path; +export const rustic = SPEC.rustic.path; type App = keyof typeof SPEC; @@ -90,7 +109,13 @@ export async function install(app?: App) { // ... // ripgrep-14.1.1-x86_64-unknown-linux-musl/rg - const { VERSION, binary, path } = SPEC[app]; + const { + VERSION, + binary, + path, + stripComponents = 1, + pathInArchive = `${app}-${VERSION}-${getOS()}/${binary}`, + } = SPEC[app] as Spec; const tmpFile = join(__dirname, `${app}-${VERSION}.tar.gz`); try { @@ -104,10 +129,10 @@ export async function install(app?: App) { execFileSync("tar", [ "xzf", tmpFile, - "--strip-components=1", + `--strip-components=${stripComponents}`, `-C`, binPath, - `${app}-${VERSION}-${getOS()}/${binary}`, + pathInArchive, ]); // - 3. Make the file rg executable diff --git a/src/packages/backend/files/sandbox/rustic.ts b/src/packages/backend/files/sandbox/rustic.ts new file mode 100644 index 0000000000..a079692723 --- /dev/null +++ b/src/packages/backend/files/sandbox/rustic.ts @@ -0,0 +1,43 @@ +import exec, { type ExecOutput /*, validate*/ } from "./exec"; +import { rustic as rusticPath } from "./install"; +import { exists } from "@cocalc/backend/misc/async-utils-node"; +import { join } from "path"; + +export interface RusticOptions { + repo: string; + timeout?: number; + maxSize?: number; + safeAbsPath?: (path: string) => Promise; +} + +export default async function rustic( + args: string[], + options: RusticOptions, +): Promise { + const { timeout, maxSize, repo, safeAbsPath } = options; + + await ensureInitialized(repo); + + return await exec({ + cmd: rusticPath, + cwd: safeAbsPath ? await safeAbsPath("") : undefined, + safety: ["--password", "", "-r", repo, ...args], + maxSize, + timeout, + }); +} + +async function ensureInitialized(repo: string) { + if (!(await exists(join(repo, "config")))) { + await exec({ + cmd: rusticPath, + safety: ["--password", "", "-r", repo, "init"], + }); + } +} + +// const whitelist = { +// backup: {}, +// restore: {}, +// snapshots: {}, +// } as const; diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index a5f9d3c4b0..c02ad3f984 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -122,6 +122,8 @@ export interface Filesystem { pattern: string, options?: RipgrepOptions, ) => Promise; + + rustic: (args: string[]) => Promise; } interface IDirent { @@ -320,6 +322,9 @@ export async function fsServer({ service, fs, client, project_id }: Options) { async ripgrep(path: string, pattern: string, options?: RipgrepOptions) { return await (await fs(this.subject)).ripgrep(path, pattern, options); }, + async rustic(args: string[]) { + return await (await fs(this.subject)).rustic(args); + }, async rm(path: string, options?) { await (await fs(this.subject)).rm(path, options); }, From b237f858d8d296036b039be06b247cbc993fa6ad Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 4 Aug 2025 00:53:25 +0000 Subject: [PATCH 177/798] working on rustic whitelist options --- .../backend/conat/files/local-path.ts | 2 +- src/packages/backend/files/sandbox/exec.ts | 2 +- src/packages/backend/files/sandbox/index.ts | 7 +- src/packages/backend/files/sandbox/rustic.ts | 225 ++++++++++++++++-- 4 files changed, 217 insertions(+), 19 deletions(-) diff --git a/src/packages/backend/conat/files/local-path.ts b/src/packages/backend/conat/files/local-path.ts index 3fa1c4a83a..0272996739 100644 --- a/src/packages/backend/conat/files/local-path.ts +++ b/src/packages/backend/conat/files/local-path.ts @@ -39,7 +39,7 @@ export async function localPathFileserver({ try { await mkdir(p); } catch {} - return new SandboxedFilesystem(p, { unsafeMode }); + return new SandboxedFilesystem(p, { unsafeMode, project_id }); } }, }); diff --git a/src/packages/backend/files/sandbox/exec.ts b/src/packages/backend/files/sandbox/exec.ts index b0d477b622..b1c3a4d474 100644 --- a/src/packages/backend/files/sandbox/exec.ts +++ b/src/packages/backend/files/sandbox/exec.ts @@ -140,7 +140,7 @@ export default async function exec({ }); } -function parseAndValidateOptions(options: string[], whitelist): string[] { +export function parseAndValidateOptions(options: string[], whitelist): string[] { const validatedOptions: string[] = []; let i = 0; diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index 90946e7c62..00a698a5b8 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -87,6 +87,7 @@ interface Options { unsafeMode?: boolean; // readonly -- only allow operations that don't change files readonly?: boolean; + project_id?: string; } // If you add any methods below that are NOT for the public api @@ -99,19 +100,22 @@ const INTERNAL_METHODS = new Set([ "readonly", "assertWritable", "rusticRepo", + "project_id", ]); export class SandboxedFilesystem { public readonly unsafeMode: boolean; public readonly readonly: boolean; private readonly rusticRepo: string = rusticRepo; + private project_id?: string; constructor( // path should be the path to a FOLDER on the filesystem (not a file) public readonly path: string, - { unsafeMode = false, readonly = false }: Options = {}, + { unsafeMode = false, readonly = false, project_id }: Options = {}, ) { this.unsafeMode = !!unsafeMode; this.readonly = !!readonly; + this.project_id = project_id; for (const f in this) { if (INTERNAL_METHODS.has(f)) { continue; @@ -235,6 +239,7 @@ export class SandboxedFilesystem { safeAbsPath: this.safeAbsPath, timeout: 120_000, maxSize: 10_000, + host: this.project_id ?? "global", }); }; diff --git a/src/packages/backend/files/sandbox/rustic.ts b/src/packages/backend/files/sandbox/rustic.ts index a079692723..9c991dcc9d 100644 --- a/src/packages/backend/files/sandbox/rustic.ts +++ b/src/packages/backend/files/sandbox/rustic.ts @@ -1,30 +1,130 @@ -import exec, { type ExecOutput /*, validate*/ } from "./exec"; +/* +Whitelist: + +The idea is that + - the client can only work with snapshots with exactly the given host. + - any snapshots they create have that host + - snapshots are only of data in their sandbox + - snapshots can only be restored to their sandbox + +The subcommands with some whitelisted support are: + + - backup + - snapshots + - ls + - restore + - find + - forget + +The source options are relative paths and the command is run from the +root of the sandbox_path. + + rustic backup --host=sandbox_path [whitelisted options]... [source]... + + rustic snapshots --filter-host=... [whitelisted options]... + + +Here the snapshot id will be checked to have the right host before +the command is run. Destination is relative to sandbox_path. + + rustic restore [whitelisted options] + + +Dump is used for viewing a version of a file via timetravel: + + rustic dump + +Find is used for getting info about all versions of a file that are backed up: + + rustic find --filter-host=... + + rustic find --filter-host=... --glob='foo/x.txt' -h + + +Delete snapshots: + +- delete snapshot with specific id, which must have the specified host. + + rustic forget [id] + +- + + +*/ + +import exec, { + type ExecOutput, + parseAndValidateOptions, + validate, +} from "./exec"; import { rustic as rusticPath } from "./install"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import { join } from "path"; +import { rusticRepo } from "@cocalc/backend/data"; +import LRU from "lru-cache"; export interface RusticOptions { - repo: string; + repo?: string; timeout?: number; maxSize?: number; - safeAbsPath?: (path: string) => Promise; + safeAbsPath: (path: string) => Promise; + host: string; } export default async function rustic( args: string[], options: RusticOptions, ): Promise { - const { timeout, maxSize, repo, safeAbsPath } = options; + const { timeout, maxSize, repo = rusticRepo, safeAbsPath, host } = options; await ensureInitialized(repo); + const base = await safeAbsPath(""); - return await exec({ - cmd: rusticPath, - cwd: safeAbsPath ? await safeAbsPath("") : undefined, - safety: ["--password", "", "-r", repo, ...args], - maxSize, - timeout, - }); + const common = ["--password", "", "-r", repo]; + + const run = async (sanitizedArgs: string[]) => { + return await exec({ + cmd: rusticPath, + cwd: base, + safety: [...common, ...sanitizedArgs], + maxSize, + timeout, + }); + }; + + if (args[0] == "backup") { + if (args.length == 1) { + throw Error("missing backup source"); + } + const source = (await safeAbsPath(args.slice(-1)[0])).slice(base.length); + const options = parseAndValidateOptions( + args.slice(1, -1), + whitelist.backup, + ); + + return await run([ + "backup", + ...options, + "--no-scan", + "--host", + host, + "--", + source, + ]); + } else if (args[0] == "snapshots") { + const options = parseAndValidateOptions(args.slice(1), whitelist.snapshots); + return await run(["snapshots", ...options, "--filter-host", host]); + } else if (args[0] == "ls") { + if (args.length <= 1) { + throw Error("missing "); + } + const snapshot = args.slice(-1)[0]; // + await assertValidSnapshot({ snapshot, host, repo }); + const options = parseAndValidateOptions(args.slice(1, -1), whitelist.ls); + return await run(["ls", ...options, snapshot]); + } else { + throw Error(`subcommand not allowed: ${args[0]}`); + } } async function ensureInitialized(repo: string) { @@ -36,8 +136,101 @@ async function ensureInitialized(repo: string) { } } -// const whitelist = { -// backup: {}, -// restore: {}, -// snapshots: {}, -// } as const; +const whitelist = { + backup: { + "--label": validate.str, + "--tag": validate.str, + "--description": validate.str, + "--time": validate.str, + "--delete-after": validate.str, + "--as-path": validate.str, + "--with-atime": true, + "--ignore-devid": true, + "--json": true, + "--long": true, + "--quiet": true, + "-h": true, + "--help": true, + "--glob": validate.str, + "--iglob": validate.str, + "--git-ignore": true, + "--no-require-git": true, + "-x": true, + "--one-file-system": true, + "--exclude-larger-than": validate.str, + }, + snapshots: { + "-g": validate.str, + "--group-by": validate.str, + "--long": true, + "--json": true, + "--all": true, + "-h": true, + "--help": true, + "--filter-label": validate.str, + "--filter-paths": validate.str, + "--filter-paths-exact": validate.str, + "--filter-after": validate.str, + "--filter-before": validate.str, + "--filter-size": validate.str, + "--filter-size-added": validate.str, + "--filter-jq": validate.str, + }, + restore: {}, + ls: { + "-s": true, + "--summary": true, + "-l": true, + "--long": true, + "--json": true, + "--numeric-uid-gid": true, + "--recursive": true, + "-h": true, + "--help": true, + "--glob": validate.str, + "--iglob": validate.str, + }, +} as const; + +async function assertValidSnapshot({ snapshot, host, repo }) { + const id = snapshot.split(":")[0]; + if (id == "latest") { + // possible race condition so do not allow + throw Error("latest is not allowed"); + } + const actualHost = await getHost({ id, repo }); + if (actualHost != host) { + throw Error( + `host for snapshot with id ${id} must be '${host}' but it is ${actualHost}`, + ); + } +} + +// we do not allow changing host so this is safe to cache. +const hostCache = new LRU({ + max: 10000, +}); + +export async function getHost(opts) { + if (hostCache.has(opts.id)) { + return hostCache.get(opts.id); + } + const info = await getSnapshot(opts); + const hostname = info[0][1][0]["hostname"]; + hostCache.set(opts.id, hostname); + return hostname; +} + +export async function getSnapshot({ + id, + repo = rusticRepo, +}: { + id: string; + repo?: string; +}) { + const { stdout } = await exec({ + cmd: rusticPath, + safety: ["--password", "", "-r", repo, "snapshots", "--json", id], + }); + return JSON.parse(stdout.toString()); +} From e99b4fba75d64a24f9625c0cf93c000a60e9d301 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 4 Aug 2025 01:14:57 +0000 Subject: [PATCH 178/798] finished rustic whitelisting --- src/packages/backend/files/sandbox/rustic.ts | 116 ++++++++++++++++--- 1 file changed, 101 insertions(+), 15 deletions(-) diff --git a/src/packages/backend/files/sandbox/rustic.ts b/src/packages/backend/files/sandbox/rustic.ts index 9c991dcc9d..5118b1548f 100644 --- a/src/packages/backend/files/sandbox/rustic.ts +++ b/src/packages/backend/files/sandbox/rustic.ts @@ -86,7 +86,7 @@ export default async function rustic( return await exec({ cmd: rusticPath, cwd: base, - safety: [...common, ...sanitizedArgs], + safety: [...common, args[0], ...sanitizedArgs], maxSize, timeout, }); @@ -102,18 +102,10 @@ export default async function rustic( whitelist.backup, ); - return await run([ - "backup", - ...options, - "--no-scan", - "--host", - host, - "--", - source, - ]); + return await run([...options, "--no-scan", "--host", host, "--", source]); } else if (args[0] == "snapshots") { const options = parseAndValidateOptions(args.slice(1), whitelist.snapshots); - return await run(["snapshots", ...options, "--filter-host", host]); + return await run([args[0], ...options, "--filter-host", host]); } else if (args[0] == "ls") { if (args.length <= 1) { throw Error("missing "); @@ -121,7 +113,32 @@ export default async function rustic( const snapshot = args.slice(-1)[0]; // await assertValidSnapshot({ snapshot, host, repo }); const options = parseAndValidateOptions(args.slice(1, -1), whitelist.ls); - return await run(["ls", ...options, snapshot]); + return await run([...options, snapshot]); + } else if (args[0] == "restore") { + if (args.length <= 2) { + throw Error("missing "); + } + const snapshot = args.slice(-2)[0]; // + await assertValidSnapshot({ snapshot, host, repo }); + const destination = await safeAbsPath(args.slice(-1)[0]); // + const options = parseAndValidateOptions( + args.slice(1, -2), + whitelist.restore, + ); + return await run([...options, snapshot, destination]); + } else if (args[0] == "find") { + const options = parseAndValidateOptions(args.slice(1), whitelist.find); + return await run([...options, "--filter-host", host]); + } else if (args[0] == "forget") { + if (args.length == 2 && !args[1].startsWith("-")) { + // delete exactly id + const snapshot = args[1]; + await assertValidSnapshot({ snapshot, host, repo }); + return await run([snapshot]); + } + // delete several defined by rules. + const options = parseAndValidateOptions(args.slice(1), whitelist.forget); + return await run([...options, "--filter-host", host]); } else { throw Error(`subcommand not allowed: ${args[0]}`); } @@ -131,7 +148,7 @@ async function ensureInitialized(repo: string) { if (!(await exists(join(repo, "config")))) { await exec({ cmd: rusticPath, - safety: ["--password", "", "-r", repo, "init"], + safety: ["--no-progress", "--password", "", "-r", repo, "init"], }); } } @@ -176,20 +193,89 @@ const whitelist = { "--filter-size-added": validate.str, "--filter-jq": validate.str, }, - restore: {}, + restore: { + "--delete": true, + "--verify-existing": true, + "--recursive": true, + "-h": true, + "--help": true, + "--glob": validate.str, + "--iglob": validate.str, + }, ls: { "-s": true, "--summary": true, "-l": true, "--long": true, "--json": true, - "--numeric-uid-gid": true, "--recursive": true, "-h": true, "--help": true, "--glob": validate.str, "--iglob": validate.str, }, + find: { + "--glob": validate.str, + "--iglob": validate.str, + "--path": validate.str, + "-g": validate.str, + "--group-by": validate.str, + "--all": true, + "--show-misses": true, + "-h": true, + "--help": true, + "--filter-label": validate.str, + "--filter-paths": validate.str, + "--filter-paths-exact": validate.str, + "--filter-after": validate.str, + "--filter-before": validate.str, + "--filter-size": validate.str, + "--filter-size-added": validate.str, + "--filter-jq": validate.str, + }, + forget: { + "--json": true, + "-g": validate.str, + "--group-by": validate.str, + "-h": true, + "--help": true, + "--filter-label": validate.str, + "--filter-paths": validate.str, + "--filter-paths-exact": validate.str, + "--filter-after": validate.str, + "--filter-before": validate.str, + "--filter-size": validate.str, + "--filter-size-added": validate.str, + "--filter-jq": validate.str, + "--keep-tags": validate.str, + "--keep-id": validate.str, + "-l": validate.int, + "--keep-last": validate.int, + "-M": validate.int, + "--keep-minutely": validate.int, + "-H": validate.int, + "--keep-hourly": validate.int, + "-d": validate.int, + "--keep-daily": validate.int, + "-w": validate.int, + "--keep-weekly": validate.int, + "-m": validate.int, + "--keep-monthly": validate.int, + "--keep-quarter-yearly": validate.int, + "--keep-half-yearly": validate.int, + "-y": validate.int, + "--keep-yearly": validate.int, + "--keep-within": validate.str, + "--keep-within-minutely": validate.str, + "--keep-within-hourly": validate.str, + "--keep-within-daily": validate.str, + "--keep-within-weekly": validate.str, + "--keep-within-monthly": validate.str, + "--keep-within-quarter-yearly": validate.str, + "--keep-within-half-yearly": validate.str, + "--keep-within-yearly": validate.str, + "--keep-none": validate.str, + }, } as const; async function assertValidSnapshot({ snapshot, host, repo }) { From dde1d5c2741e8c01181aebddb208cba260a9a2ac Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 4 Aug 2025 03:56:11 +0000 Subject: [PATCH 179/798] add a test of rustic --- src/packages/backend/files/sandbox/exec.ts | 7 +- src/packages/backend/files/sandbox/install.ts | 4 + .../backend/files/sandbox/rustic.test.ts | 60 +++++++++ src/packages/backend/files/sandbox/rustic.ts | 117 ++++++++++-------- src/packages/backend/package.json | 16 +-- 5 files changed, 137 insertions(+), 67 deletions(-) create mode 100644 src/packages/backend/files/sandbox/rustic.test.ts diff --git a/src/packages/backend/files/sandbox/exec.ts b/src/packages/backend/files/sandbox/exec.ts index b1c3a4d474..be17c9b676 100644 --- a/src/packages/backend/files/sandbox/exec.ts +++ b/src/packages/backend/files/sandbox/exec.ts @@ -75,7 +75,7 @@ export default async function exec({ args.push("--", ...positionalArgs); } - // console.log(`${cmd} ${args.join(" ")}`); + // console.log(`${cmd} ${args.join(" ")}`, { cmd, args }); logger.debug({ cmd, args }); const child = spawn(cmd, args, { stdio: ["ignore", "pipe", "pipe"], @@ -140,7 +140,10 @@ export default async function exec({ }); } -export function parseAndValidateOptions(options: string[], whitelist): string[] { +export function parseAndValidateOptions( + options: string[], + whitelist, +): string[] { const validatedOptions: string[] = []; let i = 0; diff --git a/src/packages/backend/files/sandbox/install.ts b/src/packages/backend/files/sandbox/install.ts index 7b5bd947c8..d22f1d3634 100644 --- a/src/packages/backend/files/sandbox/install.ts +++ b/src/packages/backend/files/sandbox/install.ts @@ -16,6 +16,9 @@ import { arch, platform } from "os"; import { execFileSync } from "child_process"; import { writeFile, stat, unlink, mkdir, chmod } from "fs/promises"; import { join } from "path"; +import getLogger from "@cocalc/backend/logger"; + +const logger = getLogger("files:sandbox:install"); const i = __dirname.lastIndexOf("packages/backend"); const binPath = join( @@ -98,6 +101,7 @@ export async function install(app?: App) { return; } const url = getUrl(app); + logger.debug("install", { app, url }); // - 1. Fetch the tarball from the github url (using the fetch library) const response = await downloadFromGithub(url); const tarballBuffer = Buffer.from(await response.arrayBuffer()); diff --git a/src/packages/backend/files/sandbox/rustic.test.ts b/src/packages/backend/files/sandbox/rustic.test.ts new file mode 100644 index 0000000000..db1c9ae1f9 --- /dev/null +++ b/src/packages/backend/files/sandbox/rustic.test.ts @@ -0,0 +1,60 @@ +/* +Test the rustic backup api. + +https://github.com/rustic-rs/rustic +*/ + +import rustic from "./rustic"; +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; + +let tempDir, options; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); + const repo = join(tempDir, "repo"); + const home = join(tempDir, "home"); + await mkdir(home); + const safeAbsPath = (path: string) => join(home, resolve("/", path)); + options = { + host: "my-host", + repo, + safeAbsPath, + }; +}); +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); + +describe("rustic does something", () => { + it("there are initially no backups", async () => { + const { stdout, truncated } = await rustic( + ["snapshots", "--json"], + options, + ); + const s = JSON.parse(Buffer.from(stdout).toString()); + expect(s).toEqual([]); + expect(truncated).toBe(false); + }); + + it("create a file and back it up", async () => { + await writeFile(join(tempDir, "a.txt"), "hello"); + const { stdout, truncated } = await rustic( + ["backup", "--json", "a.txt"], + options, + ); + const s = JSON.parse(Buffer.from(stdout).toString()); + expect(s.paths).toEqual(["a.txt"]); + expect(truncated).toBe(false); + }); + + // it("it appears in the snapshots list", async () => { + // const { stdout, truncated } = await rustic( + // ["snapshots", "--json"], + // options, + // ); + // const s = JSON.parse(Buffer.from(stdout).toString()); + // expect(s).toEqual([]); + // expect(truncated).toBe(false); + // }); +}); diff --git a/src/packages/backend/files/sandbox/rustic.ts b/src/packages/backend/files/sandbox/rustic.ts index 5118b1548f..03d9240cc7 100644 --- a/src/packages/backend/files/sandbox/rustic.ts +++ b/src/packages/backend/files/sandbox/rustic.ts @@ -92,64 +92,65 @@ export default async function rustic( }); }; - if (args[0] == "backup") { - if (args.length == 1) { - throw Error("missing backup source"); + switch (args[0]) { + case "backup": { + if (args.length == 1) { + throw Error("missing backup source"); + } + const source = (await safeAbsPath(args.slice(-1)[0])).slice(base.length); + const options = parseAndValidateOptions( + args.slice(1, -1), + whitelist.backup, + ); + + return await run([...options, "--no-scan", "--host", host, "--", source]); } - const source = (await safeAbsPath(args.slice(-1)[0])).slice(base.length); - const options = parseAndValidateOptions( - args.slice(1, -1), - whitelist.backup, - ); - - return await run([...options, "--no-scan", "--host", host, "--", source]); - } else if (args[0] == "snapshots") { - const options = parseAndValidateOptions(args.slice(1), whitelist.snapshots); - return await run([args[0], ...options, "--filter-host", host]); - } else if (args[0] == "ls") { - if (args.length <= 1) { - throw Error("missing "); + case "snapshots": { + const options = parseAndValidateOptions( + args.slice(1), + whitelist.snapshots, + ); + return await run([...options, "--filter-host", host]); } - const snapshot = args.slice(-1)[0]; // - await assertValidSnapshot({ snapshot, host, repo }); - const options = parseAndValidateOptions(args.slice(1, -1), whitelist.ls); - return await run([...options, snapshot]); - } else if (args[0] == "restore") { - if (args.length <= 2) { - throw Error("missing "); + case "ls": { + if (args.length <= 1) { + throw Error("missing "); + } + const snapshot = args.slice(-1)[0]; // + await assertValidSnapshot({ snapshot, host, repo }); + const options = parseAndValidateOptions(args.slice(1, -1), whitelist.ls); + return await run([...options, snapshot]); } - const snapshot = args.slice(-2)[0]; // - await assertValidSnapshot({ snapshot, host, repo }); - const destination = await safeAbsPath(args.slice(-1)[0]); // - const options = parseAndValidateOptions( - args.slice(1, -2), - whitelist.restore, - ); - return await run([...options, snapshot, destination]); - } else if (args[0] == "find") { - const options = parseAndValidateOptions(args.slice(1), whitelist.find); - return await run([...options, "--filter-host", host]); - } else if (args[0] == "forget") { - if (args.length == 2 && !args[1].startsWith("-")) { - // delete exactly id - const snapshot = args[1]; + case "restore": { + if (args.length <= 2) { + throw Error("missing "); + } + const snapshot = args.slice(-2)[0]; // await assertValidSnapshot({ snapshot, host, repo }); - return await run([snapshot]); + const destination = await safeAbsPath(args.slice(-1)[0]); // + const options = parseAndValidateOptions( + args.slice(1, -2), + whitelist.restore, + ); + return await run([...options, snapshot, destination]); } - // delete several defined by rules. - const options = parseAndValidateOptions(args.slice(1), whitelist.forget); - return await run([...options, "--filter-host", host]); - } else { - throw Error(`subcommand not allowed: ${args[0]}`); - } -} - -async function ensureInitialized(repo: string) { - if (!(await exists(join(repo, "config")))) { - await exec({ - cmd: rusticPath, - safety: ["--no-progress", "--password", "", "-r", repo, "init"], - }); + case "find": { + const options = parseAndValidateOptions(args.slice(1), whitelist.find); + return await run([...options, "--filter-host", host]); + } + case "forget": { + if (args.length == 2 && !args[1].startsWith("-")) { + // delete exactly id + const snapshot = args[1]; + await assertValidSnapshot({ snapshot, host, repo }); + return await run([snapshot]); + } + // delete several defined by rules. + const options = parseAndValidateOptions(args.slice(1), whitelist.forget); + return await run([...options, "--filter-host", host]); + } + default: + throw Error(`subcommand not allowed: ${args[0]}`); } } @@ -278,6 +279,16 @@ const whitelist = { }, } as const; +async function ensureInitialized(repo: string) { + const config = join(repo, "config"); + if (!(await exists(config))) { + await exec({ + cmd: rusticPath, + safety: ["--no-progress", "--password", "", "-r", repo, "init"], + }); + } +} + async function assertValidSnapshot({ snapshot, host, repo }) { const id = snapshot.split(":")[0]; if (id == "latest") { diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index c74cf851ab..2c45ae00dd 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -13,15 +13,12 @@ "./auth/*": "./dist/auth/*.js", "./auth/tokens/*": "./dist/auth/tokens/*.js" }, - "keywords": [ - "utilities", - "cocalc" - ], + "keywords": ["utilities", "cocalc"], "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf dist node_modules", - "install-ripgrep": "echo 'require(\"@cocalc/backend/files/sandbox/install\").install()' | node", - "build": "pnpm exec tsc --build && pnpm install-ripgrep", + "install-sandbox-tools": "echo 'require(\"@cocalc/backend/files/sandbox/install\").install()' | node", + "build": "pnpm exec tsc --build && pnpm install-sandbox-tools", "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", "test": "pnpm exec jest --forceExit", "test-conat": " pnpm exec jest --forceExit conat", @@ -34,12 +31,7 @@ "conat-persist": "DEBUG=cocalc:* node ./bin/conat-persist.cjs", "conat-test-server": "node ./bin/conat-test-server.cjs" }, - "files": [ - "dist/**", - "bin/**", - "README.md", - "package.json" - ], + "files": ["dist/**", "bin/**", "README.md", "package.json"], "author": "SageMath, Inc.", "license": "SEE LICENSE.md", "dependencies": { From a999dcf5c206da7b2250a3c85aaa1d4f72f70a30 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 4 Aug 2025 21:57:37 +0000 Subject: [PATCH 180/798] search -- switch to using fs ripgrep, first version --- src/packages/frontend/project/search/body.tsx | 4 +- src/packages/frontend/project/search/run.ts | 180 ++++-------------- src/packages/frontend/project_actions.ts | 2 - 3 files changed, 38 insertions(+), 148 deletions(-) diff --git a/src/packages/frontend/project/search/body.tsx b/src/packages/frontend/project/search/body.tsx index ef3a2b66e4..667c042855 100644 --- a/src/packages/frontend/project/search/body.tsx +++ b/src/packages/frontend/project/search/body.tsx @@ -120,8 +120,8 @@ export const ProjectSearchBody: React.FC<{ checked={git_grep} onChange={() => actions?.toggle_search_checkbox_git_grep()} > - Git search: in GIT repo, use "git grep" - to only search files in the git repo. + .gitignore aware: exclude files via + .gitignore and similar rules. {neural_search_enabled && ( = max_output || - results.length > max_results || - err - ); - let num_results = 0; - const search_results: {}[] = []; - for (const line of results) { - if (line.trim() === "") { + const search_results: SearchResult[] = []; + for (const line of lines) { + let result; + try { + result = JSON.parse(line); + } catch { continue; } - let i = line.indexOf(":"); - num_results += 1; - if (i !== -1) { - // all valid lines have a ':', the last line may have been truncated too early - let filename = line.slice(0, i); - if (filename.slice(0, 2) === "./") { - filename = filename.slice(2); - } - let context = line.slice(i + 1); - // strip codes in worksheet output - if (context.length > 0 && context[0] === MARKERS.output) { - i = context.slice(1).indexOf(MARKERS.output); - context = context.slice(i + 2, context.length - 1); - } - - const m = /^(\d+):/.exec(context); - let line_number: number | undefined; - if (m != null) { - try { - line_number = parseInt(m[1]); - } catch (e) {} - } - + if (result.type == "match") { + const { line_number, lines, path } = result.data; search_results.push({ - filename, - description: context, + filename: path?.text ?? "-", + description: `${(line_number.toString() + ":").padEnd(8, " ")}${lines.text}`, + filter: `${path?.text?.toLowerCase?.() ?? ""} ${lines.text.toLowerCase()}`, line_number, - filter: `${filename.toLowerCase()} ${context.toLowerCase()}`, }); } - if (num_results >= max_results) { - break; - } } setState({ - too_many_results, + too_many_results: truncated, search_results, + most_recent_search: query, + most_recent_path: path, }); } diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index abe12a9d31..cbbd0fd8bc 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -3562,8 +3562,6 @@ export class ProjectActions extends Actions { fs: this.fs(), query: store.get("user_input").trim(), path: store.get("current_path"), - project_id: this.project_id, - compute_server_id: this.getComputeServerId(), options: { case_sensitive: store.get("case_sensitive"), git_grep: store.get("git_grep"), From c60a5113bc2ee84e483fdf4a52407c0c99fa5e0e Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 4 Aug 2025 23:25:00 +0000 Subject: [PATCH 181/798] search: make consistent with flyout; use virtuoso so displaying a large number is very fast --- src/packages/frontend/chat/chat-log.tsx | 3 +- .../frontend/project/page/flyouts/body.tsx | 3 +- .../frontend/project/page/flyouts/header.tsx | 1 - .../frontend/project/page/flyouts/search.tsx | 4 +- src/packages/frontend/project/search/body.tsx | 317 ++++++------------ src/packages/frontend/project/search/run.ts | 15 +- .../frontend/project/search/search.tsx | 2 +- 7 files changed, 122 insertions(+), 223 deletions(-) diff --git a/src/packages/frontend/chat/chat-log.tsx b/src/packages/frontend/chat/chat-log.tsx index e1f25a778b..37f0d29cf1 100644 --- a/src/packages/frontend/chat/chat-log.tsx +++ b/src/packages/frontend/chat/chat-log.tsx @@ -13,11 +13,10 @@ import { Alert, Button } from "antd"; import { Set as immutableSet } from "immutable"; import { MutableRefObject, useEffect, useMemo, useRef } from "react"; import { Virtuoso, VirtuosoHandle } from "react-virtuoso"; - +import useVirtuosoScrollHook from "@cocalc/frontend/components/virtuoso-scroll-hook"; import { chatBotName, isChatBot } from "@cocalc/frontend/account/chatbot"; import { useRedux, useTypedRedux } from "@cocalc/frontend/app-framework"; import { Icon } from "@cocalc/frontend/components"; -import useVirtuosoScrollHook from "@cocalc/frontend/components/virtuoso-scroll-hook"; import { HashtagBar } from "@cocalc/frontend/editors/task-editor/hashtag-bar"; import { DivTempHeight } from "@cocalc/frontend/jupyter/cell-list"; import { diff --git a/src/packages/frontend/project/page/flyouts/body.tsx b/src/packages/frontend/project/page/flyouts/body.tsx index d71a3c2c3d..cc7883ddda 100644 --- a/src/packages/frontend/project/page/flyouts/body.tsx +++ b/src/packages/frontend/project/page/flyouts/body.tsx @@ -4,7 +4,6 @@ */ import { debounce } from "lodash"; - import { CSS, redux, @@ -97,7 +96,7 @@ export function FlyoutBody({ flyout, flyoutWidth }: FlyoutBodyProps) { style={style} onFocus={() => { // Remove any active key handler that is next to this side chat. - // E.g, this is critical for taks lists... + // E.g, this is critical for task lists... redux.getActions("page").erase_active_key_handler(); }} > diff --git a/src/packages/frontend/project/page/flyouts/header.tsx b/src/packages/frontend/project/page/flyouts/header.tsx index bacbf7573e..ad40f53e1a 100644 --- a/src/packages/frontend/project/page/flyouts/header.tsx +++ b/src/packages/frontend/project/page/flyouts/header.tsx @@ -6,7 +6,6 @@ import { Button, Tooltip } from "antd"; import { useEffect, useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; - import { TourName } from "@cocalc/frontend/account/tours"; import { redux, useTypedRedux } from "@cocalc/frontend/app-framework"; import { Icon } from "@cocalc/frontend/components"; diff --git a/src/packages/frontend/project/page/flyouts/search.tsx b/src/packages/frontend/project/page/flyouts/search.tsx index fadb01c57d..18b09784c4 100644 --- a/src/packages/frontend/project/page/flyouts/search.tsx +++ b/src/packages/frontend/project/page/flyouts/search.tsx @@ -5,6 +5,6 @@ import { ProjectSearchBody } from "@cocalc/frontend/project/search/body"; -export function SearchFlyout({ wrap }) { - return ; +export function SearchFlyout() { + return ; } diff --git a/src/packages/frontend/project/search/body.tsx b/src/packages/frontend/project/search/body.tsx index 667c042855..8af738422a 100644 --- a/src/packages/frontend/project/search/body.tsx +++ b/src/packages/frontend/project/search/body.tsx @@ -10,13 +10,12 @@ of course, a disaster waiting to happen. They all need to be in a single namespace somehow...! */ -import { Button, Card, Col, Input, Row, Space, Tag } from "antd"; -import { useEffect, useMemo, useState } from "react"; +import { Button, Card, Col, Input, Row, Tag } from "antd"; +import { useMemo, useState } from "react"; import { useProjectContext } from "@cocalc/frontend/project/context"; -import { Alert, Checkbox, Well } from "@cocalc/frontend/antd-bootstrap"; +import { Alert, Checkbox } from "@cocalc/frontend/antd-bootstrap"; import { useActions, useTypedRedux } from "@cocalc/frontend/app-framework"; import { - Gap, HelpIcon, Icon, Loading, @@ -34,21 +33,19 @@ import { filename_extension, path_split, path_to_file, + plural, search_match, search_split, unreachable, } from "@cocalc/util/misc"; import { COLORS } from "@cocalc/util/theme"; import SelectComputeServerForFileExplorer from "@cocalc/frontend/compute/select-server-for-explorer"; - -const RESULTS_WELL_STYLE: React.CSSProperties = { - backgroundColor: "white", -} as const; +import { Virtuoso } from "react-virtuoso"; +import useVirtuosoScrollHook from "@cocalc/frontend/components/virtuoso-scroll-hook"; export const ProjectSearchBody: React.FC<{ mode: "project" | "flyout"; - wrap?: Function; -}> = ({ mode = "project", wrap }) => { +}> = ({ mode = "project" }) => { const { project_id } = useProjectContext(); const subdirectories = useTypedRedux({ project_id }, "subdirectories"); const case_sensitive = useTypedRedux({ project_id }, "case_sensitive"); @@ -62,28 +59,17 @@ export const ProjectSearchBody: React.FC<{ const actions = useActions({ project_id }); - const isFlyout = mode === "flyout"; - function renderResultList() { - if (isFlyout) { - return ( - - ); - } else { - return ( - - - - - - ); - } + return ; } function renderHeaderProject() { return ( - + {mode != "flyout" ? ( @@ -120,8 +106,8 @@ export const ProjectSearchBody: React.FC<{ checked={git_grep} onChange={() => actions?.toggle_search_checkbox_git_grep()} > - .gitignore aware: exclude files via - .gitignore and similar rules. + Git aware: exclude files via .gitignore + and similar rules. {neural_search_enabled && ( - {renderHeader()} - {renderResultList()} -
- ); - } - - if (isFlyout) { - return ( -
- {renderContent()} -
- ); - } else { - return {renderContent()}; - } + return ( +
+ {renderHeader()} + {renderResultList()} +
+ ); }; interface ProjectSearchInputProps { @@ -298,12 +256,10 @@ interface ProjectSearchOutputProps { function ProjectSearchOutput({ project_id, - wrap, mode = "project", }: ProjectSearchOutputProps) { const [filter, setFilter] = useState(""); const [currentFilter, setCurrentFilter] = useState(""); - const isFlyout = mode === "flyout"; const most_recent_search = useTypedRedux( { project_id }, "most_recent_search", @@ -316,10 +272,9 @@ function ProjectSearchOutput({ const search_error = useTypedRedux({ project_id }, "search_error"); const too_many_results = useTypedRedux({ project_id }, "too_many_results"); - useEffect(() => { - setFilter(""); - setCurrentFilter(""); - }, [unfiltered_search_results]); + const virtuosoScroll = useVirtuosoScrollHook({ + cacheId: `search-${project_id}`, + }); const search_results = useMemo(() => { const f = filter?.trim(); @@ -357,148 +312,99 @@ function ProjectSearchOutput({ if (search_results?.size == 0) { return ( - There were no results for your search. + There are no results for your search. ); } - const v: React.JSX.Element[] = []; - let i = 0; - for (const result of search_results) { - v.push( - , - ); - i += 1; - } - return v; + return ( + { + const result = search_results.get(index); + return ( + + ); + }} + {...virtuosoScroll} + /> + ); } function renderResultList() { - if (isFlyout) { - return wrap?.( - - {render_get_results()} - , - { borderTop: `1px solid ${COLORS.GRAY_L}` }, - ); - } else { - return {render_get_results()}; - } + return
{render_get_results()}
; } return ( - <> +
setCurrentFilter(e.target.value)} - placeholder="Filter... (regexp in / /)" + placeholder="Filter results... (regexp in / /)" onSearch={setFilter} enterButton="Filter" style={{ width: "350px", maxWidth: "100%", marginBottom: "15px" }} /> {too_many_results && ( + + {search_results.size} {plural(search_results.size, "Result")}: + {" "} There were more results than displayed below. Try making your search more specific. )} + {!too_many_results && ( + + + {search_results.size} {plural(search_results.size, "Result")} + + + )} {renderResultList()} - +
); } function ProjectSearchOutputHeader({ project_id }: { project_id: string }) { const actions = useActions({ project_id }); - const info_visible = useTypedRedux({ project_id }, "info_visible"); - const search_results = useTypedRedux({ project_id }, "search_results"); - const command = useTypedRedux({ project_id }, "command"); const most_recent_search = useTypedRedux( { project_id }, "most_recent_search", ); const most_recent_path = useTypedRedux({ project_id }, "most_recent_path"); - function output_path() { - return !most_recent_path ? : most_recent_path; - } - - function render_get_info() { - return ( - -
    -
  • - Search command (in a terminal): 
    {command}
    -
    • -
  • - Number of results:{" "} - {search_results ? search_results?.size : } -
    • -
- - ); - } - if (most_recent_search == null || most_recent_path == null) { return ; } return ( -
- - -

- Results of searching in {output_path()} for "{most_recent_search}" - - -

- - {info_visible && render_get_info()} + ); } -const DESC_STYLE: React.CSSProperties = { - color: COLORS.GRAY_M, - marginBottom: "5px", - border: "1px solid #eee", - borderRadius: "5px", - maxHeight: "300px", - padding: "15px", - overflowY: "auto", -} as const; - interface ProjectSearchResultLineProps { project_id: string; filename: string; @@ -509,17 +415,14 @@ interface ProjectSearchResultLineProps { mode?: "project" | "flyout"; } -function ProjectSearchResultLine(_: Readonly) { - const { - project_id, - filename, - description, - line_number, - fragment_id, - most_recent_path, - mode = "project", - } = _; - const isFlyout = mode === "flyout"; +function ProjectSearchResultLine({ + project_id, + filename, + description, + line_number, + fragment_id, + most_recent_path, +}: Readonly) { const actions = useActions({ project_id }); const ext = filename_extension(filename); const icon = file_associations[ext]?.icon ?? "file"; @@ -565,40 +468,32 @@ function ProjectSearchResultLine(_: Readonly) { ); } - if (isFlyout) { - return ( - - } - > - - - ); - } else { - return ( -
- {renderFileLink()} -
- -
-
- ); - } + } + > + + + ); } const MARKDOWN_EXTS = ["tasks", "slides", "board", "sage-chat"] as const; diff --git a/src/packages/frontend/project/search/run.ts b/src/packages/frontend/project/search/run.ts index 840d2f7d30..317783574d 100644 --- a/src/packages/frontend/project/search/run.ts +++ b/src/packages/frontend/project/search/run.ts @@ -1,4 +1,10 @@ import { type FilesystemClient } from "@cocalc/conat/files/fs"; +import { trunc } from "@cocalc/util/misc"; + +// we get about this many bytes of results from the filesystem, then stop... +const MAX_SIZE = 1_000_000; + +const MAX_LINE_LENGTH = 256; interface SearchResult { filename: string; @@ -30,7 +36,7 @@ export async function search({ return; } - const rgOptions = ["--json", "-M", "256"]; + const rgOptions = ["--json"]; // note that -M doesn't seem to combine with --json, so can't do -M {MAX_LINE_LENGTH} if (!options.subdirectories) { rgOptions.push("-d", "1"); } @@ -46,7 +52,7 @@ export async function search({ const { stdout, truncated } = await fs.ripgrep(path, query, { options: rgOptions, - maxSize: 100_000, + maxSize: MAX_SIZE, }); const lines = Buffer.from(stdout).toString().split("\n"); @@ -60,10 +66,11 @@ export async function search({ } if (result.type == "match") { const { line_number, lines, path } = result.data; + const description = trunc(lines?.text ?? "", MAX_LINE_LENGTH); search_results.push({ filename: path?.text ?? "-", - description: `${(line_number.toString() + ":").padEnd(8, " ")}${lines.text}`, - filter: `${path?.text?.toLowerCase?.() ?? ""} ${lines.text.toLowerCase()}`, + description: `${(line_number.toString() + ":").padEnd(8, " ")}${description}`, + filter: `${path?.text?.toLowerCase?.() ?? ""} ${description.toLowerCase()}`, line_number, }); } diff --git a/src/packages/frontend/project/search/search.tsx b/src/packages/frontend/project/search/search.tsx index c9b6a3d53d..19bfa377aa 100644 --- a/src/packages/frontend/project/search/search.tsx +++ b/src/packages/frontend/project/search/search.tsx @@ -4,7 +4,7 @@ import { ProjectSearchHeader } from "./header"; export const ProjectSearch: React.FC = () => { return ( -
+
From 96c1ad48b6c88131d94ff728d11331abe4dfba13 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 5 Aug 2025 00:24:57 +0000 Subject: [PATCH 182/798] add archiver to sandbox for easily making zip/tar files. --- .../backend/files/sandbox/archiver.ts | 51 +++++++ src/packages/backend/files/sandbox/index.ts | 16 ++ src/packages/backend/package.json | 13 +- src/packages/conat/files/fs.ts | 27 ++++ src/packages/pnpm-lock.yaml | 144 ++++++++++++++++++ 5 files changed, 249 insertions(+), 2 deletions(-) create mode 100644 src/packages/backend/files/sandbox/archiver.ts diff --git a/src/packages/backend/files/sandbox/archiver.ts b/src/packages/backend/files/sandbox/archiver.ts new file mode 100644 index 0000000000..73f447a4e7 --- /dev/null +++ b/src/packages/backend/files/sandbox/archiver.ts @@ -0,0 +1,51 @@ +import Archiver from "archiver"; +import { createWriteStream } from "fs"; +import { stat } from "fs/promises"; +import { once } from "@cocalc/util/async-utils"; +import { type ArchiverOptions } from "@cocalc/conat/files/fs"; +export { type ArchiverOptions }; + +export default async function archiver( + path: string, + paths: string[] | string, + options?: ArchiverOptions, +) { + if (options == null) { + if (path.endsWith(".zip")) { + options = { format: "zip" }; + } else if (path.endsWith(".tar")) { + options = { format: "tar" }; + } else if (path.endsWith(".tar.gz")) { + options = { format: "tar", gzip: true }; + } + } + if (typeof paths == "string") { + paths = [paths]; + } + const archive = new Archiver(options!.format, options); + const output = createWriteStream(path); + // docs say to listen before calling finalize + const closed = once(output, "close"); + archive.pipe(output); + + let error: any = undefined; + archive.on("error", (err) => { + error = err; + }); + + const isDir = async (path) => (await stat(path)).isDirectory(); + const v = await Promise.all(paths.map(isDir)); + for (let i = 0; i < paths.length; i++) { + if (v[i]) { + archive.directory(paths[i]); + } else { + archive.file(paths[i]); + } + } + + await archive.finalize(); + await closed; + if (error) { + throw error; + } +} diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index 00a698a5b8..5302f546f0 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -76,6 +76,7 @@ import dust, { type DustOptions } from "./dust"; import rustic from "./rustic"; import { type ExecOutput } from "./exec"; import { rusticRepo } from "@cocalc/backend/data"; +import archiver, { type ArchiverOptions } from "./archiver"; // max time code can run (in safe mode), e.g., for find, // ripgrep, fd, and dust. @@ -243,6 +244,21 @@ export class SandboxedFilesystem { }); }; + archiver = async ( + path: string, + paths: string[] | string, + options?: ArchiverOptions, + ): Promise => { + if (typeof paths == "string") { + paths = [paths]; + } + await archiver( + await this.safeAbsPath(path), + await Promise.all(paths.map(this.safeAbsPath)), + options, + ); + }; + ripgrep = async ( path: string, pattern: string, diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index 2c45ae00dd..056ce72824 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -13,7 +13,10 @@ "./auth/*": "./dist/auth/*.js", "./auth/tokens/*": "./dist/auth/tokens/*.js" }, - "keywords": ["utilities", "cocalc"], + "keywords": [ + "utilities", + "cocalc" + ], "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf dist node_modules", @@ -31,13 +34,19 @@ "conat-persist": "DEBUG=cocalc:* node ./bin/conat-persist.cjs", "conat-test-server": "node ./bin/conat-test-server.cjs" }, - "files": ["dist/**", "bin/**", "README.md", "package.json"], + "files": [ + "dist/**", + "bin/**", + "README.md", + "package.json" + ], "author": "SageMath, Inc.", "license": "SEE LICENSE.md", "dependencies": { "@cocalc/backend": "workspace:*", "@cocalc/conat": "workspace:*", "@cocalc/util": "workspace:*", + "archiver": "^7.0.1", "awaiting": "^3.0.0", "better-sqlite3": "^12.2.0", "chokidar": "^3.6.0", diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index c02ad3f984..f675a08ecd 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -59,6 +59,24 @@ export interface DustOptions { maxSize?: number; } +interface ZipOptions { + format: "zip"; + comment?: string; + forceLocalTime?: boolean; + forceZip64?: boolean; + namePrependSlash?: boolean; + store?: boolean; + zlib?: object; +} + +interface TarOptions { + format: "tar"; + gzip?: boolean; + gzipOPtions?: object; +} + +export type ArchiverOptions = ZipOptions | TarOptions; + export interface Filesystem { appendFile: (path: string, data: string | Buffer, encoding?) => Promise; chmod: (path: string, mode: string | number) => Promise; @@ -91,6 +109,12 @@ export interface Filesystem { // todo: typing watch: (path: string, options?) => Promise; + archiver: ( + path: string, // archive to create, e.g., a.zip, a.tar or a.tar.gz + paths: string, // paths to include in archive -- can be files or directories in the sandbox. + options?: ArchiverOptions, // options -- see https://www.archiverjs.com/ + ) => Promise; + // We add very little to the Filesystem api, but we have to add // a sandboxed "find" command, since it is a 1-call way to get // arbitrary directory listing info, which is just not possible @@ -263,6 +287,9 @@ export async function fsServer({ service, fs, client, project_id }: Options) { async appendFile(path: string, data: string | Buffer, encoding?) { await (await fs(this.subject)).appendFile(path, data, encoding); }, + async archiver(path: string, paths: string, options?: ArchiverOptions) { + return await (await fs(this.subject)).archiver(path, paths, options); + }, async chmod(path: string, mode: string | number) { await (await fs(this.subject)).chmod(path, mode); }, diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 00ebbf5056..4d90e0f376 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -87,6 +87,9 @@ importers: '@cocalc/util': specifier: workspace:* version: link:../util + archiver: + specifier: ^7.0.1 + version: 7.0.1 awaiting: specifier: ^3.0.0 version: 3.0.0 @@ -4804,6 +4807,14 @@ packages: resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==} engines: {node: '>= 8'} + archiver-utils@5.0.2: + resolution: {integrity: sha512-wuLJMmIBQYCsGZgYLTy5FIB2pF6Lfb6cXMSF8Qywwk3t20zWnAi7zLcQFdKQmIB8wyZpY5ER38x08GbwtR2cLA==} + engines: {node: '>= 14'} + + archiver@7.0.1: + resolution: {integrity: sha512-ZcbTaIqJOfCc03QwD468Unz/5Ir8ATtvAHsK+FdXbDIbGfihqh9mrvdcYunQzqn4HrvWWaFyaxJhGZagaJJpPQ==} + engines: {node: '>= 14'} + argparse@1.0.10: resolution: {integrity: sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==} @@ -4970,6 +4981,9 @@ packages: axios@1.11.0: resolution: {integrity: sha512-1Lx3WLFQWm3ooKDYZD1eXmoGO9fxYQjrycfHFC8P0sCfQVXyROp0p9PFWBehewBOdCwHc+f/b8I0fMto5eSfwA==} + b4a@1.6.7: + resolution: {integrity: sha512-OnAYlL5b7LEkALw87fUVafQw5rVR9RjwGd4KUwNQ6DrrNmaVaUCgLipfVlzrPQ4tWOR9P0IXGNOx50jYCCdSJg==} + babel-jest@29.7.0: resolution: {integrity: sha512-BrvGY3xZSwEcCzKvKsCi2GgHqDqsYkOP4/by5xCgIwGXQxIEh+8ew3gmrE1y7XRR6LHZIj6yLYnUi/mm2KXKBg==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} @@ -5014,6 +5028,9 @@ packages: balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} + bare-events@2.6.0: + resolution: {integrity: sha512-EKZ5BTXYExaNqi3I3f9RtEsaI/xBSGjE0XZCZilPzFAV/goswFHuPd9jEZlPIZ/iNZJwDSao9qRiScySz7MbQg==} + base-64@1.0.0: resolution: {integrity: sha512-kwDPIFCGx0NZHog36dj+tHiwP4QMzsZ3AgMViUBKI0+V5n4U0ufTCUMhnQ04diaRI8EX/QcPfql7zlhZ7j4zgg==} @@ -5138,6 +5155,10 @@ packages: bser@2.1.1: resolution: {integrity: sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==} + buffer-crc32@1.0.0: + resolution: {integrity: sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==} + engines: {node: '>=8.0.0'} + buffer-equal-constant-time@1.0.1: resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} @@ -5500,6 +5521,10 @@ packages: compare-versions@4.1.4: resolution: {integrity: sha512-FemMreK9xNyL8gQevsdRMrvO4lFCkQP7qbuktn1q8ndcNk1+0mz7lgE7b/sNvbhVgY4w6tMN1FDp6aADjqw2rw==} + compress-commons@6.0.2: + resolution: {integrity: sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg==} + engines: {node: '>= 14'} + compressible@2.0.18: resolution: {integrity: sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==} engines: {node: '>= 0.6'} @@ -5629,6 +5654,15 @@ packages: country-regex@1.1.0: resolution: {integrity: sha512-iSPlClZP8vX7MC3/u6s3lrDuoQyhQukh5LyABJ3hvfzbQ3Yyayd4fp04zjLnfi267B/B2FkumcWWgrbban7sSA==} + crc-32@1.2.2: + resolution: {integrity: sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==} + engines: {node: '>=0.8'} + hasBin: true + + crc32-stream@6.0.0: + resolution: {integrity: sha512-piICUB6ei4IlTv1+653yq5+KoqfBYmj9bw6LqXoOneTMDXk5nM1qt12mFW1caG3LlJXEKW1Bp0WggEmIfQB34g==} + engines: {node: '>= 14'} + create-error-class@3.0.2: resolution: {integrity: sha512-gYTKKexFO3kh200H1Nit76sRwRtOY32vQd3jpAQKpLtZqyNsSQNfI4N7o3eP2wUjV35pTWKRYqFUDBvUha/Pkw==} engines: {node: '>=0.10.0'} @@ -6671,6 +6705,9 @@ packages: resolution: {integrity: sha512-V7/RktU11J3I36Nwq2JnZEM7tNm17eBJz+u25qdxBZeCKiX6BkVSZQjwWIr+IobgnZy+ag73tTZgZi7tr0LrBw==} engines: {node: '>=6.0.0'} + fast-fifo@1.3.2: + resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==} + fast-glob@3.3.3: resolution: {integrity: sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==} engines: {node: '>=8.6.0'} @@ -8259,6 +8296,10 @@ packages: layout-base@2.0.1: resolution: {integrity: sha512-dp3s92+uNI1hWIpPGH3jK2kxE2lMjdXdr+DH8ynZHpd6PUlH6x6cbuXnoMmiNumznqaNO31xu9e79F0uuZ0JFg==} + lazystream@1.0.1: + resolution: {integrity: sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==} + engines: {node: '>= 0.6.3'} + ldap-filter@0.3.3: resolution: {integrity: sha512-/tFkx5WIn4HuO+6w9lsfxq4FN3O+fDZeO9Mek8dCD8rTUpqzRa766BOBO7BcGkn3X86m5+cBm1/2S/Shzz7gMg==} engines: {node: '>=0.8'} @@ -10044,6 +10085,13 @@ packages: resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} engines: {node: '>= 6'} + readable-stream@4.7.0: + resolution: {integrity: sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + + readdir-glob@1.1.3: + resolution: {integrity: sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==} + readdirp@3.6.0: resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==} engines: {node: '>=8.10.0'} @@ -10603,6 +10651,9 @@ packages: resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} engines: {node: '>=10.0.0'} + streamx@2.22.1: + resolution: {integrity: sha512-znKXEBxfatz2GBNK02kRnCXjV+AA4kjZIUxeWSr3UGirZMJfTE9uiwKHobnbgxWyL/JWro8tTq+vOqAK1/qbSA==} + string-convert@0.2.1: resolution: {integrity: sha512-u/1tdPl4yQnPBjnVrmdLo9gtuLvELKsAoRapekWggdiQNvvvum+jYF329d84NAa660KQw7pB2n36KrIKVoXa3A==} @@ -10795,6 +10846,9 @@ packages: resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} engines: {node: '>=6'} + tar-stream@3.1.7: + resolution: {integrity: sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==} + tdigest@0.1.2: resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==} @@ -10836,6 +10890,9 @@ packages: resolution: {integrity: sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==} engines: {node: '>=8'} + text-decoder@1.2.3: + resolution: {integrity: sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA==} + text-hex@1.0.0: resolution: {integrity: sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==} @@ -11747,6 +11804,10 @@ packages: resolution: {integrity: sha512-vWOrt19lvcXTxu5tiHXfEGQuldSlU+qZn2TT+4EbRQzaciWGwNZ99QQTolQOmcwVgZLodv+1QfC6UZs2PX/6pQ==} engines: {node: '>= 12'} + zip-stream@6.0.1: + resolution: {integrity: sha512-zK7YHHz4ZXpW89AHXUPbQVGKI7uvkd3hzusTdotCg1UxyaVtg0zFJSTfW/Dq5f7OBBVnq6cZIaC8Ti4hb6dtCA==} + engines: {node: '>= 14'} + zlibjs@0.3.1: resolution: {integrity: sha512-+J9RrgTKOmlxFSDHo0pI1xM6BLVUv+o0ZT9ANtCxGkjIVCCUdx9alUF8Gm+dGLKbkkkidWIHFDZHDMpfITt4+w==} @@ -15195,6 +15256,26 @@ snapshots: normalize-path: 3.0.0 picomatch: 2.3.1 + archiver-utils@5.0.2: + dependencies: + glob: 10.4.5 + graceful-fs: 4.2.11 + is-stream: 2.0.1 + lazystream: 1.0.1 + lodash: 4.17.21 + normalize-path: 3.0.0 + readable-stream: 4.7.0 + + archiver@7.0.1: + dependencies: + archiver-utils: 5.0.2 + async: 3.2.6 + buffer-crc32: 1.0.0 + readable-stream: 4.7.0 + readdir-glob: 1.1.3 + tar-stream: 3.1.7 + zip-stream: 6.0.1 + argparse@1.0.10: dependencies: sprintf-js: 1.0.3 @@ -15374,6 +15455,8 @@ snapshots: transitivePeerDependencies: - debug + b4a@1.6.7: {} + babel-jest@29.7.0(@babel/core@7.28.0): dependencies: '@babel/core': 7.28.0 @@ -15450,6 +15533,9 @@ snapshots: balanced-match@1.0.2: {} + bare-events@2.6.0: + optional: true + base-64@1.0.0: {} base16@1.0.0: {} @@ -15572,6 +15658,8 @@ snapshots: dependencies: node-int64: 0.4.0 + buffer-crc32@1.0.0: {} + buffer-equal-constant-time@1.0.1: {} buffer-from@1.1.2: {} @@ -15965,6 +16053,14 @@ snapshots: compare-versions@4.1.4: {} + compress-commons@6.0.2: + dependencies: + crc-32: 1.2.2 + crc32-stream: 6.0.0 + is-stream: 2.0.1 + normalize-path: 3.0.0 + readable-stream: 4.7.0 + compressible@2.0.18: dependencies: mime-db: 1.54.0 @@ -16097,6 +16193,13 @@ snapshots: country-regex@1.1.0: {} + crc-32@1.2.2: {} + + crc32-stream@6.0.0: + dependencies: + crc-32: 1.2.2 + readable-stream: 4.7.0 + create-error-class@3.0.2: dependencies: capture-stack-trace: 1.0.2 @@ -17384,6 +17487,8 @@ snapshots: fast-equals@5.2.2: {} + fast-fifo@1.3.2: {} + fast-glob@3.3.3: dependencies: '@nodelib/fs.stat': 2.0.5 @@ -19399,6 +19504,10 @@ snapshots: layout-base@2.0.1: {} + lazystream@1.0.1: + dependencies: + readable-stream: 2.3.8 + ldap-filter@0.3.3: dependencies: assert-plus: 1.0.0 @@ -21516,6 +21625,18 @@ snapshots: string_decoder: 1.3.0 util-deprecate: 1.0.2 + readable-stream@4.7.0: + dependencies: + abort-controller: 3.0.0 + buffer: 6.0.3 + events: 3.3.0 + process: 0.11.10 + string_decoder: 1.3.0 + + readdir-glob@1.1.3: + dependencies: + minimatch: 5.1.6 + readdirp@3.6.0: dependencies: picomatch: 2.3.1 @@ -22241,6 +22362,13 @@ snapshots: streamsearch@1.1.0: {} + streamx@2.22.1: + dependencies: + fast-fifo: 1.3.2 + text-decoder: 1.2.3 + optionalDependencies: + bare-events: 2.6.0 + string-convert@0.2.1: {} string-length@4.0.2: @@ -22464,6 +22592,12 @@ snapshots: inherits: 2.0.4 readable-stream: 3.6.2 + tar-stream@3.1.7: + dependencies: + b4a: 1.6.7 + fast-fifo: 1.3.2 + streamx: 2.22.1 + tdigest@0.1.2: dependencies: bintrees: 1.0.2 @@ -22524,6 +22658,10 @@ snapshots: glob: 7.2.3 minimatch: 3.1.2 + text-decoder@1.2.3: + dependencies: + b4a: 1.6.7 + text-hex@1.0.0: {} text-table@0.2.0: {} @@ -23469,6 +23607,12 @@ snapshots: cmake-ts: 1.0.2 node-addon-api: 8.4.0 + zip-stream@6.0.1: + dependencies: + archiver-utils: 5.0.2 + compress-commons: 6.0.2 + readable-stream: 4.7.0 + zlibjs@0.3.1: {} zod-to-json-schema@3.21.4(zod@3.25.76): From 4f788e4eadaf7900271a80145436c20d7bcf0ea4 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 5 Aug 2025 05:38:45 +0000 Subject: [PATCH 183/798] ouch -- rust based compression --- .../backend/files/sandbox/archiver.ts | 56 ++++++++++++------- src/packages/backend/files/sandbox/index.ts | 36 +++++++++--- src/packages/backend/files/sandbox/install.ts | 34 +++++++++-- .../backend/files/sandbox/ouch.test.ts | 56 +++++++++++++++++++ src/packages/backend/files/sandbox/ouch.ts | 52 +++++++++++++++++ src/packages/conat/files/fs.ts | 33 +++++++++-- 6 files changed, 229 insertions(+), 38 deletions(-) create mode 100644 src/packages/backend/files/sandbox/ouch.test.ts create mode 100644 src/packages/backend/files/sandbox/ouch.ts diff --git a/src/packages/backend/files/sandbox/archiver.ts b/src/packages/backend/files/sandbox/archiver.ts index 73f447a4e7..18ca316ca0 100644 --- a/src/packages/backend/files/sandbox/archiver.ts +++ b/src/packages/backend/files/sandbox/archiver.ts @@ -1,5 +1,5 @@ import Archiver from "archiver"; -import { createWriteStream } from "fs"; +import { createReadStream, createWriteStream } from "fs"; import { stat } from "fs/promises"; import { once } from "@cocalc/util/async-utils"; import { type ArchiverOptions } from "@cocalc/conat/files/fs"; @@ -7,45 +7,61 @@ export { type ArchiverOptions }; export default async function archiver( path: string, - paths: string[] | string, + // map from absolute path to path as it should appear in the archive + pathMap: { [absolutePath: string]: string | null }, options?: ArchiverOptions, ) { - if (options == null) { - if (path.endsWith(".zip")) { - options = { format: "zip" }; - } else if (path.endsWith(".tar")) { - options = { format: "tar" }; - } else if (path.endsWith(".tar.gz")) { - options = { format: "tar", gzip: true }; - } - } - if (typeof paths == "string") { - paths = [paths]; - } - const archive = new Archiver(options!.format, options); + options = { ...options }; + const format = getFormat(path, options); + const archive = new Archiver(format, options); + + let error: any = undefined; + + const timer = options.timeout + ? setTimeout(() => { + error = `Timeout after ${options.timeout} ms`; + archive.abort(); + }, options.timeout) + : undefined; + const output = createWriteStream(path); // docs say to listen before calling finalize const closed = once(output, "close"); archive.pipe(output); - let error: any = undefined; archive.on("error", (err) => { error = err; }); + const paths = Object.keys(pathMap); const isDir = async (path) => (await stat(path)).isDirectory(); const v = await Promise.all(paths.map(isDir)); for (let i = 0; i < paths.length; i++) { + const name = pathMap[paths[i]]; if (v[i]) { - archive.directory(paths[i]); + archive.directory(paths[i], name); } else { - archive.file(paths[i]); + archive.append(createReadStream(paths[i]), { name }); } } - await archive.finalize(); + archive.finalize(); await closed; + if (timer) { + clearTimeout(timer); + } if (error) { - throw error; + throw Error(error); + } +} + +function getFormat(path: string, options) { + if (path.endsWith(".zip")) { + return "zip"; + } else if (path.endsWith(".tar")) { + return "tar"; + } else if (path.endsWith(".tar.gz")) { + options.gzip = true; + return "tar"; } } diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index 5302f546f0..1f361dbc36 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -76,6 +76,7 @@ import dust, { type DustOptions } from "./dust"; import rustic from "./rustic"; import { type ExecOutput } from "./exec"; import { rusticRepo } from "@cocalc/backend/data"; +import ouch, { type OuchOptions } from "./ouch"; import archiver, { type ArchiverOptions } from "./archiver"; // max time code can run (in safe mode), e.g., for find, @@ -218,6 +219,7 @@ export class SandboxedFilesystem { ); }; + // find files fd = async (path: string, options?: FdOptions): Promise => { return await fd( await this.safeAbsPath(path), @@ -225,6 +227,7 @@ export class SandboxedFilesystem { ); }; + // disk usage dust = async (path: string, options?: DustOptions): Promise => { return await dust( await this.safeAbsPath(path), @@ -234,6 +237,19 @@ export class SandboxedFilesystem { ); }; + // compression + ouch = async (args: string[], options?: OuchOptions): Promise => { + options = { ...options }; + if (options.cwd) { + options.cwd = await this.safeAbsPath(options.cwd); + } + return await ouch( + [args[0]].concat(await Promise.all(args.slice(1).map(this.safeAbsPath))), + capTimeout(options, 6 * MAX_TIMEOUT), + ); + }; + + // backups rustic = async (args: string[]): Promise => { return await rustic(args, { repo: this.rusticRepo, @@ -246,17 +262,21 @@ export class SandboxedFilesystem { archiver = async ( path: string, - paths: string[] | string, + // map from path relative to sandbox to the name that path should get in the archive. + pathMap0: { [path: string]: string | null }, options?: ArchiverOptions, ): Promise => { - if (typeof paths == "string") { - paths = [paths]; + const pathMap: { [absPath: string]: string } = {}; + const v = Object.keys(pathMap0); + const absPaths = await Promise.all(v.map(this.safeAbsPath)); + for (let i = 0; i < v.length; i++) { + pathMap[absPaths[i]] = + pathMap0[v[i]] ?? absPaths[i].slice(this.path.length + 1); } - await archiver( - await this.safeAbsPath(path), - await Promise.all(paths.map(this.safeAbsPath)), - options, - ); + await archiver(await this.safeAbsPath(path), pathMap, { + timeout: 4 * MAX_TIMEOUT, + ...options, + }); }; ripgrep = async ( diff --git a/src/packages/backend/files/sandbox/install.ts b/src/packages/backend/files/sandbox/install.ts index d22f1d3634..9a66d895f7 100644 --- a/src/packages/backend/files/sandbox/install.ts +++ b/src/packages/backend/files/sandbox/install.ts @@ -33,6 +33,7 @@ interface Spec { path: string; stripComponents?: number; pathInArchive?: string; + skip?: string[]; } const SPEC = { @@ -57,6 +58,16 @@ const SPEC = { binary: "dust", path: join(binPath, "dust"), }, + ouch: { + // See https://github.com/ouch-org/ouch/releases + VERSION: "0.6.1", + BASE: "https://github.com/ouch-org/ouch/releases/download", + binary: "ouch", + path: join(binPath, "ouch"), + // See https://github.com/ouch-org/ouch/issues/45; note that ouch is in home brew + // for this platform. + skip: ["aarch64-apple-darwin"], + }, rustic: { // See https://github.com/rustic-rs/rustic/releases VERSION: "v0.9.5", @@ -72,6 +83,7 @@ export const ripgrep = SPEC.ripgrep.path; export const fd = SPEC.fd.path; export const dust = SPEC.dust.path; export const rustic = SPEC.rustic.path; +export const ouch = SPEC.ouch.path; type App = keyof typeof SPEC; @@ -101,6 +113,10 @@ export async function install(app?: App) { return; } const url = getUrl(app); + if (!url) { + logger.debug("install: skipping ", app); + return; + } logger.debug("install", { app, url }); // - 1. Fetch the tarball from the github url (using the fetch library) const response = await downloadFromGithub(url); @@ -118,7 +134,9 @@ export async function install(app?: App) { binary, path, stripComponents = 1, - pathInArchive = `${app}-${VERSION}-${getOS()}/${binary}`, + pathInArchive = app == "ouch" + ? `${app}-${getOS()}/${binary}` + : `${app}-${VERSION}-${getOS()}/${binary}`, } = SPEC[app] as Spec; const tmpFile = join(__dirname, `${app}-${VERSION}.tar.gz`); @@ -186,7 +204,7 @@ async function downloadFromGithub(url: string) { const delay = baseDelay * Math.pow(2, attempt - 1); console.log( - `Fetch failed. Retrying in ${delay}ms (attempt ${attempt}/${maxRetries})`, + `Fetch ${url} failed. Retrying in ${delay}ms (attempt ${attempt}/${maxRetries})`, ); await new Promise((resolve) => setTimeout(resolve, delay)); } @@ -195,8 +213,16 @@ async function downloadFromGithub(url: string) { } function getUrl(app: App) { - const { BASE, VERSION } = SPEC[app]; - return `${BASE}/${VERSION}/${app}-${VERSION}-${getOS()}.tar.gz`; + const { BASE, VERSION, skip } = SPEC[app] as Spec; + const os = getOS(); + if (skip?.includes(os)) { + return ""; + } + if (app == "ouch") { + return `${BASE}/${VERSION}/${app}-${os}.tar.gz`; + } else { + return `${BASE}/${VERSION}/${app}-${VERSION}-${os}.tar.gz`; + } } function getOS() { diff --git a/src/packages/backend/files/sandbox/ouch.test.ts b/src/packages/backend/files/sandbox/ouch.test.ts new file mode 100644 index 0000000000..5d9ded0afb --- /dev/null +++ b/src/packages/backend/files/sandbox/ouch.test.ts @@ -0,0 +1,56 @@ +/* +Test the ouch compression api. +*/ + +import ouch from "./ouch"; +import { mkdtemp, mkdir, rm, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { exists } from "@cocalc/backend/misc/async-utils-node"; + +let tempDir, options; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); + options = { cwd: tempDir }; +}); +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); + +describe("ouch works on a little file", () => { + for (const ext of [ + "zip", + "7z", + "tar.gz", + "tar.xz", + "tar.bz", + "tar.bz2", + "tar.bz3", + "tar.lz4", + "tar.sz", + "tar.zst", + "tar.br", + ]) { + it(`create file and compress it up using ${ext}`, async () => { + await writeFile(join(tempDir, "a.txt"), "hello"); + const { truncated, code } = await ouch( + ["compress", "a.txt", `a.${ext}`], + options, + ); + expect(code).toBe(0); + expect(truncated).toBe(false); + expect(await exists(join(tempDir, `a.${ext}`))).toBe(true); + }); + + it(`extract ${ext} in subdirectory`, async () => { + await mkdir(join(tempDir, `target-${ext}`)); + const { code } = await ouch(["decompress", join(tempDir, `a.${ext}`)], { + cwd: join(tempDir, `target-${ext}`), + }); + expect(code).toBe(0); + expect( + (await readFile(join(tempDir, `target-${ext}`, "a.txt"))).toString(), + ).toEqual("hello"); + }); + } +}); diff --git a/src/packages/backend/files/sandbox/ouch.ts b/src/packages/backend/files/sandbox/ouch.ts new file mode 100644 index 0000000000..51b94a8270 --- /dev/null +++ b/src/packages/backend/files/sandbox/ouch.ts @@ -0,0 +1,52 @@ +/* + +https://github.com/ouch-org/ouch + +ouch stands for Obvious Unified Compression Helper. + +The .tar.gz support in 'ouch' is excellent -- super fast and memory efficient, +since it is fully parallel. + +*/ + +import exec, { type ExecOutput, validate } from "./exec"; +import { type OuchOptions } from "@cocalc/conat/files/fs"; +export { type OuchOptions }; +import { ouch as ouchPath } from "./install"; + +export default async function ouch( + args: string[], + { timeout, options, cwd }: OuchOptions = {}, +): Promise { + const command = args[0]; + if (!commands.includes(command)) { + throw Error(`first argument must be one of ${commands.join(", ")}`); + } + + return await exec({ + cmd: ouchPath, + cwd, + positionalArgs: args.slice(1), + safety: [command, "-y", "-q"], + timeout, + options, + whitelist, + }); +} + +const commands = ["compress", "c", "decompress", "d", "list", "l", "ls"]; + +const whitelist = { + "-H": true, + "--hidden": true, + g: true, + "--gitignore": true, + "-f": validate.str, + "--format": validate.str, + "-p": validate.str, + "--password": validate.str, + "-h": true, + "--help": true, + "-V": true, + "--version": true, +} as const; diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index f675a08ecd..e3a1670651 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -59,20 +59,26 @@ export interface DustOptions { maxSize?: number; } +export interface OuchOptions { + cwd?: string; + options?: string[]; + timeout?: number; +} + interface ZipOptions { - format: "zip"; comment?: string; forceLocalTime?: boolean; forceZip64?: boolean; namePrependSlash?: boolean; store?: boolean; zlib?: object; + timeout?: number; } interface TarOptions { - format: "tar"; gzip?: boolean; gzipOPtions?: object; + timeout?: number; } export type ArchiverOptions = ZipOptions | TarOptions; @@ -111,10 +117,14 @@ export interface Filesystem { archiver: ( path: string, // archive to create, e.g., a.zip, a.tar or a.tar.gz - paths: string, // paths to include in archive -- can be files or directories in the sandbox. + // map from path relative to sandbox to the name that path should get in the archive. + pathMap: { [path: string]: string | null }, options?: ArchiverOptions, // options -- see https://www.archiverjs.com/ ) => Promise; + // compression + ouch: (args: string[], options?: OuchOptions) => Promise; + // We add very little to the Filesystem api, but we have to add // a sandboxed "find" command, since it is a 1-call way to get // arbitrary directory listing info, which is just not possible @@ -287,8 +297,12 @@ export async function fsServer({ service, fs, client, project_id }: Options) { async appendFile(path: string, data: string | Buffer, encoding?) { await (await fs(this.subject)).appendFile(path, data, encoding); }, - async archiver(path: string, paths: string, options?: ArchiverOptions) { - return await (await fs(this.subject)).archiver(path, paths, options); + async archiver( + path: string, + pathMap: { [path: string]: string | null }, + options?: ArchiverOptions, + ) { + return await (await fs(this.subject)).archiver(path, pathMap, options); }, async chmod(path: string, mode: string | number) { await (await fs(this.subject)).chmod(path, mode); @@ -323,6 +337,9 @@ export async function fsServer({ service, fs, client, project_id }: Options) { async mkdir(path: string, options?) { await (await fs(this.subject)).mkdir(path, options); }, + async ouch(args: string[], options?: OuchOptions) { + return await (await fs(this.subject)).ouch(args, options); + }, async readFile(path: string, encoding?) { return await (await fs(this.subject)).readFile(path, encoding); }, @@ -451,15 +468,19 @@ export function fsSubject({ return `${getService({ service, compute_server_id })}.project-${project_id}`; } +const DEFAULT_FS_CALL_TIMEOUT = 5 * 60_000; + export function fsClient({ client, subject, + timeout = DEFAULT_FS_CALL_TIMEOUT, }: { client?: Client; subject: string; + timeout?: number; }): FilesystemClient { client ??= conat(); - let call = client.call(subject); + let call = client.call(subject, { timeout }); const readdir0 = call.readdir.bind(call); call.readdir = async (path: string, options?) => { From 606a5978b8b23783ff14aae32cde5f9befad4275 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 5 Aug 2025 05:45:04 +0000 Subject: [PATCH 184/798] remove archiver integration completely -- it's just not fast enough to be worth it --- .../backend/files/sandbox/archiver.ts | 67 ------------------- src/packages/backend/files/sandbox/index.ts | 20 ------ .../backend/files/sandbox/ouch.test.ts | 2 +- src/packages/conat/files/fs.ts | 32 --------- 4 files changed, 1 insertion(+), 120 deletions(-) delete mode 100644 src/packages/backend/files/sandbox/archiver.ts diff --git a/src/packages/backend/files/sandbox/archiver.ts b/src/packages/backend/files/sandbox/archiver.ts deleted file mode 100644 index 18ca316ca0..0000000000 --- a/src/packages/backend/files/sandbox/archiver.ts +++ /dev/null @@ -1,67 +0,0 @@ -import Archiver from "archiver"; -import { createReadStream, createWriteStream } from "fs"; -import { stat } from "fs/promises"; -import { once } from "@cocalc/util/async-utils"; -import { type ArchiverOptions } from "@cocalc/conat/files/fs"; -export { type ArchiverOptions }; - -export default async function archiver( - path: string, - // map from absolute path to path as it should appear in the archive - pathMap: { [absolutePath: string]: string | null }, - options?: ArchiverOptions, -) { - options = { ...options }; - const format = getFormat(path, options); - const archive = new Archiver(format, options); - - let error: any = undefined; - - const timer = options.timeout - ? setTimeout(() => { - error = `Timeout after ${options.timeout} ms`; - archive.abort(); - }, options.timeout) - : undefined; - - const output = createWriteStream(path); - // docs say to listen before calling finalize - const closed = once(output, "close"); - archive.pipe(output); - - archive.on("error", (err) => { - error = err; - }); - - const paths = Object.keys(pathMap); - const isDir = async (path) => (await stat(path)).isDirectory(); - const v = await Promise.all(paths.map(isDir)); - for (let i = 0; i < paths.length; i++) { - const name = pathMap[paths[i]]; - if (v[i]) { - archive.directory(paths[i], name); - } else { - archive.append(createReadStream(paths[i]), { name }); - } - } - - archive.finalize(); - await closed; - if (timer) { - clearTimeout(timer); - } - if (error) { - throw Error(error); - } -} - -function getFormat(path: string, options) { - if (path.endsWith(".zip")) { - return "zip"; - } else if (path.endsWith(".tar")) { - return "tar"; - } else if (path.endsWith(".tar.gz")) { - options.gzip = true; - return "tar"; - } -} diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index 1f361dbc36..0994d207fd 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -77,7 +77,6 @@ import rustic from "./rustic"; import { type ExecOutput } from "./exec"; import { rusticRepo } from "@cocalc/backend/data"; import ouch, { type OuchOptions } from "./ouch"; -import archiver, { type ArchiverOptions } from "./archiver"; // max time code can run (in safe mode), e.g., for find, // ripgrep, fd, and dust. @@ -260,25 +259,6 @@ export class SandboxedFilesystem { }); }; - archiver = async ( - path: string, - // map from path relative to sandbox to the name that path should get in the archive. - pathMap0: { [path: string]: string | null }, - options?: ArchiverOptions, - ): Promise => { - const pathMap: { [absPath: string]: string } = {}; - const v = Object.keys(pathMap0); - const absPaths = await Promise.all(v.map(this.safeAbsPath)); - for (let i = 0; i < v.length; i++) { - pathMap[absPaths[i]] = - pathMap0[v[i]] ?? absPaths[i].slice(this.path.length + 1); - } - await archiver(await this.safeAbsPath(path), pathMap, { - timeout: 4 * MAX_TIMEOUT, - ...options, - }); - }; - ripgrep = async ( path: string, pattern: string, diff --git a/src/packages/backend/files/sandbox/ouch.test.ts b/src/packages/backend/files/sandbox/ouch.test.ts index 5d9ded0afb..96ae332833 100644 --- a/src/packages/backend/files/sandbox/ouch.test.ts +++ b/src/packages/backend/files/sandbox/ouch.test.ts @@ -5,7 +5,7 @@ Test the ouch compression api. import ouch from "./ouch"; import { mkdtemp, mkdir, rm, readFile, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join, resolve } from "node:path"; +import { join } from "node:path"; import { exists } from "@cocalc/backend/misc/async-utils-node"; let tempDir, options; diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index e3a1670651..67e397176f 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -65,24 +65,6 @@ export interface OuchOptions { timeout?: number; } -interface ZipOptions { - comment?: string; - forceLocalTime?: boolean; - forceZip64?: boolean; - namePrependSlash?: boolean; - store?: boolean; - zlib?: object; - timeout?: number; -} - -interface TarOptions { - gzip?: boolean; - gzipOPtions?: object; - timeout?: number; -} - -export type ArchiverOptions = ZipOptions | TarOptions; - export interface Filesystem { appendFile: (path: string, data: string | Buffer, encoding?) => Promise; chmod: (path: string, mode: string | number) => Promise; @@ -115,13 +97,6 @@ export interface Filesystem { // todo: typing watch: (path: string, options?) => Promise; - archiver: ( - path: string, // archive to create, e.g., a.zip, a.tar or a.tar.gz - // map from path relative to sandbox to the name that path should get in the archive. - pathMap: { [path: string]: string | null }, - options?: ArchiverOptions, // options -- see https://www.archiverjs.com/ - ) => Promise; - // compression ouch: (args: string[], options?: OuchOptions) => Promise; @@ -297,13 +272,6 @@ export async function fsServer({ service, fs, client, project_id }: Options) { async appendFile(path: string, data: string | Buffer, encoding?) { await (await fs(this.subject)).appendFile(path, data, encoding); }, - async archiver( - path: string, - pathMap: { [path: string]: string | null }, - options?: ArchiverOptions, - ) { - return await (await fs(this.subject)).archiver(path, pathMap, options); - }, async chmod(path: string, mode: string | number) { await (await fs(this.subject)).chmod(path, mode); }, From 31b040a31913f164767d56289c84de42e580503b Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 5 Aug 2025 05:56:33 +0000 Subject: [PATCH 185/798] ability to compress directory using new fs api --- .../project/explorer/create-archive.tsx | 34 ++++++++++++------- src/packages/frontend/project_actions.ts | 1 - 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/packages/frontend/project/explorer/create-archive.tsx b/src/packages/frontend/project/explorer/create-archive.tsx index 1ae72fb230..2318a45e24 100644 --- a/src/packages/frontend/project/explorer/create-archive.tsx +++ b/src/packages/frontend/project/explorer/create-archive.tsx @@ -8,6 +8,9 @@ import { labels } from "@cocalc/frontend/i18n"; import { useProjectContext } from "@cocalc/frontend/project/context"; import { path_split, plural } from "@cocalc/util/misc"; import CheckedFiles from "./checked-files"; +import { join } from "path"; + +const FORMAT = ".tar.gz"; export default function CreateArchive({ clear }) { const intl = useIntl(); @@ -41,19 +44,22 @@ export default function CreateArchive({ clear }) { setLoading(true); const files = checked_files.toArray(); const path = store.get("current_path"); - await actions.zip_files({ - src: path ? files.map((x) => x.slice(path.length + 1)) : files, - dest: target + ".zip", - path, - }); + const fs = actions.fs(); + const { code, stderr } = await fs.ouch([ + "compress", + ...files, + join(path, target + FORMAT), + ]); + if (code) { + throw Error(Buffer.from(stderr).toString()); + } + clear(); } catch (err) { setLoading(false); setError(err); } finally { setLoading(false); } - - clear(); }; if (actions == null) { @@ -63,8 +69,8 @@ export default function CreateArchive({ clear }) { return ( - Create a zip file from the following {checked_files?.size} selected{" "} - {plural(checked_files?.size, "item")} + Create a downloadable {FORMAT} archive from the following{" "} + {checked_files?.size} selected {plural(checked_files?.size, "item")} > @@ -74,9 +80,9 @@ export default function CreateArchive({ clear }) { autoFocus onChange={(e) => setTarget(e.target.value)} value={target} - placeholder="Name of zip archive..." + placeholder="Name of archive..." onPressEnter={doCompress} - suffix=".zip" + suffix={FORMAT} />
- + ); } diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index cbbd0fd8bc..df812641e0 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -1746,7 +1746,6 @@ export class ProjectActions extends Actions { }; set_file_action = (action?: FileAction): void => { - console.trace("set_file_action", action); const store = this.get_store(); if (store == null) { return; From 7eafe94b1a098f6b9014bfde616305af5811846c Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 5 Aug 2025 15:16:56 +0000 Subject: [PATCH 186/798] ouch: add some missing options --- .../backend/files/sandbox/exec.test.ts | 30 +++++++++++++++++++ src/packages/backend/files/sandbox/exec.ts | 28 ++++++++++++++++- src/packages/backend/files/sandbox/ouch.ts | 17 +++++++++++ 3 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 src/packages/backend/files/sandbox/exec.test.ts diff --git a/src/packages/backend/files/sandbox/exec.test.ts b/src/packages/backend/files/sandbox/exec.test.ts new file mode 100644 index 0000000000..8dcbc38332 --- /dev/null +++ b/src/packages/backend/files/sandbox/exec.test.ts @@ -0,0 +1,30 @@ +/* +Test the exec command. +*/ + +import exec from "./exec"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let tempDir; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); +}); +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); + +describe("exec works", () => { + it(`create file and run ls command`, async () => { + await writeFile(join(tempDir, "a.txt"), "hello"); + const { stderr, stdout, truncated, code } = await exec({ + cmd: "ls", + cwd: tempDir, + }); + expect(code).toBe(0); + expect(truncated).toBe(false); + expect(Buffer.from(stdout).toString()).toEqual("a.txt\n"); + expect(Buffer.from(stderr).toString()).toEqual(""); + }); +}); diff --git a/src/packages/backend/files/sandbox/exec.ts b/src/packages/backend/files/sandbox/exec.ts index be17c9b676..d84dd688a6 100644 --- a/src/packages/backend/files/sandbox/exec.ts +++ b/src/packages/backend/files/sandbox/exec.ts @@ -1,4 +1,4 @@ -import { spawn } from "node:child_process"; +import { execFile, spawn } from "node:child_process"; import { arch } from "node:os"; import { type ExecOutput } from "@cocalc/conat/files/fs"; export { type ExecOutput }; @@ -39,6 +39,10 @@ export interface Options { // options that are always included first for safety and need NOT match whitelist safety?: string[]; + + // if nodejs is running as root and give this username, then cmd runs as this + // user instead. + username?: string; } type ValidateFunction = (value: string) => void; @@ -55,6 +59,7 @@ export default async function exec({ timeout = DEFAULT_TIMEOUT, whitelist = {}, cwd, + username, }: Options): Promise { if (arch() == "darwin") { options = options.concat(darwin); @@ -62,6 +67,7 @@ export default async function exec({ options = options.concat(linux); } options = safety.concat(parseAndValidateOptions(options, whitelist)); + const userId = username ? await getUserIds(username) : undefined; return new Promise((resolve, reject) => { const stdoutChunks: Buffer[] = []; @@ -81,6 +87,7 @@ export default async function exec({ stdio: ["ignore", "pipe", "pipe"], env: {}, cwd, + ...userId, }); let timeoutHandle: NodeJS.Timeout | null = null; @@ -196,3 +203,22 @@ export const validate = { } }, }; + +async function getUserIds( + username: string, +): Promise<{ uid: number; gid: number }> { + return Promise.all([ + new Promise((resolve, reject) => { + execFile("id", ["-u", username], (err, stdout) => { + if (err) return reject(err); + resolve(parseInt(stdout.trim(), 10)); + }); + }), + new Promise((resolve, reject) => { + execFile("id", ["-g", username], (err, stdout) => { + if (err) return reject(err); + resolve(parseInt(stdout.trim(), 10)); + }); + }), + ]).then(([uid, gid]) => ({ uid, gid })); +} diff --git a/src/packages/backend/files/sandbox/ouch.ts b/src/packages/backend/files/sandbox/ouch.ts index 51b94a8270..69754330b4 100644 --- a/src/packages/backend/files/sandbox/ouch.ts +++ b/src/packages/backend/files/sandbox/ouch.ts @@ -37,6 +37,7 @@ export default async function ouch( const commands = ["compress", "c", "decompress", "d", "list", "l", "ls"]; const whitelist = { + // general options, "-H": true, "--hidden": true, g: true, @@ -49,4 +50,20 @@ const whitelist = { "--help": true, "-V": true, "--version": true, + + // compression-specific options + // do NOT enable '-S, --follow-symlinks' as that could escape the sandbox! + // It's off by default. + + "-l": validate.str, + "--level": validate.str, + "--fast": true, + "--slow": true, + + // decompress specific options + "-d": validate.str, + "--dir": validate.str, + r: true, + "--remove": true, + "--no-smart-unpack": true, } as const; From 2036af7a90d934388d1a9363ca9389004181ca08 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 5 Aug 2025 15:29:51 +0000 Subject: [PATCH 187/798] compress -- allow user to select format --- src/packages/conat/files/fs.ts | 14 ++++++++++ .../project/explorer/create-archive.tsx | 27 +++++++++++++++---- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index 67e397176f..e86ccbfcad 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -65,6 +65,20 @@ export interface OuchOptions { timeout?: number; } +export const OUCH_FORMATS = [ + "zip", + "7z", + "tar.gz", + "tar.xz", + "tar.bz", + "tar.bz2", + "tar.bz3", + "tar.lz4", + "tar.sz", + "tar.zst", + "tar.br", +]; + export interface Filesystem { appendFile: (path: string, data: string | Buffer, encoding?) => Promise; chmod: (path: string, mode: string | number) => Promise; diff --git a/src/packages/frontend/project/explorer/create-archive.tsx b/src/packages/frontend/project/explorer/create-archive.tsx index 2318a45e24..74bcf4550e 100644 --- a/src/packages/frontend/project/explorer/create-archive.tsx +++ b/src/packages/frontend/project/explorer/create-archive.tsx @@ -1,4 +1,4 @@ -import { Button, Card, Input, Space, Spin } from "antd"; +import { Button, Card, Input, Select, Space, Spin } from "antd"; import { useEffect, useRef, useState } from "react"; import { useIntl } from "react-intl"; import { default_filename } from "@cocalc/frontend/account"; @@ -9,10 +9,16 @@ import { useProjectContext } from "@cocalc/frontend/project/context"; import { path_split, plural } from "@cocalc/util/misc"; import CheckedFiles from "./checked-files"; import { join } from "path"; +import { OUCH_FORMATS } from "@cocalc/conat/files/fs"; -const FORMAT = ".tar.gz"; +const defaultFormat = OUCH_FORMATS.includes("tar.gz") + ? "tar.gz" + : OUCH_FORMATS[0]; export default function CreateArchive({ clear }) { + const [format, setFormat] = useState( + localStorage.defaultCompressionFormat ?? defaultFormat, + ); const intl = useIntl(); const inputRef = useRef(null); const { actions } = useProjectContext(); @@ -48,7 +54,7 @@ export default function CreateArchive({ clear }) { const { code, stderr } = await fs.ouch([ "compress", ...files, - join(path, target + FORMAT), + join(path, target + "." + format), ]); if (code) { throw Error(Buffer.from(stderr).toString()); @@ -69,7 +75,7 @@ export default function CreateArchive({ clear }) { return ( - Create a downloadable {FORMAT} archive from the following{" "} + Create a downloadable {format} archive from the following{" "} {checked_files?.size} selected {plural(checked_files?.size, "item")} > @@ -82,7 +88,7 @@ export default function CreateArchive({ clear }) { value={target} placeholder="Name of archive..." onPressEnter={doCompress} - suffix={FORMAT} + suffix={"." + format} />
+ { - return { value }; - })} - onChange={(format) => { - setFormat(format); - localStorage.defaultCompressionFormat = format; - }} - /> + ); } + +export async function createArchive({ path, files, target, format, actions }) { + const fs = actions.fs(); + const { code, stderr } = await fs.ouch([ + "compress", + ...files, + join(path, target + "." + format), + ]); + if (code) { + throw Error(Buffer.from(stderr).toString()); + } +} + +export function SelectFormat({ format, setFormat }) { + useEffect(() => { + if (!OUCH_FORMATS.includes(format)) { + if (OUCH_FORMATS.includes(localStorage.defaultCompressionFormat)) { + setFormat(localStorage.defaultCompressionFormat); + } else { + setFormat(defaultFormat); + } + } + }, [format]); + + return ( + { + setTitle(e.target.value); + titleRef.current = e.target.value; + }} + /> +
+ ); +} diff --git a/src/packages/frontend/project/explorer/misc-side-buttons.tsx b/src/packages/frontend/project/explorer/misc-side-buttons.tsx index 5d9fd3d090..c8c8d36530 100644 --- a/src/packages/frontend/project/explorer/misc-side-buttons.tsx +++ b/src/packages/frontend/project/explorer/misc-side-buttons.tsx @@ -21,6 +21,7 @@ import { type JSX, type MouseEvent } from "react"; const SHOW_SERVER_LAUNCHERS = false; import TourButton from "./tour/button"; +import ForkProject from "./fork"; const OPEN_MSG = defineMessage({ id: "project.explorer.misc-side-buttons.open_dir.tooltip", @@ -185,6 +186,7 @@ export function MiscSideButtons() { {render_hidden_toggle()} {render_backup()} +
diff --git a/src/packages/frontend/project/page/flyouts/files-header.tsx b/src/packages/frontend/project/page/flyouts/files-header.tsx index 19874e28dd..02f384b0e7 100644 --- a/src/packages/frontend/project/page/flyouts/files-header.tsx +++ b/src/packages/frontend/project/page/flyouts/files-header.tsx @@ -7,7 +7,6 @@ import { Alert, Button, Input, InputRef, Radio, Space, Tooltip } from "antd"; import immutable from "immutable"; import { FormattedMessage, useIntl } from "react-intl"; import { VirtuosoHandle } from "react-virtuoso"; - import { Button as BootstrapButton } from "@cocalc/frontend/antd-bootstrap"; import { CSS, @@ -36,6 +35,7 @@ import { ActiveFileSort } from "./files"; import { FilesSelectedControls } from "./files-controls"; import { FilesSelectButtons } from "./files-select-extra"; import { FlyoutClearFilter, FlyoutFilterWarning } from "./filter-warning"; +import ForkProject from "@cocalc/frontend/project/explorer/fork"; function searchToFilename(search: string): string { if (search.endsWith(" ")) { @@ -479,6 +479,7 @@ export function FilesHeader(props: Readonly): React.JSX.Element { } icon={} /> + ) : undefined}
diff --git a/src/packages/frontend/projects/actions.ts b/src/packages/frontend/projects/actions.ts index 5fb8e793e0..ba281d4465 100644 --- a/src/packages/frontend/projects/actions.ts +++ b/src/packages/frontend/projects/actions.ts @@ -417,7 +417,7 @@ export class ProjectsActions extends Actions { } // Open the given project - public async open_project(opts: { + open_project = async (opts: { project_id: string; // id of the project to open target?: string; // The file path to open fragmentId?: FragmentId; // if given, an uri fragment in the editor that is opened. @@ -425,7 +425,7 @@ export class ProjectsActions extends Actions { ignore_kiosk?: boolean; // Ignore ?fullscreen=kiosk change_history?: boolean; // (default: true) Whether or not to alter browser history restore_session?: boolean; // (default: true) Opens up previously closed editor tabs - }) { + }) => { opts = defaults(opts, { project_id: undefined, target: undefined, @@ -486,7 +486,7 @@ export class ProjectsActions extends Actions { } // initialize project project_actions.init(); - } + }; // tab at old_index taken out and then inserted into the resulting array's new index public move_project_tab({ diff --git a/src/packages/frontend/projects/project-row.tsx b/src/packages/frontend/projects/project-row.tsx index 5631972a8d..08c03de776 100644 --- a/src/packages/frontend/projects/project-row.tsx +++ b/src/packages/frontend/projects/project-row.tsx @@ -34,7 +34,7 @@ import track from "@cocalc/frontend/user-tracking"; import { DEFAULT_COMPUTE_IMAGE } from "@cocalc/util/db-schema"; import { KUCALC_COCALC_COM } from "@cocalc/util/db-schema/site-defaults"; import { COLORS } from "@cocalc/util/theme"; -import { Avatar } from "antd"; +import { Avatar, Button, Tooltip } from "antd"; import { CSSProperties, useEffect } from "react"; import { ProjectUsers } from "./project-users"; @@ -215,7 +215,7 @@ export const ProjectRow: React.FC = ({ project_id, index }: Props) => { = ({ project_id, index }: Props) => { /> )} + + {!is_anonymous && ( + + + + )} + ); From e0ac383ba0a8e5a0a427329a8a76c31148cd9031 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 11 Aug 2025 04:15:20 +0000 Subject: [PATCH 259/798] file-server -- start adding rustic support --- .../backend/conat/files/local-path.ts | 2 +- src/packages/backend/sandbox/exec.ts | 2 +- src/packages/backend/sandbox/index.ts | 35 +++++-- src/packages/backend/sandbox/rustic.ts | 40 ++++++-- src/packages/conat/hub/api/projects.ts | 8 ++ src/packages/file-server/btrfs/filesystem.ts | 12 +++ .../file-server/btrfs/subvolume-rustic.ts | 92 +++++++++++++++++++ src/packages/file-server/btrfs/subvolume.ts | 8 +- src/packages/file-server/btrfs/subvolumes.ts | 1 - .../file-server/btrfs/test/subvolume.test.ts | 22 ++++- src/packages/frontend/compute/clone.tsx | 2 +- .../frontend/project/explorer/fork.tsx | 7 +- src/packages/server/conat/api/projects.ts | 14 +++ .../server/conat/file-server/index.ts | 4 +- 14 files changed, 223 insertions(+), 26 deletions(-) create mode 100644 src/packages/file-server/btrfs/subvolume-rustic.ts diff --git a/src/packages/backend/conat/files/local-path.ts b/src/packages/backend/conat/files/local-path.ts index 6ad630ad51..17857d27fb 100644 --- a/src/packages/backend/conat/files/local-path.ts +++ b/src/packages/backend/conat/files/local-path.ts @@ -36,7 +36,7 @@ export async function localPathFileserver({ const project_id = getProjectId(subject); const fsclient = createFileClient({ client }); const { path } = await fsclient.mount({ project_id }); - return new SandboxedFilesystem(path, { unsafeMode, project_id }); + return new SandboxedFilesystem(path, { unsafeMode, host: project_id }); } }, }); diff --git a/src/packages/backend/sandbox/exec.ts b/src/packages/backend/sandbox/exec.ts index fd558c92ee..cfbb06421c 100644 --- a/src/packages/backend/sandbox/exec.ts +++ b/src/packages/backend/sandbox/exec.ts @@ -93,7 +93,7 @@ export default async function exec({ cmd = nsjailPath; } - // console.log(`${cmd} ${args.join(" ")}`); + // console.log(`${cmd} ${args.join(" ")}`, { cwd }); const child = spawn(cmd, args, { stdio: ["ignore", "pipe", "pipe"], env: {}, diff --git a/src/packages/backend/sandbox/index.ts b/src/packages/backend/sandbox/index.ts index 8400e53c54..6a87e40514 100644 --- a/src/packages/backend/sandbox/index.ts +++ b/src/packages/backend/sandbox/index.ts @@ -92,7 +92,8 @@ interface Options { unsafeMode?: boolean; // readonly -- only allow operations that don't change files readonly?: boolean; - project_id?: string; + host?: string; + rusticRepo?: string; } // If you add any methods below that are NOT for the public api @@ -106,22 +107,28 @@ const INTERNAL_METHODS = new Set([ "readonly", "assertWritable", "rusticRepo", - "project_id", + "host", ]); export class SandboxedFilesystem { public readonly unsafeMode: boolean; public readonly readonly: boolean; - private readonly rusticRepo: string = rusticRepo; - private project_id?: string; + public rusticRepo: string; + private host?: string; constructor( // path should be the path to a FOLDER on the filesystem (not a file) public readonly path: string, - { unsafeMode = false, readonly = false, project_id }: Options = {}, + { + unsafeMode = false, + readonly = false, + host = "global", + rusticRepo: repo, + }: Options = {}, ) { this.unsafeMode = !!unsafeMode; this.readonly = !!readonly; - this.project_id = project_id; + this.host = host; + this.rusticRepo = repo ?? rusticRepo; for (const f in this) { if (INTERNAL_METHODS.has(f)) { continue; @@ -285,13 +292,21 @@ export class SandboxedFilesystem { }; // backups - rustic = async (args: string[]): Promise => { + rustic = async ( + args: string[], + { + timeout = 120_000, + maxSize = 10_000, + cwd, + }: { timeout?: number; maxSize?: number; cwd?: string } = {}, + ): Promise => { return await rustic(args, { repo: this.rusticRepo, safeAbsPath: this.safeAbsPath, - timeout: 120_000, - maxSize: 10_000, - host: this.project_id ?? "global", + timeout, + maxSize, + host: this.host, + cwd, }); }; diff --git a/src/packages/backend/sandbox/rustic.ts b/src/packages/backend/sandbox/rustic.ts index 03d9240cc7..28bb7e4a96 100644 --- a/src/packages/backend/sandbox/rustic.ts +++ b/src/packages/backend/sandbox/rustic.ts @@ -67,25 +67,32 @@ export interface RusticOptions { repo?: string; timeout?: number; maxSize?: number; - safeAbsPath: (path: string) => Promise; - host: string; + safeAbsPath?: (path: string) => Promise; + host?: string; + cwd?: string; } export default async function rustic( args: string[], options: RusticOptions, ): Promise { - const { timeout, maxSize, repo = rusticRepo, safeAbsPath, host } = options; + const { + timeout, + maxSize, + repo = rusticRepo, + safeAbsPath, + host = "host", + } = options; await ensureInitialized(repo); - const base = await safeAbsPath(""); + const cwd = await safeAbsPath?.(options.cwd ?? ""); const common = ["--password", "", "-r", repo]; const run = async (sanitizedArgs: string[]) => { return await exec({ cmd: rusticPath, - cwd: base, + cwd, safety: [...common, args[0], ...sanitizedArgs], maxSize, timeout, @@ -93,17 +100,33 @@ export default async function rustic( }; switch (args[0]) { + case "init": { + if (safeAbsPath != null) { + throw Error("init not allowed"); + } + return await run([]); + } case "backup": { + if (safeAbsPath == null || cwd == null) { + throw Error("safeAbsPath must be specified when making a backup"); + } if (args.length == 1) { throw Error("missing backup source"); } - const source = (await safeAbsPath(args.slice(-1)[0])).slice(base.length); + const source = (await safeAbsPath(args.slice(-1)[0])).slice(cwd.length); const options = parseAndValidateOptions( args.slice(1, -1), whitelist.backup, ); - return await run([...options, "--no-scan", "--host", host, "--", source]); + return await run([ + ...options, + "--no-scan", + "--host", + host, + "--", + source ? source : ".", + ]); } case "snapshots": { const options = parseAndValidateOptions( @@ -125,6 +148,9 @@ export default async function rustic( if (args.length <= 2) { throw Error("missing "); } + if (safeAbsPath == null) { + throw Error("safeAbsPath must be specified when restoring"); + } const snapshot = args.slice(-2)[0]; // await assertValidSnapshot({ snapshot, host, repo }); const destination = await safeAbsPath(args.slice(-1)[0]); // diff --git a/src/packages/conat/hub/api/projects.ts b/src/packages/conat/hub/api/projects.ts index 95d8efd1b5..fa79963587 100644 --- a/src/packages/conat/hub/api/projects.ts +++ b/src/packages/conat/hub/api/projects.ts @@ -10,6 +10,8 @@ export const projects = { inviteCollaborator: authFirstRequireAccount, inviteCollaboratorWithoutAccount: authFirstRequireAccount, setQuotas: authFirstRequireAccount, + + getDiskQuota: authFirstRequireAccount, }; export type AddCollaborator = @@ -89,6 +91,7 @@ export interface Projects { }; }) => Promise; + // for admins only! setQuotas: (opts: { account_id?: string; project_id: string; @@ -102,4 +105,9 @@ export interface Projects { member_host?: number; always_running?: number; }) => Promise; + + getDiskQuota: (opts: { + account_id?: string; + project_id: string; + }) => Promise<{ used: number; size: number }>; } diff --git a/src/packages/file-server/btrfs/filesystem.ts b/src/packages/file-server/btrfs/filesystem.ts index 248e086630..e29de633cb 100644 --- a/src/packages/file-server/btrfs/filesystem.ts +++ b/src/packages/file-server/btrfs/filesystem.ts @@ -18,6 +18,7 @@ import { Subvolumes } from "./subvolumes"; import { mkdir } from "fs/promises"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import { executeCode } from "@cocalc/backend/execute-code"; +import rustic from "@cocalc/backend/sandbox/rustic"; // default size of btrfs filesystem if creating an image file. const DEFAULT_FILESYSTEM_SIZE = "10G"; @@ -48,6 +49,7 @@ export interface Options { export class Filesystem { public readonly opts: Options; public readonly bup: string; + public readonly rustic: string; public readonly subvolumes: Subvolumes; constructor(opts: Options) { @@ -58,6 +60,7 @@ export class Filesystem { }; this.opts = opts; this.bup = join(this.opts.mount, "bup"); + this.rustic = join(this.opts.mount, "rustic"); this.subvolumes = new Subvolumes(this); } @@ -69,6 +72,7 @@ export class Filesystem { args: ["quota", "enable", "--simple", this.opts.mount], }); await this.initBup(); + await this.initRustic(); await this.sync(); }; @@ -186,6 +190,14 @@ export class Filesystem { env: { BUP_DIR: this.bup }, }); }; + + private initRustic = async () => { + if (await exists(this.rustic)) { + return; + } + await mkdir(this.rustic); + await rustic(["init"], { repo: this.rustic }); + }; } function isImageFile(name: string) { diff --git a/src/packages/file-server/btrfs/subvolume-rustic.ts b/src/packages/file-server/btrfs/subvolume-rustic.ts new file mode 100644 index 0000000000..0a0e0a79ca --- /dev/null +++ b/src/packages/file-server/btrfs/subvolume-rustic.ts @@ -0,0 +1,92 @@ +/* +Rustic Architecture: + +The minimal option is a single global repo stored in the btrfs filesystem. +Obviously, admins should rsync this regularly to a separate location as a +genuine backup strategy. It's better to configure repo on separate +storage. Rustic has a very wide range of options. + +Instead of using btrfs send/recv for backups, we use Rustic because: + - much easier to check backups are valid + - globally compressed and dedup'd! btrfs send/recv is NOT globally dedupd + - decoupled from any btrfs issues + - rustic has full support for using cloud buckets as hot/cold storage + - not tied to any specific filesystem at all + - easier to offsite via incremental rsync + - much more space efficient with *global* dedup and compression + - rustic "is" restic, which is very mature and proven + - rustic is VERY fast, being parallel and in rust. +*/ + +import { type Subvolume } from "./subvolume"; +import getLogger from "@cocalc/backend/logger"; + +const RUSTIC_SNAPSHOT = "temp-rustic-snapshot"; + +const logger = getLogger("file-server:btrfs:subvolume-rustic"); + +interface Snapshot { + id: string; + time: Date; +} + +export class SubvolumeRustic { + constructor(private subvolume: Subvolume) {} + + // create a new rustic backup + backup = async ({ timeout = 30 * 60 * 1000 }: { timeout?: number } = {}) => { + if (await this.subvolume.snapshots.exists(RUSTIC_SNAPSHOT)) { + logger.debug(`backup: deleting existing ${RUSTIC_SNAPSHOT}`); + await this.subvolume.snapshots.delete(RUSTIC_SNAPSHOT); + } + const target = this.subvolume.snapshots.path(RUSTIC_SNAPSHOT); + try { + logger.debug( + `backup: creating ${RUSTIC_SNAPSHOT} to get a consistent backup`, + ); + await this.subvolume.snapshots.create(RUSTIC_SNAPSHOT); + logger.debug(`backup: backing up ${RUSTIC_SNAPSHOT} using rustic`); + const { stderr, code } = await this.subvolume.fs.rustic( + ["backup", "-x", "."], + { + timeout, + cwd: target, + }, + ); + if (code) { + throw Error(stderr.toString()); + } + } finally { + logger.debug(`backup: deleting temporary ${RUSTIC_SNAPSHOT}`); + await this.subvolume.snapshots.delete(RUSTIC_SNAPSHOT); + } + }; + + snapshots = async (): Promise => { + const { stdout, stderr, code } = await this.subvolume.fs.rustic([ + "snapshots", + "--json", + ]); + if (code) { + throw Error(stderr.toString()); + } else { + const x = JSON.parse(stdout.toString()); + return x[0][1].map(({ time, id }) => { + return { time: new Date(time), id }; + }); + } + }; + + ls = async (id: string) => { + const { stdout, stderr, code } = await this.subvolume.fs.rustic([ + "ls", + "--json", + id, + ]); + if (code) { + throw Error(stderr.toString()); + } else { + return JSON.parse(stdout.toString()); + } + }; +} diff --git a/src/packages/file-server/btrfs/subvolume.ts b/src/packages/file-server/btrfs/subvolume.ts index a02be9d7cd..7eedfc0806 100644 --- a/src/packages/file-server/btrfs/subvolume.ts +++ b/src/packages/file-server/btrfs/subvolume.ts @@ -7,6 +7,7 @@ import refCache from "@cocalc/util/refcache"; import { sudo } from "./util"; import { join } from "path"; import { SubvolumeBup } from "./subvolume-bup"; +import { SubvolumeRustic } from "./subvolume-rustic"; import { SubvolumeSnapshots } from "./subvolume-snapshots"; import { SubvolumeQuota } from "./subvolume-quota"; import { SandboxedFilesystem } from "@cocalc/backend/sandbox"; @@ -27,6 +28,7 @@ export class Subvolume { public readonly path: string; public readonly fs: SandboxedFilesystem; public readonly bup: SubvolumeBup; + public readonly rustic: SubvolumeRustic; public readonly snapshots: SubvolumeSnapshots; public readonly quota: SubvolumeQuota; @@ -34,8 +36,12 @@ export class Subvolume { this.filesystem = filesystem; this.name = name; this.path = join(filesystem.opts.mount, name); - this.fs = new SandboxedFilesystem(this.path); + this.fs = new SandboxedFilesystem(this.path, { + rusticRepo: filesystem.rustic, + host: this.name, + }); this.bup = new SubvolumeBup(this); + this.rustic = new SubvolumeRustic(this); this.snapshots = new SubvolumeSnapshots(this); this.quota = new SubvolumeQuota(this); } diff --git a/src/packages/file-server/btrfs/subvolumes.ts b/src/packages/file-server/btrfs/subvolumes.ts index 63a6ee992f..e4f10191b9 100644 --- a/src/packages/file-server/btrfs/subvolumes.ts +++ b/src/packages/file-server/btrfs/subvolumes.ts @@ -26,7 +26,6 @@ export class Subvolumes { return await subvolume({ filesystem: this.filesystem, name }); }; - // create a subvolume by cloning an existing one. clone = async (source: string, dest: string) => { logger.debug("clone ", { source, dest }); if (RESERVED.has(dest)) { diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index fe9286935e..21a46c6f07 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -227,7 +227,27 @@ describe("test snapshots", () => { }); }); -describe("test bup backups", () => { +describe("test rustic backups", () => { + let vol: Subvolume; + it("creates a volume", async () => { + vol = await fs.subvolumes.get("rustic-test"); + await vol.fs.writeFile("a.txt", "hello"); + }); + + it("create a rustic backup", async () => { + await vol.rustic.backup(); + }); + + it("confirm a.txt is in our backup", async () => { + const v = await vol.rustic.snapshots(); + expect(v.length == 1); + expect(Math.abs(Date.now() - v[0].time.valueOf())).toBeLessThan(10000); + const w = await vol.rustic.ls(v[0].id); + expect(w).toEqual([".snapshots", "a.txt"]); + }); +}); + +describe.skip("test bup backups", () => { let vol: Subvolume; it("creates a volume", async () => { vol = await fs.subvolumes.get("bup-test"); diff --git a/src/packages/frontend/compute/clone.tsx b/src/packages/frontend/compute/clone.tsx index 6c7c281526..c2c5dcc696 100644 --- a/src/packages/frontend/compute/clone.tsx +++ b/src/packages/frontend/compute/clone.tsx @@ -78,7 +78,7 @@ export async function cloneConfiguration({ const server = servers[0] as ComputeServerUserInfo; if (!noChange) { let n = 1; - let title = `Clone of ${server.title}`; + let title = `Fork of ${server.title}`; const titles = new Set(servers.map((x) => x.title)); if (titles.has(title)) { while (titles.has(title + ` (${n})`)) { diff --git a/src/packages/frontend/project/explorer/fork.tsx b/src/packages/frontend/project/explorer/fork.tsx index f3af68fb42..240f5cd047 100644 --- a/src/packages/frontend/project/explorer/fork.tsx +++ b/src/packages/frontend/project/explorer/fork.tsx @@ -1,4 +1,4 @@ -import { useRef, useState } from "react"; +import { useEffect, useRef, useState } from "react"; import { Button, Input, Popconfirm, Tooltip } from "antd"; import { Icon } from "@cocalc/frontend/components/icon"; import { ProjectTitle } from "@cocalc/frontend/projects/project-title"; @@ -59,11 +59,14 @@ export default function ForkProject({ project_id, flyout }: Props) { function Description({ project_id, titleRef }) { const [title, setTitle] = useState( - `Clone of ${ + `Fork of ${ redux.getStore("projects").getIn(["project_map", project_id, "title"]) ?? "project" }`, ); + useEffect(() => { + titleRef.current = title; + }, []); return (
A fork is an exact copy of a project. Forking a project allows you to diff --git a/src/packages/server/conat/api/projects.ts b/src/packages/server/conat/api/projects.ts index 3e79bc7420..86070a4106 100644 --- a/src/packages/server/conat/api/projects.ts +++ b/src/packages/server/conat/api/projects.ts @@ -62,3 +62,17 @@ export async function setQuotas(opts: { // @ts-ignore await project?.setAllQuotas(); } + +export async function getDiskQuota({ + account_id, + project_id, +}: { + account_id: string; + project_id: string; +}): Promise<{ used: number; size: number }> { + if (!(await isCollaborator({ account_id, project_id }))) { + throw Error("user must be a collaborator on project to get quota"); + } + const client = filesystemClient(); + return await client.getQuota({ project_id }); +} diff --git a/src/packages/server/conat/file-server/index.ts b/src/packages/server/conat/file-server/index.ts index 995dc6792a..ce5ace5ee3 100644 --- a/src/packages/server/conat/file-server/index.ts +++ b/src/packages/server/conat/file-server/index.ts @@ -173,6 +173,8 @@ export function close() { server = null; } +let cachedClient: null | Fileserver = null; export function client(): Fileserver { - return createFileClient({ client: conat() }); + cachedClient ??= createFileClient({ client: conat() }); + return cachedClient!; } From d8c1d5f25f621c65cbcb54693ab5f1a6cc508011 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 11 Aug 2025 20:00:06 +0000 Subject: [PATCH 260/798] implement more rustic backup and testing --- src/packages/backend/sandbox/exec.ts | 13 ++++ .../file-server/btrfs/subvolume-rustic.ts | 65 +++++++++++------ .../btrfs/test/rustic-stress.test.ts | 45 ++++++++++++ .../file-server/btrfs/test/rustic.test.ts | 73 +++++++++++++++++++ .../file-server/btrfs/test/subvolume.test.ts | 20 ----- 5 files changed, 173 insertions(+), 43 deletions(-) create mode 100644 src/packages/file-server/btrfs/test/rustic-stress.test.ts create mode 100644 src/packages/file-server/btrfs/test/rustic.test.ts diff --git a/src/packages/backend/sandbox/exec.ts b/src/packages/backend/sandbox/exec.ts index cfbb06421c..a966474db9 100644 --- a/src/packages/backend/sandbox/exec.ts +++ b/src/packages/backend/sandbox/exec.ts @@ -233,3 +233,16 @@ async function getUserIds( }), ]).then(([uid, gid]) => ({ uid, gid })); } + +// take the output of exec and convert stdout, stderr to strings. If code is nonzero, +// instead throw an error with message stderr. +export function parseOutput({ stdout, stderr, code, truncated }: ExecOutput) { + if (code) { + throw new Error(Buffer.from(stderr).toString()); + } + return { + stdout: Buffer.from(stdout).toString(), + stderr: Buffer.from(stderr).toString(), + truncated, + }; +} diff --git a/src/packages/file-server/btrfs/subvolume-rustic.ts b/src/packages/file-server/btrfs/subvolume-rustic.ts index 0a0e0a79ca..6200a42f0c 100644 --- a/src/packages/file-server/btrfs/subvolume-rustic.ts +++ b/src/packages/file-server/btrfs/subvolume-rustic.ts @@ -20,6 +20,7 @@ Instead of using btrfs send/recv for backups, we use Rustic because: import { type Subvolume } from "./subvolume"; import getLogger from "@cocalc/backend/logger"; +import { parseOutput } from "@cocalc/backend/sandbox/exec"; const RUSTIC_SNAPSHOT = "temp-rustic-snapshot"; @@ -62,31 +63,49 @@ export class SubvolumeRustic { } }; + restore = async ({ + id, + path = "", + dest, + timeout = 30 * 60 * 1000, + }: { + id: string; + path?: string; + dest?: string; + timeout?: number; + }) => { + dest ??= path; + const { stdout } = parseOutput( + await this.subvolume.fs.rustic( + ["restore", `${id}${path != null ? ":" + path : ""}`, dest], + { timeout }, + ), + ); + return stdout; + }; + snapshots = async (): Promise => { - const { stdout, stderr, code } = await this.subvolume.fs.rustic([ - "snapshots", - "--json", - ]); - if (code) { - throw Error(stderr.toString()); - } else { - const x = JSON.parse(stdout.toString()); - return x[0][1].map(({ time, id }) => { - return { time: new Date(time), id }; - }); - } + const { stdout } = parseOutput( + await this.subvolume.fs.rustic(["snapshots", "--json"]), + ); + const x = JSON.parse(stdout); + return x[0][1].map(({ time, id }) => { + return { time: new Date(time), id }; + }); }; - ls = async (id: string) => { - const { stdout, stderr, code } = await this.subvolume.fs.rustic([ - "ls", - "--json", - id, - ]); - if (code) { - throw Error(stderr.toString()); - } else { - return JSON.parse(stdout.toString()); - } + ls = async ({ id }: { id: string }) => { + const { stdout } = parseOutput( + await this.subvolume.fs.rustic(["ls", "--json", id]), + ); + return JSON.parse(stdout); + }; + + // (this doesn't actually clean up disk space -- purge must be done separately) + forget = async ({ id }: { id: string }) => { + const { stdout } = parseOutput( + await this.subvolume.fs.rustic(["forget", id]), + ); + return stdout; }; } diff --git a/src/packages/file-server/btrfs/test/rustic-stress.test.ts b/src/packages/file-server/btrfs/test/rustic-stress.test.ts new file mode 100644 index 0000000000..2eb0514df7 --- /dev/null +++ b/src/packages/file-server/btrfs/test/rustic-stress.test.ts @@ -0,0 +1,45 @@ +import { before, after, fs } from "./setup"; +import { type Subvolume } from "../subvolume"; + +beforeAll(before); + +const count = 10; +describe(`make backups of ${count} different volumes at the same time`, () => { + const vols: Subvolume[] = []; + it(`creates ${count} volumes`, async () => { + for (let i = 0; i < count; i++) { + const vol = await fs.subvolumes.get(`rustic-multi-${i}`); + await vol.fs.writeFile(`a-${i}.txt`, `hello-${i}`); + vols.push(vol); + } + }); + + it(`create ${count} rustic backup in parallel`, async () => { + await Promise.all(vols.map((vol) => vol.rustic.backup())); + }); + + it("delete file from each volume, then restore them all in parallel and confirm restore worked", async () => { + const snapshots = await Promise.all( + vols.map((vol) => vol.rustic.snapshots()), + ); + const ids = snapshots.map((x) => x[0].id); + for (let i = 0; i < count; i++) { + await vols[i].fs.unlink(`a-${i}.txt`); + } + + const v: any[] = []; + for (let i = 0; i < count; i++) { + v.push(vols[i].rustic.restore({ id: ids[i] })); + } + await Promise.all(v); + + for (let i = 0; i < count; i++) { + const vol = vols[i]; + expect((await vol.fs.readFile(`a-${i}.txt`)).toString("utf8")).toEqual( + `hello-${i}`, + ); + } + }); +}); + +afterAll(after); diff --git a/src/packages/file-server/btrfs/test/rustic.test.ts b/src/packages/file-server/btrfs/test/rustic.test.ts new file mode 100644 index 0000000000..feb36ce5a5 --- /dev/null +++ b/src/packages/file-server/btrfs/test/rustic.test.ts @@ -0,0 +1,73 @@ +import { before, after, fs } from "./setup"; +import { type Subvolume } from "../subvolume"; + +beforeAll(before); + +describe("test rustic backups", () => { + let vol: Subvolume; + it("creates a volume", async () => { + vol = await fs.subvolumes.get("rustic-test"); + await vol.fs.writeFile("a.txt", "hello"); + }); + + it("create a rustic backup", async () => { + await vol.rustic.backup(); + }); + + it("confirm a.txt is in our backup", async () => { + const v = await vol.rustic.snapshots(); + expect(v.length == 1); + expect(Math.abs(Date.now() - v[0].time.valueOf())).toBeLessThan(10000); + const { id } = v[0]; + const w = await vol.rustic.ls({ id }); + expect(w).toEqual([".snapshots", "a.txt"]); + }); + + it("delete a.txt, then restore it from the backup", async () => { + await vol.fs.unlink("a.txt"); + const { id } = (await vol.rustic.snapshots())[0]; + await vol.rustic.restore({ id }); + expect((await vol.fs.readFile("a.txt")).toString("utf8")).toEqual("hello"); + }); + + it("create a directory, make second backup, delete directory, then restore it from backup, and also restore just one file", async () => { + await vol.fs.mkdir("my-dir"); + await vol.fs.writeFile("my-dir/file.txt", "hello"); + await vol.fs.writeFile("my-dir/file2.txt", "hello2"); + await vol.rustic.backup(); + const v = await vol.rustic.snapshots(); + expect(v.length == 2); + const { id } = v[1]; + const w = await vol.rustic.ls({ id }); + expect(w).toEqual([ + ".snapshots", + "a.txt", + "my-dir", + "my-dir/file.txt", + "my-dir/file2.txt", + ]); + await vol.fs.rm("my-dir", { recursive: true }); + + // rustic all, including the path we just deleted + await vol.rustic.restore({ id }); + expect((await vol.fs.readFile("my-dir/file.txt")).toString("utf8")).toEqual( + "hello", + ); + + // restore just one specific file overwriting current version + await vol.fs.unlink("my-dir/file2.txt"); + await vol.fs.writeFile("my-dir/file.txt", "changed"); + await vol.rustic.restore({ id, path: "my-dir/file2.txt" }); + expect( + (await vol.fs.readFile("my-dir/file2.txt")).toString("utf8"), + ).toEqual("hello2"); + + // forget the second snapshot + const z = await vol.rustic.forget({ id }); + const v2 = await vol.rustic.snapshots(); + expect(v2.length).toBe(1); + expect(v2[0].id).not.toEqual(id); + }); +}); + +afterAll(after); diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index 21a46c6f07..fcdb878f7d 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -227,26 +227,6 @@ describe("test snapshots", () => { }); }); -describe("test rustic backups", () => { - let vol: Subvolume; - it("creates a volume", async () => { - vol = await fs.subvolumes.get("rustic-test"); - await vol.fs.writeFile("a.txt", "hello"); - }); - - it("create a rustic backup", async () => { - await vol.rustic.backup(); - }); - - it("confirm a.txt is in our backup", async () => { - const v = await vol.rustic.snapshots(); - expect(v.length == 1); - expect(Math.abs(Date.now() - v[0].time.valueOf())).toBeLessThan(10000); - const w = await vol.rustic.ls(v[0].id); - expect(w).toEqual([".snapshots", "a.txt"]); - }); -}); - describe.skip("test bup backups", () => { let vol: Subvolume; it("creates a volume", async () => { From aca121e13af1aa302195af7734a0635ae5083cce Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 11 Aug 2025 21:31:11 +0000 Subject: [PATCH 261/798] delete bup from file-server codebase -- we're going with rustic --- src/packages/file-server/btrfs/filesystem.ts | 70 ++----- .../file-server/btrfs/subvolume-bup.ts | 195 ------------------ .../file-server/btrfs/subvolume-rustic.ts | 2 + src/packages/file-server/btrfs/subvolume.ts | 10 +- src/packages/file-server/btrfs/subvolumes.ts | 3 +- .../file-server/btrfs/test/filesystem.test.ts | 3 +- .../file-server/btrfs/test/rustic.test.ts | 2 +- src/packages/file-server/btrfs/test/setup.ts | 2 +- .../file-server/btrfs/test/subvolume.test.ts | 69 ------- .../server/conat/file-server/index.ts | 3 +- 10 files changed, 26 insertions(+), 333 deletions(-) delete mode 100644 src/packages/file-server/btrfs/subvolume-bup.ts diff --git a/src/packages/file-server/btrfs/filesystem.ts b/src/packages/file-server/btrfs/filesystem.ts index e29de633cb..d675c863b7 100644 --- a/src/packages/file-server/btrfs/filesystem.ts +++ b/src/packages/file-server/btrfs/filesystem.ts @@ -17,50 +17,30 @@ import { join } from "path"; import { Subvolumes } from "./subvolumes"; import { mkdir } from "fs/promises"; import { exists } from "@cocalc/backend/misc/async-utils-node"; -import { executeCode } from "@cocalc/backend/execute-code"; import rustic from "@cocalc/backend/sandbox/rustic"; - -// default size of btrfs filesystem if creating an image file. -const DEFAULT_FILESYSTEM_SIZE = "10G"; - -// default for newly created subvolumes -export const DEFAULT_SUBVOLUME_SIZE = "1G"; - -const MOUNT_ERROR = "wrong fs type, bad option, bad superblock"; +import { RUSTIC } from "./subvolume-rustic"; export interface Options { // the underlying block device. // If this is a file (or filename) ending in .img, then it's a sparse file mounted as a loopback device. // If this starts with "/dev" then it is a raw block device. device: string; - // formatIfNeeded -- DANGEROUS! if true, format the device or image, - // if it doesn't mount with an error containing "wrong fs type, - // bad option, bad superblock". Never use this in production. Useful - // for testing and dev. - formatIfNeeded?: boolean; - // where the btrfs filesystem is mounted + // where to mount the btrfs filesystem mount: string; - - // default size of newly created subvolumes - defaultSize?: string | number; - defaultFilesystemSize?: string | number; + // size -- if true and 'device' is a path to a .img file that DOES NOT EXIST, create device + // as a sparse image file of the given size. If img already exists, it will not be touched + // in any way, and it is up to you to mkfs.btrfs it, etc. + size?: string | number; } export class Filesystem { public readonly opts: Options; - public readonly bup: string; public readonly rustic: string; public readonly subvolumes: Subvolumes; constructor(opts: Options) { - opts = { - defaultSize: DEFAULT_SUBVOLUME_SIZE, - defaultFilesystemSize: DEFAULT_FILESYSTEM_SIZE, - ...opts, - }; this.opts = opts; - this.bup = join(this.opts.mount, "bup"); - this.rustic = join(this.opts.mount, "rustic"); + this.rustic = join(this.opts.mount, RUSTIC); this.subvolumes = new Subvolumes(this); } @@ -71,7 +51,6 @@ export class Filesystem { await btrfs({ args: ["quota", "enable", "--simple", this.opts.mount], }); - await this.initBup(); await this.initRustic(); await this.sync(); }; @@ -96,10 +75,17 @@ export class Filesystem { return; } if (!(await exists(this.opts.device))) { + if (!this.opts.size) { + throw Error( + "you must specify the size of the btrfs sparse image file, or explicitly create and format it", + ); + } + // we create and format the sparse image await sudo({ command: "truncate", - args: ["-s", `${this.opts.defaultFilesystemSize}`, this.opts.device], + args: ["-s", `${this.opts.size}`, this.opts.device], }); + await sudo({ command: "mkfs.btrfs", args: [this.opts.device] }); } }; @@ -124,25 +110,10 @@ export class Filesystem { } catch {} const { stderr, exit_code } = await this._mountFilesystem(); if (exit_code) { - if (stderr.includes(MOUNT_ERROR)) { - if (this.opts.formatIfNeeded) { - await this.formatDevice(); - const { stderr, exit_code } = await this._mountFilesystem(); - if (exit_code) { - throw Error(stderr); - } else { - return; - } - } - } throw Error(stderr); } }; - private formatDevice = async () => { - await sudo({ command: "mkfs.btrfs", args: [this.opts.device] }); - }; - private _mountFilesystem = async () => { const args: string[] = isImageFile(this.opts.device) ? ["-o", "loop"] : []; args.push( @@ -180,17 +151,6 @@ export class Filesystem { return { stderr, exit_code }; }; - private initBup = async () => { - if (!(await exists(this.bup))) { - await mkdir(this.bup); - } - await executeCode({ - command: "bup", - args: ["init"], - env: { BUP_DIR: this.bup }, - }); - }; - private initRustic = async () => { if (await exists(this.rustic)) { return; diff --git a/src/packages/file-server/btrfs/subvolume-bup.ts b/src/packages/file-server/btrfs/subvolume-bup.ts deleted file mode 100644 index a8ec09a945..0000000000 --- a/src/packages/file-server/btrfs/subvolume-bup.ts +++ /dev/null @@ -1,195 +0,0 @@ -/* - -BUP Architecture: - -There is a single global dedup'd backup archive stored in the btrfs filesystem. -Obviously, admins should rsync this regularly to a separate location as a genuine -backup strategy. - -NOTE: we use bup instead of btrfs send/recv ! - -Not used. Instead we will rely on bup (and snapshots of the underlying disk) for backups, since: - - much easier to check they are valid - - decoupled from any btrfs issues - - not tied to any specific filesystem at all - - easier to offsite via incremental rsync - - much more space efficient with *global* dedup and compression - - bup is really just git, which is much more proven than even btrfs - -The drawback is speed, but that can be managed. -*/ - -import { type DirectoryListingEntry } from "@cocalc/util/types"; -import { type Subvolume } from "./subvolume"; -import { sudo, parseBupTime } from "./util"; -import { join, normalize } from "path"; -import getLogger from "@cocalc/backend/logger"; - -const BUP_SNAPSHOT = "temp-bup-snapshot"; - -const logger = getLogger("file-server:btrfs:subvolume-bup"); - -export class SubvolumeBup { - constructor(private subvolume: Subvolume) {} - - // create a new bup backup - save = async ({ - // timeout used for bup index and bup save commands - timeout = 30 * 60 * 1000, - }: { timeout?: number } = {}) => { - if (await this.subvolume.snapshots.exists(BUP_SNAPSHOT)) { - logger.debug(`createBupBackup: deleting existing ${BUP_SNAPSHOT}`); - await this.subvolume.snapshots.delete(BUP_SNAPSHOT); - } - try { - logger.debug( - `createBackup: creating ${BUP_SNAPSHOT} to get a consistent backup`, - ); - await this.subvolume.snapshots.create(BUP_SNAPSHOT); - const target = await this.subvolume.fs.safeAbsPath( - this.subvolume.snapshots.path(BUP_SNAPSHOT), - ); - - logger.debug(`createBupBackup: indexing ${BUP_SNAPSHOT}`); - await sudo({ - command: "bup", - args: [ - "-d", - this.subvolume.filesystem.bup, - "index", - "--exclude", - join(target, ".snapshots"), - "-x", - target, - ], - timeout, - }); - - logger.debug(`createBackup: saving ${BUP_SNAPSHOT}`); - await sudo({ - command: "bup", - args: [ - "-d", - this.subvolume.filesystem.bup, - "save", - "--strip", - "-n", - this.subvolume.name, - target, - ], - timeout, - }); - } finally { - logger.debug(`createBupBackup: deleting temporary ${BUP_SNAPSHOT}`); - await this.subvolume.snapshots.delete(BUP_SNAPSHOT); - } - }; - - restore = async (path: string) => { - // path -- branch/revision/path/to/dir - if (path.startsWith("/")) { - path = path.slice(1); - } - path = normalize(path); - // ... but to avoid potential data loss, we make a snapshot before deleting it. - await this.subvolume.snapshots.create(); - const i = path.indexOf("/"); // remove the commit name - // remove the target we're about to restore - await this.subvolume.fs.rm(path.slice(i + 1), { recursive: true }); - await sudo({ - command: "bup", - args: [ - "-d", - this.subvolume.filesystem.bup, - "restore", - "-C", - this.subvolume.path, - join(`/${this.subvolume.name}`, path), - "--quiet", - ], - }); - }; - - // [ ] TODO: remove this ls and instead rely only on the fs sandbox code. - ls = async (path: string = ""): Promise => { - if (!path) { - const { stdout } = await sudo({ - command: "bup", - args: ["-d", this.subvolume.filesystem.bup, "ls", this.subvolume.name], - }); - const v: DirectoryListingEntry[] = []; - let newest = 0; - for (const x of stdout.trim().split("\n")) { - const name = x.split(" ").slice(-1)[0]; - if (name == "latest") { - continue; - } - const mtime = parseBupTime(name).valueOf() / 1000; - newest = Math.max(mtime, newest); - v.push({ name, isDir: true, mtime, size: -1 }); - } - if (v.length > 0) { - v.push({ name: "latest", isDir: true, mtime: newest, size: -1 }); - } - return v; - } - - path = (await this.subvolume.fs.safeAbsPath(path)).slice( - this.subvolume.path.length, - ); - const { stdout } = await sudo({ - command: "bup", - args: [ - "-d", - this.subvolume.filesystem.bup, - "ls", - "--almost-all", - "--file-type", - "-l", - join(`/${this.subvolume.name}`, path), - ], - }); - const v: DirectoryListingEntry[] = []; - for (const x of stdout.split("\n")) { - // [-rw-------","6b851643360e435eb87ef9a6ab64a8b1/6b851643360e435eb87ef9a6ab64a8b1","5","2025-07-15","06:12","a.txt"] - const w = x.split(/\s+/); - if (w.length >= 6) { - let isDir, name; - if (w[5].endsWith("@") || w[5].endsWith("=") || w[5].endsWith("|")) { - w[5] = w[5].slice(0, -1); - } - if (w[5].endsWith("/")) { - isDir = true; - name = w[5].slice(0, -1); - } else { - name = w[5]; - isDir = false; - } - const size = parseInt(w[2]); - const mtime = new Date(w[3] + "T" + w[4]).valueOf() / 1000; - v.push({ name, size, mtime, isDir }); - } - } - return v; - }; - - prune = async ({ - dailies = "1w", - monthlies = "4m", - all = "3d", - }: { dailies?: string; monthlies?: string; all?: string } = {}) => { - await sudo({ - command: "bup", - args: [ - "-d", - this.subvolume.filesystem.bup, - "prune-older", - `--keep-dailies-for=${dailies}`, - `--keep-monthlies-for=${monthlies}`, - `--keep-all-for=${all}`, - "--unsafe", - this.subvolume.name, - ], - }); - }; -} diff --git a/src/packages/file-server/btrfs/subvolume-rustic.ts b/src/packages/file-server/btrfs/subvolume-rustic.ts index 6200a42f0c..545ca41e7b 100644 --- a/src/packages/file-server/btrfs/subvolume-rustic.ts +++ b/src/packages/file-server/btrfs/subvolume-rustic.ts @@ -22,6 +22,8 @@ import { type Subvolume } from "./subvolume"; import getLogger from "@cocalc/backend/logger"; import { parseOutput } from "@cocalc/backend/sandbox/exec"; +export const RUSTIC = "rustic"; + const RUSTIC_SNAPSHOT = "temp-rustic-snapshot"; const logger = getLogger("file-server:btrfs:subvolume-rustic"); diff --git a/src/packages/file-server/btrfs/subvolume.ts b/src/packages/file-server/btrfs/subvolume.ts index 7eedfc0806..75b4836539 100644 --- a/src/packages/file-server/btrfs/subvolume.ts +++ b/src/packages/file-server/btrfs/subvolume.ts @@ -2,11 +2,10 @@ A subvolume */ -import { type Filesystem, DEFAULT_SUBVOLUME_SIZE } from "./filesystem"; +import { type Filesystem } from "./filesystem"; import refCache from "@cocalc/util/refcache"; import { sudo } from "./util"; import { join } from "path"; -import { SubvolumeBup } from "./subvolume-bup"; import { SubvolumeRustic } from "./subvolume-rustic"; import { SubvolumeSnapshots } from "./subvolume-snapshots"; import { SubvolumeQuota } from "./subvolume-quota"; @@ -27,7 +26,6 @@ export class Subvolume { public readonly filesystem: Filesystem; public readonly path: string; public readonly fs: SandboxedFilesystem; - public readonly bup: SubvolumeBup; public readonly rustic: SubvolumeRustic; public readonly snapshots: SubvolumeSnapshots; public readonly quota: SubvolumeQuota; @@ -40,7 +38,6 @@ export class Subvolume { rusticRepo: filesystem.rustic, host: this.name, }); - this.bup = new SubvolumeBup(this); this.rustic = new SubvolumeRustic(this); this.snapshots = new SubvolumeSnapshots(this); this.quota = new SubvolumeQuota(this); @@ -54,9 +51,6 @@ export class Subvolume { args: ["subvolume", "create", this.path], }); await this.chown(this.path); - await this.quota.set( - this.filesystem.opts.defaultSize ?? DEFAULT_SUBVOLUME_SIZE, - ); } }; @@ -69,7 +63,7 @@ export class Subvolume { delete this.path; // @ts-ignore delete this.snapshotsDir; - for (const sub of ["fs", "bup", "snapshots", "quota"]) { + for (const sub of ["fs", "rustic", "snapshots", "quota"]) { this[sub].close?.(); delete this[sub]; } diff --git a/src/packages/file-server/btrfs/subvolumes.ts b/src/packages/file-server/btrfs/subvolumes.ts index e4f10191b9..a193b37a1a 100644 --- a/src/packages/file-server/btrfs/subvolumes.ts +++ b/src/packages/file-server/btrfs/subvolumes.ts @@ -7,8 +7,9 @@ import { join } from "path"; import { btrfs } from "./util"; import { chmod, rename, rm } from "node:fs/promises"; import { SandboxedFilesystem } from "@cocalc/backend/sandbox"; +import { RUSTIC } from "./subvolume-rustic"; -const RESERVED = new Set(["bup", SNAPSHOTS]); +const RESERVED = new Set([RUSTIC, SNAPSHOTS]); const logger = getLogger("file-server:btrfs:subvolumes"); diff --git a/src/packages/file-server/btrfs/test/filesystem.test.ts b/src/packages/file-server/btrfs/test/filesystem.test.ts index fc5dbc1c04..c4eea6992a 100644 --- a/src/packages/file-server/btrfs/test/filesystem.test.ts +++ b/src/packages/file-server/btrfs/test/filesystem.test.ts @@ -1,5 +1,6 @@ import { before, after, fs } from "./setup"; import { isValidUUID } from "@cocalc/util/misc"; +import { RUSTIC } from "@cocalc/file-server/btrfs/subvolume-rustic"; beforeAll(before); @@ -22,7 +23,7 @@ describe("some basic tests", () => { describe("operations with subvolumes", () => { it("can't use a reserved subvolume name", async () => { expect(async () => { - await fs.subvolumes.get("bup"); + await fs.subvolumes.get(RUSTIC); }).rejects.toThrow("is reserved"); }); diff --git a/src/packages/file-server/btrfs/test/rustic.test.ts b/src/packages/file-server/btrfs/test/rustic.test.ts index feb36ce5a5..674cd1e6ed 100644 --- a/src/packages/file-server/btrfs/test/rustic.test.ts +++ b/src/packages/file-server/btrfs/test/rustic.test.ts @@ -63,7 +63,7 @@ describe("test rustic backups", () => { ).toEqual("hello2"); // forget the second snapshot - const z = await vol.rustic.forget({ id }); + await vol.rustic.forget({ id }); const v2 = await vol.rustic.snapshots(); expect(v2.length).toBe(1); expect(v2[0].id).not.toEqual(id); diff --git a/src/packages/file-server/btrfs/test/setup.ts b/src/packages/file-server/btrfs/test/setup.ts index c9736c8b79..6ee6d8460b 100644 --- a/src/packages/file-server/btrfs/test/setup.ts +++ b/src/packages/file-server/btrfs/test/setup.ts @@ -50,7 +50,7 @@ export async function before() { await chmod(mount, 0o777); fs = await filesystem({ device: join(tempDir, "btrfs.img"), - formatIfNeeded: true, + size: "1G", mount: join(tempDir, "mnt"), }); } diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index fcdb878f7d..7f1a03050b 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -1,6 +1,4 @@ import { before, after, fs, sudo } from "./setup"; -import { mkdir } from "fs/promises"; -import { join } from "path"; import { wait } from "@cocalc/backend/conat/test/util"; import { randomBytes } from "crypto"; import { type Subvolume } from "../subvolume"; @@ -227,71 +225,4 @@ describe("test snapshots", () => { }); }); -describe.skip("test bup backups", () => { - let vol: Subvolume; - it("creates a volume", async () => { - vol = await fs.subvolumes.get("bup-test"); - await vol.fs.writeFile("a.txt", "hello"); - }); - - it("create a bup backup", async () => { - await vol.bup.save(); - }); - - it("list bup backups of this vol -- there are 2, one for the date and 'latest'", async () => { - const v = await vol.bup.ls(); - expect(v.length).toBe(2); - const t = (v[0].mtime ?? 0) * 1000; - expect(Math.abs(t.valueOf() - Date.now())).toBeLessThan(10_000); - }); - - it("confirm a.txt is in our backup", async () => { - const x = await vol.bup.ls("latest"); - expect(x).toEqual([ - { name: "a.txt", size: 5, mtime: x[0].mtime, isDir: false }, - ]); - }); - - it("restore a.txt from our backup", async () => { - await vol.fs.writeFile("a.txt", "hello2"); - await vol.bup.restore("latest/a.txt"); - expect(await vol.fs.readFile("a.txt", "utf8")).toEqual("hello"); - }); - - it("prune bup backups does nothing since we have so few", async () => { - await vol.bup.prune(); - expect((await vol.bup.ls()).length).toBe(2); - }); - - it("add a directory and back up", async () => { - await mkdir(join(vol.path, "mydir")); - await vol.fs.writeFile(join("mydir", "file.txt"), "hello3"); - expect((await vol.fs.readdir("mydir"))[0]).toBe("file.txt"); - await vol.bup.save(); - const x = await vol.bup.ls("latest"); - expect(x).toEqual([ - { name: "a.txt", size: 5, mtime: x[0].mtime, isDir: false }, - { name: "mydir", size: 0, mtime: x[1].mtime, isDir: true }, - ]); - expect(Math.abs((x[0].mtime ?? 0) * 1000 - Date.now())).toBeLessThan( - 5 * 60_000, - ); - }); - - it("change file in the directory, then restore from backup whole dir", async () => { - await vol.fs.writeFile(join("mydir", "file.txt"), "changed"); - await vol.bup.restore("latest/mydir"); - expect(await vol.fs.readFile(join("mydir", "file.txt"), "utf8")).toEqual( - "hello3", - ); - }); - - it("most recent snapshot has a backup before the restore", async () => { - const s = await vol.snapshots.readdir(); - const recent = s.slice(-1)[0]; - const p = vol.snapshots.path(recent, "mydir", "file.txt"); - expect(await vol.fs.readFile(p, "utf8")).toEqual("changed"); - }); -}); - afterAll(after); diff --git a/src/packages/server/conat/file-server/index.ts b/src/packages/server/conat/file-server/index.ts index ce5ace5ee3..61f49bd004 100644 --- a/src/packages/server/conat/file-server/index.ts +++ b/src/packages/server/conat/file-server/index.ts @@ -152,9 +152,8 @@ export async function init() { fs = await filesystem({ device: btrfsDevice, - formatIfNeeded: true, + size: "25G", mount: mountPoint, - defaultFilesystemSize: "25G", }); server = await createFileServer({ From 7b026e2a731e28deb93d6da63c753b2a23159ae0 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 11 Aug 2025 23:19:06 +0000 Subject: [PATCH 262/798] rustic: work in progress exposing backups of file-server over conat --- src/packages/conat/files/file-server.ts | 32 ++++++++ src/packages/conat/hub/api/projects.ts | 2 + .../file-server/btrfs/subvolume-rustic.ts | 16 ++-- .../file-server/btrfs/test/rustic.test.ts | 4 +- .../server/conat/file-server/index.ts | 8 +- .../server/conat/file-server/rustic.ts | 76 +++++++++++++++++++ 6 files changed, 128 insertions(+), 10 deletions(-) create mode 100644 src/packages/server/conat/file-server/rustic.ts diff --git a/src/packages/conat/files/file-server.ts b/src/packages/conat/files/file-server.ts index f6bd47ff89..dbbd59787b 100644 --- a/src/packages/conat/files/file-server.ts +++ b/src/packages/conat/files/file-server.ts @@ -58,6 +58,38 @@ export interface Fileserver { dest: { project_id: string; path: string }; options?: CopyOptions; }) => Promise; + + // create new complete backup of the project; this first snapshots the + // project, makes a backup of the snapshot, then deletes the snapshot, so the + // backup is guranteed to be consistent. + backup: (opts: { project_id: string }) => Promise<{ time: Date; id: string }>; + + // restore the given path in the backup to the given dest. The default + // path is '' (the whole project) and the default destination is the + // same as the path. + restore: (opts: { + project_id: string; + id: string; + path?: string; + dest?: string; + }) => Promise; + + // delete the given backup + deleteBackup: (opts: { project_id: string; id: string }) => Promise; + + // Return list of id's and timestamps of all backups of this project. + getBackups: (opts: { project_id: string }) => Promise< + { + id: string; + time: Date; + }[] + >; + + // Return list of all files in the given backup. + getBackupFiles: (opts: { + project_id: string; + id: string; + }) => Promise; } export interface Options extends Fileserver { diff --git a/src/packages/conat/hub/api/projects.ts b/src/packages/conat/hub/api/projects.ts index fa79963587..c089a3ea25 100644 --- a/src/packages/conat/hub/api/projects.ts +++ b/src/packages/conat/hub/api/projects.ts @@ -110,4 +110,6 @@ export interface Projects { account_id?: string; project_id: string; }) => Promise<{ used: number; size: number }>; + + } diff --git a/src/packages/file-server/btrfs/subvolume-rustic.ts b/src/packages/file-server/btrfs/subvolume-rustic.ts index 545ca41e7b..040c91cc90 100644 --- a/src/packages/file-server/btrfs/subvolume-rustic.ts +++ b/src/packages/file-server/btrfs/subvolume-rustic.ts @@ -37,7 +37,9 @@ export class SubvolumeRustic { constructor(private subvolume: Subvolume) {} // create a new rustic backup - backup = async ({ timeout = 30 * 60 * 1000 }: { timeout?: number } = {}) => { + backup = async ({ + timeout = 30 * 60 * 1000, + }: { timeout?: number } = {}): Promise => { if (await this.subvolume.snapshots.exists(RUSTIC_SNAPSHOT)) { logger.debug(`backup: deleting existing ${RUSTIC_SNAPSHOT}`); await this.subvolume.snapshots.delete(RUSTIC_SNAPSHOT); @@ -49,16 +51,14 @@ export class SubvolumeRustic { ); await this.subvolume.snapshots.create(RUSTIC_SNAPSHOT); logger.debug(`backup: backing up ${RUSTIC_SNAPSHOT} using rustic`); - const { stderr, code } = await this.subvolume.fs.rustic( - ["backup", "-x", "."], - { + const { stdout } = parseOutput( + await this.subvolume.fs.rustic(["backup", "-x", "--json", "."], { timeout, cwd: target, - }, + }), ); - if (code) { - throw Error(stderr.toString()); - } + const { time, id } = JSON.parse(stdout); + return { time: new Date(time), id }; } finally { logger.debug(`backup: deleting temporary ${RUSTIC_SNAPSHOT}`); await this.subvolume.snapshots.delete(RUSTIC_SNAPSHOT); diff --git a/src/packages/file-server/btrfs/test/rustic.test.ts b/src/packages/file-server/btrfs/test/rustic.test.ts index 674cd1e6ed..5c75e46dde 100644 --- a/src/packages/file-server/btrfs/test/rustic.test.ts +++ b/src/packages/file-server/btrfs/test/rustic.test.ts @@ -10,13 +10,15 @@ describe("test rustic backups", () => { await vol.fs.writeFile("a.txt", "hello"); }); + let x; it("create a rustic backup", async () => { - await vol.rustic.backup(); + x = await vol.rustic.backup(); }); it("confirm a.txt is in our backup", async () => { const v = await vol.rustic.snapshots(); expect(v.length == 1); + expect(v[0]).toEqual(x); expect(Math.abs(Date.now() - v[0].time.valueOf())).toBeLessThan(10000); const { id } = v[0]; const w = await vol.rustic.ls({ id }); diff --git a/src/packages/server/conat/file-server/index.ts b/src/packages/server/conat/file-server/index.ts index 61f49bd004..d2f78cb76c 100644 --- a/src/packages/server/conat/file-server/index.ts +++ b/src/packages/server/conat/file-server/index.ts @@ -26,6 +26,7 @@ import { join } from "node:path"; import { mkdir } from "fs/promises"; import { filesystem, type Filesystem } from "@cocalc/file-server/btrfs"; import { exists } from "@cocalc/backend/misc/async-utils-node"; +import * as rustic from "./rustic"; const logger = getLogger("server:conat:file-server"); @@ -33,7 +34,7 @@ function name(project_id: string) { return `project-${project_id}`; } -async function getVolume(project_id) { +export async function getVolume(project_id) { if (fs == null) { throw Error("file server not initialized"); } @@ -164,6 +165,11 @@ export async function init() { getQuota: reuseInFlight(getQuota), setQuota, cp, + backup: reuseInFlight(rustic.backup), + restore: rustic.restore, + deleteBackup: rustic.deleteBackup, + getBackups: reuseInFlight(rustic.getBackups), + getBackupFiles: reuseInFlight(rustic.getBackupFiles), }); } diff --git a/src/packages/server/conat/file-server/rustic.ts b/src/packages/server/conat/file-server/rustic.ts new file mode 100644 index 0000000000..b89bcff4eb --- /dev/null +++ b/src/packages/server/conat/file-server/rustic.ts @@ -0,0 +1,76 @@ +import { getVolume } from "./index"; +import getLogger from "@cocalc/backend/logger"; + +const logger = getLogger("server:conat:file-server:rustic"); + +// create new complete backup of the project; this first snapshots the +// project, makes a backup of the snapshot, then deletes the snapshot, so the +// backup is guranteed to be consistent. +export async function backup({ + project_id, +}: { + project_id: string; +}): Promise<{ time: Date; id: string }> { + logger.debug("backup", { project_id }); + const vol = await getVolume(project_id); + return await vol.rustic.backup(); +} + +// restore the given path in the backup to the given dest. The default +// path is '' (the whole project) and the default destination is the +// same as the path. +export async function restore({ + project_id, + id, + path, + dest, +}: { + project_id: string; + id: string; + path?: string; + dest?: string; +}): Promise { + logger.debug("restore", { project_id, id, path, dest }); + const vol = await getVolume(project_id); + await vol.rustic.restore({ id, path, dest }); +} + +export async function deleteBackup({ + project_id, + id, +}: { + project_id: string; + id: string; +}): Promise { + logger.debug("deleteBackup", { project_id, id }); + const vol = await getVolume(project_id); + await vol.rustic.forget({ id }); +} + +// Return list of id's and timestamps of all backups of this project. +export async function getBackups({ + project_id, +}: { + project_id: string; +}): Promise< + { + id: string; + time: Date; + }[] +> { + logger.debug("getBackups", { project_id }); + const vol = await getVolume(project_id); + return await vol.rustic.snapshots(); +} +// Return list of all files in the given backup. +export async function getBackupFiles({ + project_id, + id, +}: { + project_id: string; + id: string; +}): Promise { + logger.debug("getBackupFiles", { project_id, id }); + const vol = await getVolume(project_id); + return await vol.rustic.ls({ id }); +} From f9ef536e75701e5e10a1279b640be6277cfdcac0 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 12 Aug 2025 00:48:27 +0000 Subject: [PATCH 263/798] rustic: support toml config for much more sophisticated use of rustic backups; and move backup out of btrfs fs. --- .../conat/test/files/file-server.test.ts | 36 ++++++++++ src/packages/backend/sandbox/rustic.test.ts | 24 ++++++- src/packages/backend/sandbox/rustic.ts | 13 +++- src/packages/file-server/btrfs/filesystem.ts | 66 +++++++++---------- src/packages/file-server/btrfs/test/setup.ts | 2 +- .../server/conat/file-server/index.ts | 3 +- 6 files changed, 104 insertions(+), 40 deletions(-) diff --git a/src/packages/backend/conat/test/files/file-server.test.ts b/src/packages/backend/conat/test/files/file-server.test.ts index da5c93ad1c..e31e835479 100644 --- a/src/packages/backend/conat/test/files/file-server.test.ts +++ b/src/packages/backend/conat/test/files/file-server.test.ts @@ -68,6 +68,42 @@ describe("create basic mocked file server and test it out", () => { dest: { project_id: string; path: string }; options?; }): Promise => {}, + + backup: async (_opts: { + project_id: string; + }): Promise<{ time: Date; id: string }> => { + return { time: new Date(), id: "0" }; + }, + + restore: async (_opts: { + project_id: string; + id: string; + path?: string; + dest?: string; + }): Promise => {}, + + deleteBackup: async (_opts: { + project_id: string; + id: string; + }): Promise => {}, + + getBackups: async (_opts: { + project_id: string; + }): Promise< + { + id: string; + time: Date; + }[] + > => { + return []; + }, + + getBackupFiles: async (_opts: { + project_id: string; + id: string; + }): Promise => { + return []; + }, }); }); diff --git a/src/packages/backend/sandbox/rustic.test.ts b/src/packages/backend/sandbox/rustic.test.ts index db1c9ae1f9..0e9561c27c 100644 --- a/src/packages/backend/sandbox/rustic.test.ts +++ b/src/packages/backend/sandbox/rustic.test.ts @@ -8,12 +8,13 @@ import rustic from "./rustic"; import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; +import { parseOutput } from "./exec"; -let tempDir, options; +let tempDir, options, home; beforeAll(async () => { tempDir = await mkdtemp(join(tmpdir(), "cocalc")); const repo = join(tempDir, "repo"); - const home = join(tempDir, "home"); + home = join(tempDir, "home"); await mkdir(home); const safeAbsPath = (path: string) => join(home, resolve("/", path)); options = { @@ -48,6 +49,25 @@ describe("rustic does something", () => { expect(truncated).toBe(false); }); + it("use a .toml file instead of explicitly passing in a repo", async () => { + await mkdir(join(home, "x")); + await writeFile( + join(home, "x/a.toml"), + ` +[repository] +repository = "${options.repo}" +password = "" +`, + ); + const options2 = { ...options, repo: join(home, "x/a.toml") }; + const { stdout } = parseOutput( + await rustic(["snapshots", "--json"], options2), + ); + const s = JSON.parse(stdout); + expect(s.length).toEqual(1); + expect(s[0][0].hostname).toEqual("my-host"); + }); + // it("it appears in the snapshots list", async () => { // const { stdout, truncated } = await rustic( // ["snapshots", "--json"], diff --git a/src/packages/backend/sandbox/rustic.ts b/src/packages/backend/sandbox/rustic.ts index 28bb7e4a96..eb4d2d7d5e 100644 --- a/src/packages/backend/sandbox/rustic.ts +++ b/src/packages/backend/sandbox/rustic.ts @@ -84,11 +84,16 @@ export default async function rustic( host = "host", } = options; + let common; + if (repo.endsWith(".toml")) { + common = ["-P", repo.slice(0, -".toml".length)]; + } else { + common = ["--password", "", "-r", repo]; + } + await ensureInitialized(repo); const cwd = await safeAbsPath?.(options.cwd ?? ""); - const common = ["--password", "", "-r", repo]; - const run = async (sanitizedArgs: string[]) => { return await exec({ cmd: rusticPath, @@ -306,6 +311,10 @@ const whitelist = { } as const; async function ensureInitialized(repo: string) { + if (repo.endsWith(".toml")) { + // nothing to do + return; + } const config = join(repo, "config"); if (!(await exists(config))) { await exec({ diff --git a/src/packages/file-server/btrfs/filesystem.ts b/src/packages/file-server/btrfs/filesystem.ts index d675c863b7..e3c46a5036 100644 --- a/src/packages/file-server/btrfs/filesystem.ts +++ b/src/packages/file-server/btrfs/filesystem.ts @@ -13,24 +13,31 @@ a = require('@cocalc/file-server/btrfs'); fs = await a.filesystem({device:'/tmp/ import refCache from "@cocalc/util/refcache"; import { mkdirp, btrfs, sudo } from "./util"; -import { join } from "path"; import { Subvolumes } from "./subvolumes"; import { mkdir } from "fs/promises"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import rustic from "@cocalc/backend/sandbox/rustic"; -import { RUSTIC } from "./subvolume-rustic"; export interface Options { - // the underlying block device. - // If this is a file (or filename) ending in .img, then it's a sparse file mounted as a loopback device. - // If this starts with "/dev" then it is a raw block device. - device: string; - // where to mount the btrfs filesystem + // mount = root mountpoint of the btrfs filesystem. If you specify the image + // path below, then a btrfs filesystem will get automatically created (via sudo + // and a loopback device). mount: string; - // size -- if true and 'device' is a path to a .img file that DOES NOT EXIST, create device - // as a sparse image file of the given size. If img already exists, it will not be touched - // in any way, and it is up to you to mkfs.btrfs it, etc. + + // image = optioanlly use a image file at this location for the btrfs filesystem. + // This is used for development and in Docker. It will be created as a sparse image file + // with given size, and mounted at opts.mount if it does not exist. If you create + // it be sure to use mkfs.btrfs to format it. + image?: string; size?: string | number; + + // rustic = the rustic backups path. + // If this path ends in .toml, it is the configuration file for rustic, e.g., you can + // configure rustic however you want by pointing this at a toml cofig file. + // Otherwise, if this path does not exist, it will be created a new rustic repo + // initialized here. + // If not given, then backups will throw an error. + rustic?: string; } export class Filesystem { @@ -40,7 +47,6 @@ export class Filesystem { constructor(opts: Options) { this.opts = opts; - this.rustic = join(this.opts.mount, RUSTIC); this.subvolumes = new Subvolumes(this); } @@ -51,7 +57,9 @@ export class Filesystem { await btrfs({ args: ["quota", "enable", "--simple", this.opts.mount], }); - await this.initRustic(); + if (this.opts.rustic) { + await this.initRustic(); + } await this.sync(); }; @@ -70,22 +78,16 @@ export class Filesystem { close = () => {}; private initDevice = async () => { - if (!isImageFile(this.opts.device)) { - // raw block device -- nothing to do + if (!this.opts.image) { return; } - if (!(await exists(this.opts.device))) { - if (!this.opts.size) { - throw Error( - "you must specify the size of the btrfs sparse image file, or explicitly create and format it", - ); - } + if (!(await exists(this.opts.image))) { // we create and format the sparse image await sudo({ command: "truncate", - args: ["-s", `${this.opts.size}`, this.opts.device], + args: ["-s", `${this.opts.size ?? "10G"}`, this.opts.image], }); - await sudo({ command: "mkfs.btrfs", args: [this.opts.device] }); + await sudo({ command: "mkfs.btrfs", args: [this.opts.image] }); } }; @@ -115,7 +117,10 @@ export class Filesystem { }; private _mountFilesystem = async () => { - const args: string[] = isImageFile(this.opts.device) ? ["-o", "loop"] : []; + if (!this.opts.image) { + throw Error(`there must be a btrfs filesystem at ${this.opts.mount}`); + } + const args: string[] = ["-o", "loop"]; args.push( "-o", "compress=zstd", @@ -125,7 +130,7 @@ export class Filesystem { "space_cache=v2", "-o", "autodefrag", - this.opts.device, + this.opts.image, "-t", "btrfs", this.opts.mount, @@ -152,22 +157,17 @@ export class Filesystem { }; private initRustic = async () => { - if (await exists(this.rustic)) { + if (!this.rustic || (await exists(this.rustic))) { return; } + if (this.rustic.endsWith(".toml")) { + throw Error(`file not found: ${this.rustic}`); + } await mkdir(this.rustic); await rustic(["init"], { repo: this.rustic }); }; } -function isImageFile(name: string) { - if (name.startsWith("/dev")) { - return false; - } - // TODO: could probably check os for a device with given name? - return name.endsWith(".img"); -} - const cache = refCache({ name: "btrfs-filesystems", createObject: async (options: Options) => { diff --git a/src/packages/file-server/btrfs/test/setup.ts b/src/packages/file-server/btrfs/test/setup.ts index 6ee6d8460b..b6acd1bf6a 100644 --- a/src/packages/file-server/btrfs/test/setup.ts +++ b/src/packages/file-server/btrfs/test/setup.ts @@ -49,7 +49,7 @@ export async function before() { await mkdir(mount); await chmod(mount, 0o777); fs = await filesystem({ - device: join(tempDir, "btrfs.img"), + image: join(tempDir, "btrfs.img"), size: "1G", mount: join(tempDir, "mnt"), }); diff --git a/src/packages/server/conat/file-server/index.ts b/src/packages/server/conat/file-server/index.ts index d2f78cb76c..3ef3628efb 100644 --- a/src/packages/server/conat/file-server/index.ts +++ b/src/packages/server/conat/file-server/index.ts @@ -145,14 +145,13 @@ export async function init() { if (!(await exists(image))) { await mkdir(image, { recursive: true }); } - const btrfsDevice = join(image, "btrfs.img"); const mountPoint = join(data, "btrfs", "mnt"); if (!(await exists(mountPoint))) { await mkdir(mountPoint, { recursive: true }); } fs = await filesystem({ - device: btrfsDevice, + image: join(image, "btrfs.img"), size: "25G", mount: mountPoint, }); From 80363877298fa258942e5b20037ba52648b09078 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 12 Aug 2025 01:20:09 +0000 Subject: [PATCH 264/798] size --> disk --- src/packages/file-server/btrfs/filesystem.ts | 2 +- src/packages/server/conat/project/load-balancer.ts | 4 ++-- src/packages/server/conat/project/run.ts | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/packages/file-server/btrfs/filesystem.ts b/src/packages/file-server/btrfs/filesystem.ts index e3c46a5036..c1e4947bd9 100644 --- a/src/packages/file-server/btrfs/filesystem.ts +++ b/src/packages/file-server/btrfs/filesystem.ts @@ -7,7 +7,7 @@ Start node, then: DEBUG="cocalc:*file-server*" DEBUG_CONSOLE=yes node -a = require('@cocalc/file-server/btrfs'); fs = await a.filesystem({device:'/tmp/btrfs.img', formatIfNeeded:true, mount:'/mnt/btrfs', uid:293597964}) +a = require('@cocalc/file-server/btrfs'); fs = await a.filesystem({image:'/tmp/btrfs.img', mount:'/mnt/btrfs', size:'2G'}) */ diff --git a/src/packages/server/conat/project/load-balancer.ts b/src/packages/server/conat/project/load-balancer.ts index ba7ee15dc8..6c2dbed152 100644 --- a/src/packages/server/conat/project/load-balancer.ts +++ b/src/packages/server/conat/project/load-balancer.ts @@ -38,7 +38,7 @@ async function getConfig({ project_id }) { throw Error(`no project with id ${project_id}`); } if (rows[0].settings?.admin) { - return { admin: true, size: "10G" }; + return { admin: true, disk: "25G" }; } else { // some defaults, mainly for testing return { @@ -46,7 +46,7 @@ async function getConfig({ project_id }) { memory: "8Gi", pids: 10000, swap: "5000Gi", - size: "1000M", + disk: "1G", }; } } diff --git a/src/packages/server/conat/project/run.ts b/src/packages/server/conat/project/run.ts index e844cfe573..398ad67761 100644 --- a/src/packages/server/conat/project/run.ts +++ b/src/packages/server/conat/project/run.ts @@ -152,11 +152,11 @@ async function start({ await setupDataPath(home, uid); await writeSecretToken(home, await getProjectSecretToken(project_id), uid); - if (config?.size) { + if (config?.disk) { // TODO: maybe this should be done in parallel with other things // to make startup time slightly faster (?) -- could also be incorporated // into mount. - await setQuota(project_id, config.size); + await setQuota(project_id, config.disk); } let script: string, From ee87b547a65a7da4fa40ea797154a4d6c2d879b9 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 12 Aug 2025 01:31:19 +0000 Subject: [PATCH 265/798] ts and pnpm --- src/packages/pnpm-lock.yaml | 132 ++++++--------------- src/packages/server/conat/project/types.ts | 2 +- 2 files changed, 39 insertions(+), 95 deletions(-) diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index a5a2d3de87..2d91207588 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -1212,22 +1212,22 @@ importers: version: 1.4.1 '@langchain/anthropic': specifier: ^0.3.26 - version: 0.3.26(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76))) + version: 0.3.26(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76))) '@langchain/core': specifier: ^0.3.68 - version: 0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76)) + version: 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)) '@langchain/google-genai': specifier: ^0.2.16 - version: 0.2.16(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76))) + version: 0.2.16(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76))) '@langchain/mistralai': specifier: ^0.2.1 - version: 0.2.1(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76)))(zod@3.25.76) + version: 0.2.1(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)))(zod@3.25.76) '@langchain/ollama': specifier: ^0.2.3 - version: 0.2.3(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76))) + version: 0.2.3(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76))) '@langchain/openai': specifier: ^0.6.6 - version: 0.6.6(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76)))(ws@8.18.3) + version: 0.6.7(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)))(ws@8.18.3(utf-8-validate@6.0.5)) '@node-saml/passport-saml': specifier: ^5.1.0 version: 5.1.0 @@ -1338,7 +1338,7 @@ importers: version: 6.10.1 openai: specifier: ^5.12.1 - version: 5.12.1(ws@8.18.3)(zod@3.25.76) + version: 5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76) parse-domain: specifier: ^5.0.0 version: 5.0.0(encoding@0.1.13) @@ -3060,8 +3060,8 @@ packages: peerDependencies: '@langchain/core': ^0.3.68 - '@langchain/core@0.3.68': - resolution: {integrity: sha512-dWPT1h9ObG1TK9uivFTk/pgBULZ6/tBmq8czGUjZjR+1xh9jB4tm/D5FY6o5FklXcEpnAI9peNq2x17Kl9wbMg==} + '@langchain/core@0.3.69': + resolution: {integrity: sha512-N6ZmgcnoMnGw+hQuS8w8FrNUm/5FuvtB868Jr1i1+4pASngLLVUyjeAQbKBBFMFH+WY5ga9LSvaQegUe3TLF8g==} engines: {node: '>=18'} '@langchain/google-genai@0.2.16': @@ -3082,8 +3082,8 @@ packages: peerDependencies: '@langchain/core': ^0.3.68 - '@langchain/openai@0.6.6': - resolution: {integrity: sha512-0fxSg290WTCTEM0PECDGfst2QYUiULKhzyydaOPLMQ5pvWHjJkzBudx+CyHkeQ8DvGXysJteSmZzAMjRCj4duQ==} + '@langchain/openai@0.6.7': + resolution: {integrity: sha512-mNT9AdfEvDjlWU76hEl1HgTFkgk7yFKdIRgQz3KXKZhEERXhAwYJNgPFq8+HIpgxYSnc12akZ1uo8WPS98ErPQ==} engines: {node: '>=18'} peerDependencies: '@langchain/core': ^0.3.68 @@ -4349,8 +4349,6 @@ packages: '@types/node-cleanup@2.1.5': resolution: {integrity: sha512-+82RAk5uYiqiMoEv2fPeh03AL4pB5d3TL+Pf+hz31Mme6ECFI1kRlgmxYjdSlHzDbJ9yLorTnKi4Op5FA54kQQ==} - '@types/node-fetch@2.6.13': - resolution: {integrity: sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==} '@types/node-forge@1.3.13': resolution: {integrity: sha512-zePQJSW5QkwSHKRApqWCVKeKoSOt4xvEnLENZPjyvm9Ezdf/EyDeJM7jqLzOwjVICQQzvLZ63T55MKdJB5H6ww==} @@ -4360,9 +4358,6 @@ packages: '@types/node@18.19.118': resolution: {integrity: sha512-hIPK0hSrrcaoAu/gJMzN3QClXE4QdCdFvaenJ0JsjIbExP1JFFVH+RHcBt25c9n8bx5dkIfqKE+uw6BmBns7ug==} - '@types/node@24.2.1': - resolution: {integrity: sha512-DRh5K+ka5eJic8CjH7td8QpYEV6Zo10gfRkjHCO3weqZHWDtAaSTFtl4+VMqOJ4N5jcuhZ9/l+yy8rVgw7BQeQ==} - '@types/nodemailer@6.4.17': resolution: {integrity: sha512-I9CCaIp6DTldEg7vyUTZi8+9Vo0hi1/T8gv3C89yk1rSAAzoKQ8H8ki/jBYJSFoH/BisgLP8tkZMlQ91CIquww==} @@ -5132,11 +5127,6 @@ packages: engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} hasBin: true - browserslist@4.25.2: - resolution: {integrity: sha512-0si2SJK3ooGzIawRu61ZdPCO1IncZwS8IzuX73sPZsXW6EQ/w/DAfPyKI8l1ETTCr2MnvqWitmlCUxgdul45jA==} - engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} - hasBin: true - bs-logger@0.2.6: resolution: {integrity: sha512-pd8DCoxmbgc7hyPKOvxtqNcjYoOsABPQdcCUjGp3d42VR2CX1ORhk2A87oqqu5R1kk+76nsxZupkmyd+MVtCog==} engines: {node: '>= 6'} @@ -5210,9 +5200,6 @@ packages: caniuse-lite@1.0.30001727: resolution: {integrity: sha512-pB68nIHmbN6L/4C6MH1DokyR3bYqFwjaSs/sWDHGj4CTcFtQUQMuJftVwWkXq7mNWOybD3KhUv3oWHoGxgP14Q==} - caniuse-lite@1.0.30001733: - resolution: {integrity: sha512-e4QKw/O2Kavj2VQTKZWrwzkt3IxOmIlU6ajRb6LP64LHpBo1J67k2Hi4Vu/TgJWsNtynurfS0uK3MaUTCPfu5Q==} - canvas-fit@1.5.0: resolution: {integrity: sha512-onIcjRpz69/Hx5bB5HGbYKUF2uC6QT6Gp+pfpGm3A7mPfcluSLV5v4Zu+oflDUwLdUw0rLIBhUbi0v8hM4FJQQ==} @@ -6321,9 +6308,6 @@ packages: electron-to-chromium@1.5.182: resolution: {integrity: sha512-Lv65Btwv9W4J9pyODI6EWpdnhfvrve/us5h1WspW8B2Fb0366REPtY3hX7ounk1CkV/TBjWCEvCBBbYbmV0qCA==} - electron-to-chromium@1.5.199: - resolution: {integrity: sha512-3gl0S7zQd88kCAZRO/DnxtBKuhMO4h0EaQIN3YgZfV6+pW+5+bf2AdQeHNESCoaQqo/gjGVYEf2YM4O5HJQqpQ==} - element-size@1.1.1: resolution: {integrity: sha512-eaN+GMOq/Q+BIWy0ybsgpcYImjGIdNLyjLFJU4XsLHXYQao5jCNb36GyN6C2qwmDDYSfIBmKpPpr4VnBdLCsPQ==} @@ -6382,8 +6366,8 @@ packages: resolution: {integrity: sha512-ZCkIjSYNDyGn0R6ewHDtXgns/Zre/NT6Agvq1/WobF7JXgFff4SeDroKiCO3fNJreU9YG429Sc81o4w5ok/W5g==} engines: {node: '>=10.2.0'} - enhanced-resolve@5.18.3: - resolution: {integrity: sha512-d4lC8xfavMeBjzGr2vECC3fsGXziXZQyJxD868h2M/mBI3PwAuODxAkLkq5HYuvrPYcUtiLzsTo8U3PgX3Ocww==} + enhanced-resolve@5.18.2: + resolution: {integrity: sha512-6Jw4sE1maoRJo3q8MsSIn2onJFbLTOjY9hlx4DZXmOKvLRd1Ok2kXmAGXaafL2+ijsJZ1ClYbl/pmqr9+k4iUQ==} engines: {node: '>=10.13.0'} entities@2.2.0: @@ -6815,9 +6799,6 @@ packages: resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==} engines: {node: '>=14'} - form-data-encoder@1.7.2: - resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==} - form-data@2.5.5: resolution: {integrity: sha512-jqdObeR2rxZZbPSGL+3VckHMYtu+f9//KXBsVny6JSX/pa38Fy+bGjuG8eW/H6USNQWhLi8Num++cU2yOCNz4A==} engines: {node: '>= 0.12'} @@ -6950,9 +6931,6 @@ packages: gl-matrix@3.4.3: resolution: {integrity: sha512-wcCp8vu8FT22BnvKVPjXa/ICBWRq/zjFfdofZy1WSpQZpphblv12/bOQLBC1rMM7SGOFS9ltVmKOHil5+Ml7gA==} - gl-matrix@3.4.4: - resolution: {integrity: sha512-latSnyDNt/8zYUB6VIJ6PCh2jBjJX6gnDsoCZ7LyW7GkqrD51EWwa9qCoGixj8YqBtETQK/xY7OmpTF8xz1DdQ==} - gl-text@1.4.0: resolution: {integrity: sha512-o47+XBqLCj1efmuNyCHt7/UEJmB9l66ql7pnobD6p+sgmBUdzfMZXIF0zD2+KRfpd99DJN+QXdvTFAGCKCVSmQ==} @@ -8979,8 +8957,8 @@ packages: resolution: {integrity: sha512-cxN6aIDPz6rm8hbebcP7vrQNhvRcveZoJU72Y7vskh4oIm+BZwBECnx5nTmrlres1Qapvx27Qo1Auukpf8PKXw==} engines: {node: '>=18'} - openai@5.12.1: - resolution: {integrity: sha512-26s536j4Fi7P3iUma1S9H33WRrw0Qu8pJ2nYJHffrlKHPU0JK4d0r3NcMgqEcAeTdNLGYNyoFsqN4g4YE9vutg==} + openai@5.12.2: + resolution: {integrity: sha512-xqzHHQch5Tws5PcKR2xsZGX9xtch+JQFz5zb14dGqlshmmDAFBFEWmeIpf7wVqWV+w7Emj7jRgkNJakyKE0tYQ==} hasBin: true peerDependencies: ws: ^8.18.0 @@ -11149,9 +11127,6 @@ packages: undici-types@5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} - undici-types@7.10.0: - resolution: {integrity: sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag==} - unicorn-magic@0.3.0: resolution: {integrity: sha512-+QBBXBCvifc56fsbuxZQ6Sic3wqqc3WWaqxs58gvJrcOuN83HGTCwz3oS5phzU9LthRNE9VrJCFCLUgHeeFnfA==} engines: {node: '>=18'} @@ -13140,20 +13115,20 @@ snapshots: '@lumino/properties': 2.0.3 '@lumino/signaling': 2.1.4 - '@langchain/anthropic@0.3.26(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76)))': + '@langchain/anthropic@0.3.26(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)))': dependencies: '@anthropic-ai/sdk': 0.56.0 - '@langchain/core': 0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76)) + '@langchain/core': 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)) fast-xml-parser: 4.5.3 - '@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76))': + '@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76))': dependencies: '@cfworker/json-schema': 4.1.1 ansi-styles: 5.2.0 camelcase: 6.3.0 decamelize: 1.2.0 js-tiktoken: 1.0.20 - langsmith: 0.3.46(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76)) + langsmith: 0.3.46(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)) mustache: 4.2.0 p-queue: 6.6.2 p-retry: 4.6.2 @@ -13166,31 +13141,31 @@ snapshots: - '@opentelemetry/sdk-trace-base' - openai - '@langchain/google-genai@0.2.16(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76)))': + '@langchain/google-genai@0.2.16(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)))': dependencies: '@google/generative-ai': 0.24.1 - '@langchain/core': 0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76)) + '@langchain/core': 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)) uuid: 11.1.0 - '@langchain/mistralai@0.2.1(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76)))(zod@3.25.76)': + '@langchain/mistralai@0.2.1(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)))(zod@3.25.76)': dependencies: - '@langchain/core': 0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76)) + '@langchain/core': 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)) '@mistralai/mistralai': 1.7.4(zod@3.25.76) uuid: 10.0.0 transitivePeerDependencies: - zod - '@langchain/ollama@0.2.3(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76)))': + '@langchain/ollama@0.2.3(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)))': dependencies: - '@langchain/core': 0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76)) + '@langchain/core': 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)) ollama: 0.5.16 uuid: 10.0.0 - '@langchain/openai@0.6.6(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76)))(ws@8.18.3)': + '@langchain/openai@0.6.7(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)))(ws@8.18.3(utf-8-validate@6.0.5))': dependencies: - '@langchain/core': 0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76)) + '@langchain/core': 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)) js-tiktoken: 1.0.20 - openai: 5.12.1(ws@8.18.3)(zod@3.25.76) + openai: 5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76) zod: 3.25.76 transitivePeerDependencies: - ws @@ -14630,10 +14605,6 @@ snapshots: '@types/node-cleanup@2.1.5': {} - '@types/node-fetch@2.6.13': - dependencies: - '@types/node': 18.19.118 - form-data: 4.0.4 '@types/node-forge@1.3.13': dependencies: '@types/node': 18.19.118 @@ -14646,10 +14617,6 @@ snapshots: dependencies: undici-types: 5.26.5 - '@types/node@24.2.1': - dependencies: - undici-types: 7.10.0 - '@types/nodemailer@6.4.17': dependencies: '@types/node': 18.19.118 @@ -15557,13 +15524,6 @@ snapshots: node-releases: 2.0.19 update-browserslist-db: 1.1.3(browserslist@4.25.1) - browserslist@4.25.2: - dependencies: - caniuse-lite: 1.0.30001733 - electron-to-chromium: 1.5.199 - node-releases: 2.0.19 - update-browserslist-db: 1.1.3(browserslist@4.25.2) - bs-logger@0.2.6: dependencies: fast-json-stable-stringify: 2.1.0 @@ -15645,8 +15605,6 @@ snapshots: caniuse-lite@1.0.30001727: {} - caniuse-lite@1.0.30001733: {} - canvas-fit@1.5.0: dependencies: element-size: 1.1.1 @@ -16869,8 +16827,6 @@ snapshots: electron-to-chromium@1.5.182: {} - electron-to-chromium@1.5.199: {} - element-size@1.1.1: {} elementary-circuits-directed-graph@1.3.1: @@ -16937,7 +16893,7 @@ snapshots: - supports-color - utf-8-validate - enhanced-resolve@5.18.3: + enhanced-resolve@5.18.2: dependencies: graceful-fs: 4.2.11 tapable: 2.2.2 @@ -17529,8 +17485,6 @@ snapshots: cross-spawn: 7.0.6 signal-exit: 4.1.0 - form-data@4.0.3: - form-data-encoder@1.7.2: {} form-data@2.5.5: dependencies: asynckit: 0.4.0 @@ -17684,8 +17638,6 @@ snapshots: gl-matrix@3.4.3: {} - gl-matrix@3.4.4: {} - gl-text@1.4.0: dependencies: bit-twiddle: 1.0.2 @@ -19098,7 +19050,7 @@ snapshots: jest-worker@27.5.1: dependencies: - '@types/node': 24.2.1 + '@types/node': 18.19.118 merge-stream: 2.0.0 supports-color: 8.1.1 @@ -19373,7 +19325,7 @@ snapshots: langs@2.0.0: {} - langsmith@0.3.46(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76)): + langsmith@0.3.46(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)): dependencies: '@types/uuid': 10.0.0 chalk: 4.1.2 @@ -19384,7 +19336,7 @@ snapshots: uuid: 10.0.0 optionalDependencies: '@opentelemetry/api': 1.9.0 - openai: 5.12.1(ws@8.18.3)(zod@3.25.76) + openai: 5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76) launch-editor@2.10.0: dependencies: @@ -19618,7 +19570,7 @@ snapshots: csscolorparser: 1.0.3 earcut: 2.2.4 geojson-vt: 3.2.1 - gl-matrix: 3.4.4 + gl-matrix: 3.4.3 grid-index: 1.1.0 murmurhash-js: 1.0.0 pbf: 3.3.0 @@ -20198,7 +20150,7 @@ snapshots: is-inside-container: 1.0.0 is-wsl: 3.1.0 - openai@5.12.1(ws@8.18.3)(zod@3.25.76): + openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76): optionalDependencies: ws: 8.18.3(utf-8-validate@6.0.5) zod: 3.25.76 @@ -22774,8 +22726,6 @@ snapshots: undici-types@5.26.5: {} - undici-types@7.10.0: {} - unicorn-magic@0.3.0: {} unified@11.0.5: @@ -22848,12 +22798,6 @@ snapshots: escalade: 3.2.0 picocolors: 1.1.1 - update-browserslist-db@1.1.3(browserslist@4.25.2): - dependencies: - browserslist: 4.25.2 - escalade: 3.2.0 - picocolors: 1.1.1 - update-diff@1.1.0: {} uri-js@4.4.1: @@ -23179,9 +23123,9 @@ snapshots: '@webassemblyjs/wasm-parser': 1.14.1 acorn: 8.15.0 acorn-import-phases: 1.0.4(acorn@8.15.0) - browserslist: 4.25.2 + browserslist: 4.25.1 chrome-trace-event: 1.0.4 - enhanced-resolve: 5.18.3 + enhanced-resolve: 5.18.2 es-module-lexer: 1.7.0 eslint-scope: 5.1.1 events: 3.3.0 @@ -23211,9 +23155,9 @@ snapshots: '@webassemblyjs/wasm-parser': 1.14.1 acorn: 8.15.0 acorn-import-phases: 1.0.4(acorn@8.15.0) - browserslist: 4.25.2 + browserslist: 4.25.1 chrome-trace-event: 1.0.4 - enhanced-resolve: 5.18.3 + enhanced-resolve: 5.18.2 es-module-lexer: 1.7.0 eslint-scope: 5.1.1 events: 3.3.0 diff --git a/src/packages/server/conat/project/types.ts b/src/packages/server/conat/project/types.ts index 1c29cdab9f..15ae892e85 100644 --- a/src/packages/server/conat/project/types.ts +++ b/src/packages/server/conat/project/types.ts @@ -9,5 +9,5 @@ export interface Configuration { // pid limit pids?: number | string; // disk size - size?: number | string; + disk?: number | string; } From 179242f77fa5d8673d469e83a584f1b2acaaf1ea Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 12 Aug 2025 01:50:40 +0000 Subject: [PATCH 266/798] add todo for copyPath in next api --- src/packages/next/pages/api/v2/projects/copy-path.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/packages/next/pages/api/v2/projects/copy-path.ts b/src/packages/next/pages/api/v2/projects/copy-path.ts index 6d9390e1bb..d38a53b655 100644 --- a/src/packages/next/pages/api/v2/projects/copy-path.ts +++ b/src/packages/next/pages/api/v2/projects/copy-path.ts @@ -60,8 +60,10 @@ export default async function handle(req, res) { throw Error("must be a collaborator on source project"); } } + throw Error("TODO: reimplement copyPath"); const project = getProject(src_project_id); - await project.copyPath({ + console.log({ + project, path, target_project_id, target_path, From f4e7a14fdd814f8490a36f6fd167a08d9b777026 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 12 Aug 2025 01:58:02 +0000 Subject: [PATCH 267/798] ts issue --- src/packages/backend/sandbox/exec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/packages/backend/sandbox/exec.ts b/src/packages/backend/sandbox/exec.ts index a966474db9..062292dc78 100644 --- a/src/packages/backend/sandbox/exec.ts +++ b/src/packages/backend/sandbox/exec.ts @@ -96,7 +96,7 @@ export default async function exec({ // console.log(`${cmd} ${args.join(" ")}`, { cwd }); const child = spawn(cmd, args, { stdio: ["ignore", "pipe", "pipe"], - env: {}, + env: {} as any, // sometimes next complains about this cwd, ...userId, }); From 582110afadebf286a09efbf4bfc2e7f78ad39856 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 12 Aug 2025 02:07:16 +0000 Subject: [PATCH 268/798] clearly deprecate copy-path in hub --- src/packages/hub/copy-path.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/packages/hub/copy-path.ts b/src/packages/hub/copy-path.ts index 67c0f3b7a0..fe92300beb 100644 --- a/src/packages/hub/copy-path.ts +++ b/src/packages/hub/copy-path.ts @@ -3,6 +3,7 @@ * License: MS-RSL – see LICENSE.md for details */ +// DEPRECATED // Copy Operations Provider // Used in the "Client" @@ -148,6 +149,8 @@ export class CopyPath { const project = projectControl(mesg.src_project_id); // do the copy + throw Error("DEPRECATED"); + // @ts-ignore const copy_id = await project.copyPath({ path: mesg.src_path, target_project_id: mesg.target_project_id, From d8bc9605483f3edeca7abf8cfb82644bc8d289fb Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 12 Aug 2025 04:00:13 +0000 Subject: [PATCH 269/798] ... --- src/scripts/g.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/src/scripts/g.sh b/src/scripts/g.sh index 68c84d070c..ca41f3058b 100755 --- a/src/scripts/g.sh +++ b/src/scripts/g.sh @@ -3,7 +3,6 @@ mkdir -p `pwd`/logs export LOGS=`pwd`/logs rm -f $LOGS/log unset INIT_CWD -unset PGHOST export DEBUG="cocalc:*,-cocalc:silly:*" export DEBUG_CONSOLE="no" From f551b1d92cb531e02c665a28e02c837079627a88 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 12 Aug 2025 04:33:33 +0000 Subject: [PATCH 270/798] remove no-longer-used dep --- src/packages/pnpm-lock.yaml | 15 +++------------ src/packages/server/package.json | 8 +++++--- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 2d91207588..21c7e107b4 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -1258,9 +1258,6 @@ importers: async: specifier: ^1.5.2 version: 1.5.2 - await-spawn: - specifier: ^4.0.2 - version: 4.0.2 awaiting: specifier: ^3.0.0 version: 3.0.0 @@ -4955,10 +4952,6 @@ packages: resolution: {integrity: sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==} engines: {node: '>= 0.4'} - await-spawn@4.0.2: - resolution: {integrity: sha512-GdADmeLJiMvGKJD3xWBcX40DMn07JNH1sqJYgYJZH7NTGJ3B1qDjKBKzxhhyR1hjIcnUGFUmE/+4D1HcHAJBAA==} - engines: {node: '>=10'} - awaiting@3.0.0: resolution: {integrity: sha512-19i4G7Hjxj9idgMlAM0BTRII8HfvsOdlr4D9cf3Dm1MZhvcKjBpzY8AMNEyIKyi+L9TIK15xZatmdcPG003yww==} engines: {node: '>=7.6.x'} @@ -15320,15 +15313,11 @@ snapshots: dependencies: possible-typed-array-names: 1.1.0 - await-spawn@4.0.2: - dependencies: - bl: 4.1.0 - awaiting@3.0.0: {} axios@1.11.0: dependencies: - follow-redirects: 1.15.9(debug@4.4.1) + follow-redirects: 1.15.9 form-data: 4.0.4 proxy-from-env: 1.1.0 transitivePeerDependencies: @@ -17464,6 +17453,8 @@ snapshots: dependencies: dtype: 2.0.0 + follow-redirects@1.15.9: {} + follow-redirects@1.15.9(debug@4.4.1): optionalDependencies: debug: 4.4.1 diff --git a/src/packages/server/package.json b/src/packages/server/package.json index 79d7f3e63a..0f982c8d81 100644 --- a/src/packages/server/package.json +++ b/src/packages/server/package.json @@ -28,7 +28,10 @@ "./settings": "./dist/settings/index.js", "./settings/*": "./dist/settings/*.js" }, - "keywords": ["utilities", "cocalc"], + "keywords": [ + "utilities", + "cocalc" + ], "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf node_modules dist", @@ -44,8 +47,8 @@ "@cocalc/backend": "workspace:*", "@cocalc/conat": "workspace:*", "@cocalc/database": "workspace:*", - "@cocalc/gcloud-pricing-calculator": "^1.17.0", "@cocalc/file-server": "workspace:*", + "@cocalc/gcloud-pricing-calculator": "^1.17.0", "@cocalc/server": "workspace:*", "@cocalc/util": "workspace:*", "@google-cloud/bigquery": "^7.8.0", @@ -71,7 +74,6 @@ "@zxcvbn-ts/language-common": "^3.0.4", "@zxcvbn-ts/language-en": "^3.0.2", "async": "^1.5.2", - "await-spawn": "^4.0.2", "awaiting": "^3.0.0", "axios": "^1.11.0", "base62": "^2.0.1", From 959da5f221f1ee760b736d462d40d44bbcf11639 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 12 Aug 2025 13:48:30 +0000 Subject: [PATCH 271/798] fix most failing tests (caused by a number of recent refactors) --- .../backend/conat/files/local-path.ts | 39 ++++++++++++------- src/packages/backend/sandbox/index.ts | 2 +- src/packages/conat/files/fs.ts | 2 +- src/packages/file-server/btrfs/filesystem.ts | 21 +++++----- .../file-server/btrfs/subvolume-rustic.ts | 15 +++++-- src/packages/file-server/btrfs/subvolume.ts | 2 +- src/packages/file-server/btrfs/test/setup.ts | 1 + .../server/conat/file-server/index.ts | 3 +- src/packages/tsconfig.json | 12 +++--- 9 files changed, 60 insertions(+), 37 deletions(-) diff --git a/src/packages/backend/conat/files/local-path.ts b/src/packages/backend/conat/files/local-path.ts index 17857d27fb..f432001403 100644 --- a/src/packages/backend/conat/files/local-path.ts +++ b/src/packages/backend/conat/files/local-path.ts @@ -1,6 +1,8 @@ import { fsServer, DEFAULT_FILE_SERVICE } from "@cocalc/conat/files/fs"; import { SandboxedFilesystem } from "@cocalc/backend/sandbox"; import { isValidUUID } from "@cocalc/util/misc"; +import { mkdir } from "fs/promises"; +import { join } from "path"; import { type Client } from "@cocalc/conat/core/client"; import { conat } from "@cocalc/backend/conat/conat"; import { client as createFileClient } from "@cocalc/conat/files/file-server"; @@ -14,30 +16,41 @@ export async function localPathFileserver({ }: { service?: string; client?: Client; - // if project_id is specified, use single project mode. + // if project_id is specified, only serve this one project_id project_id?: string; - // only used in single project mode + // - if path is given, serve projects from `${path}/${project_id}` + // - if path not given, connect to the file-server service on the conat network. path?: string; unsafeMode?: boolean; } = {}) { client ??= conat(); + const getPath = async (project_id2: string) => { + if (project_id != null && project_id != project_id2) { + throw Error(`only serves ${project_id}`); + } + if (path != null) { + const p = join(path, project_id2); + try { + await mkdir(p); + } catch {} + return p; + } else { + const fsclient = createFileClient({ client }); + return (await fsclient.mount({ project_id: project_id2 })).path; + } + }; + const server = await fsServer({ service, client, project_id, fs: async (subject: string) => { - if (project_id) { - if (path == null) { - throw Error("path must be specified"); - } - return new SandboxedFilesystem(path, { unsafeMode }); - } else { - const project_id = getProjectId(subject); - const fsclient = createFileClient({ client }); - const { path } = await fsclient.mount({ project_id }); - return new SandboxedFilesystem(path, { unsafeMode, host: project_id }); - } + const project_id = getProjectId(subject); + return new SandboxedFilesystem(await getPath(project_id), { + unsafeMode, + host: project_id, + }); }, }); return { server, client, path, service, close: () => server.close() }; diff --git a/src/packages/backend/sandbox/index.ts b/src/packages/backend/sandbox/index.ts index 6a87e40514..72e84b7a31 100644 --- a/src/packages/backend/sandbox/index.ts +++ b/src/packages/backend/sandbox/index.ts @@ -296,7 +296,7 @@ export class SandboxedFilesystem { args: string[], { timeout = 120_000, - maxSize = 10_000, + maxSize = 10_000_000, // the json output can be quite large cwd, }: { timeout?: number; maxSize?: number; cwd?: string } = {}, ): Promise => { diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index 4e0bf068f8..0a71e6bb26 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -302,7 +302,7 @@ interface Options { fs: (subject?: string) => Promise; // project-id: if given, ONLY serve files for this one project, and the // path must be the home of the project - // If not given, + // If not given, serves files for all projects. project_id?: string; } diff --git a/src/packages/file-server/btrfs/filesystem.ts b/src/packages/file-server/btrfs/filesystem.ts index c1e4947bd9..7701a93afc 100644 --- a/src/packages/file-server/btrfs/filesystem.ts +++ b/src/packages/file-server/btrfs/filesystem.ts @@ -36,13 +36,11 @@ export interface Options { // configure rustic however you want by pointing this at a toml cofig file. // Otherwise, if this path does not exist, it will be created a new rustic repo // initialized here. - // If not given, then backups will throw an error. - rustic?: string; + rustic: string; } export class Filesystem { public readonly opts: Options; - public readonly rustic: string; public readonly subvolumes: Subvolumes; constructor(opts: Options) { @@ -57,9 +55,7 @@ export class Filesystem { await btrfs({ args: ["quota", "enable", "--simple", this.opts.mount], }); - if (this.opts.rustic) { - await this.initRustic(); - } + await this.initRustic(); await this.sync(); }; @@ -157,14 +153,17 @@ export class Filesystem { }; private initRustic = async () => { - if (!this.rustic || (await exists(this.rustic))) { + if (!this.opts.rustic) { + throw Error("rustic repo path or toml must be specified"); + } + if (!this.opts.rustic || (await exists(this.opts.rustic))) { return; } - if (this.rustic.endsWith(".toml")) { - throw Error(`file not found: ${this.rustic}`); + if (this.opts.rustic.endsWith(".toml")) { + throw Error(`file not found: ${this.opts.rustic}`); } - await mkdir(this.rustic); - await rustic(["init"], { repo: this.rustic }); + await mkdir(this.opts.rustic); + await rustic(["init"], { repo: this.opts.rustic }); }; } diff --git a/src/packages/file-server/btrfs/subvolume-rustic.ts b/src/packages/file-server/btrfs/subvolume-rustic.ts index 040c91cc90..32f0319242 100644 --- a/src/packages/file-server/btrfs/subvolume-rustic.ts +++ b/src/packages/file-server/btrfs/subvolume-rustic.ts @@ -21,6 +21,7 @@ Instead of using btrfs send/recv for backups, we use Rustic because: import { type Subvolume } from "./subvolume"; import getLogger from "@cocalc/backend/logger"; import { parseOutput } from "@cocalc/backend/sandbox/exec"; +import { field_cmp } from "@cocalc/util/misc"; export const RUSTIC = "rustic"; @@ -86,24 +87,32 @@ export class SubvolumeRustic { return stdout; }; + // returns list of snapshots sorted from oldest to newest snapshots = async (): Promise => { const { stdout } = parseOutput( await this.subvolume.fs.rustic(["snapshots", "--json"]), ); const x = JSON.parse(stdout); - return x[0][1].map(({ time, id }) => { + const v = x[0][1].map(({ time, id }) => { return { time: new Date(time), id }; }); + v.sort(field_cmp("time")); + return v; }; + // return list of paths of files in this backup, as paths relative + // to HOME, and sorted in alphabetical order. ls = async ({ id }: { id: string }) => { const { stdout } = parseOutput( await this.subvolume.fs.rustic(["ls", "--json", id]), ); - return JSON.parse(stdout); + return JSON.parse(stdout).sort(); }; - // (this doesn't actually clean up disk space -- purge must be done separately) + // Delete this backup. It's genuinely not accessible anymore, though + // this doesn't actually clean up disk space -- purge must be done separately + // later. Rustic likes the purge to happen maybe a day later, so it + // can better support concurrent writes. forget = async ({ id }: { id: string }) => { const { stdout } = parseOutput( await this.subvolume.fs.rustic(["forget", id]), diff --git a/src/packages/file-server/btrfs/subvolume.ts b/src/packages/file-server/btrfs/subvolume.ts index 75b4836539..b3b061aed2 100644 --- a/src/packages/file-server/btrfs/subvolume.ts +++ b/src/packages/file-server/btrfs/subvolume.ts @@ -35,7 +35,7 @@ export class Subvolume { this.name = name; this.path = join(filesystem.opts.mount, name); this.fs = new SandboxedFilesystem(this.path, { - rusticRepo: filesystem.rustic, + rusticRepo: filesystem.opts.rustic, host: this.name, }); this.rustic = new SubvolumeRustic(this); diff --git a/src/packages/file-server/btrfs/test/setup.ts b/src/packages/file-server/btrfs/test/setup.ts index b6acd1bf6a..a5e98e8fdf 100644 --- a/src/packages/file-server/btrfs/test/setup.ts +++ b/src/packages/file-server/btrfs/test/setup.ts @@ -52,6 +52,7 @@ export async function before() { image: join(tempDir, "btrfs.img"), size: "1G", mount: join(tempDir, "mnt"), + rustic: join(tempDir, "rustic"), }); } diff --git a/src/packages/server/conat/file-server/index.ts b/src/packages/server/conat/file-server/index.ts index 3ef3628efb..564041334a 100644 --- a/src/packages/server/conat/file-server/index.ts +++ b/src/packages/server/conat/file-server/index.ts @@ -21,7 +21,7 @@ export type { Fileserver }; import { loadConatConfiguration } from "../configuration"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import getLogger from "@cocalc/backend/logger"; -import { data } from "@cocalc/backend/data"; +import { data, rusticRepo } from "@cocalc/backend/data"; import { join } from "node:path"; import { mkdir } from "fs/promises"; import { filesystem, type Filesystem } from "@cocalc/file-server/btrfs"; @@ -154,6 +154,7 @@ export async function init() { image: join(image, "btrfs.img"), size: "25G", mount: mountPoint, + rustic: rusticRepo, }); server = await createFileServer({ diff --git a/src/packages/tsconfig.json b/src/packages/tsconfig.json index 70ea25757d..6edbb90e35 100644 --- a/src/packages/tsconfig.json +++ b/src/packages/tsconfig.json @@ -20,11 +20,11 @@ "strictNullChecks": true, "target": "es2020", "module": "commonjs" + }, + "watchOptions": { + "fallbackPolling": "dynamicPriority", + "synchronousWatchDirectory": true, + "watchDirectory": "useFsEvents", + "watchFile": "useFsEvents" } - // "watchOptions": { - // "fallbackPolling": "dynamicPriority", - // "synchronousWatchDirectory": true, - // "watchDirectory": "useFsEvents", - // "watchFile": "useFsEvents" - // } } From 68d003726accb8680b2fecf889facb2b16bdaa32 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 12 Aug 2025 13:52:39 +0000 Subject: [PATCH 272/798] address race condition with running projects --- src/packages/conat/project/runner/run.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/packages/conat/project/runner/run.ts b/src/packages/conat/project/runner/run.ts index 9a40cc4412..36d20cf971 100644 --- a/src/packages/conat/project/runner/run.ts +++ b/src/packages/conat/project/runner/run.ts @@ -90,5 +90,5 @@ export function client({ subject: string; }): API { client ??= conat(); - return client.call(subject); + return client.call(subject, { waitForInterest: true }); } From 768c004b97b7cebbeb24c64884f474047ca9f1cd Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 12 Aug 2025 14:55:55 +0000 Subject: [PATCH 273/798] incorrect merge fixed --- .../compute/maintenance/purchases/manage-purchases.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/packages/server/compute/maintenance/purchases/manage-purchases.test.ts b/src/packages/server/compute/maintenance/purchases/manage-purchases.test.ts index 4f90c06669..5259ea5bc8 100644 --- a/src/packages/server/compute/maintenance/purchases/manage-purchases.test.ts +++ b/src/packages/server/compute/maintenance/purchases/manage-purchases.test.ts @@ -267,7 +267,7 @@ describe("confirm managing of purchases works", () => { // rule 6 it("make time long so that balance is exceeded (but not by too much), and see that server gets stopped due to too low balance, and an email is sent to the user", async () => { resetTestMessages(); - await setPurchaseStart(new Date(Date.now() - 1000 * 60 * 60 * 24 * 10)); + await setPurchaseStart(new Date(Date.now() - 1000 * 60 * 60 * 24 * 7)); const pool = getPool(); await pool.query( "UPDATE compute_servers SET state='running', update_purchase=TRUE WHERE id=$1", From 88816855508f58e4afa393ba6520c37863d1689e Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 12 Aug 2025 18:42:44 +0000 Subject: [PATCH 274/798] organizing testing of @cocalc/server so works with conat without using an external server --- src/packages/backend/conat/test/setup.ts | 2 + src/packages/file-server/btrfs/test/setup.ts | 1 + src/packages/server/compute/control.test.ts | 14 +++-- .../server/compute/create-server.test.ts | 11 ++-- .../server/compute/database-cache.test.ts | 11 ++-- .../server/compute/get-servers.test.ts | 11 ++-- .../maintenance/purchases/close.test.ts | 11 ++-- .../purchases/manage-purchases.test.ts | 13 ++--- .../purchases/ongoing-purchases.test.ts | 10 ++-- .../maintenance/purchases/util.test.ts | 10 ++-- .../server/conat/file-server/index.ts | 16 +++--- src/packages/server/conat/index.ts | 12 +---- .../server/conat/project/load-balancer.ts | 5 +- .../server/licenses/add-to-project.test.ts | 21 +++----- src/packages/server/llm/test/models.test.ts | 8 ++- src/packages/server/package.json | 9 ++-- .../control/stop-idle-projects.test.ts | 12 ++--- .../server/purchases/closing-date.test.ts | 11 ++-- .../server/purchases/edit-license.test.ts | 15 ++---- .../server/purchases/get-balance.test.ts | 11 ++-- .../server/purchases/get-min-balance.test.ts | 11 ++-- .../server/purchases/get-purchases.test.ts | 11 ++-- .../server/purchases/get-service-cost.test.ts | 15 ++---- .../server/purchases/get-spend-rate.test.ts | 11 ++-- .../purchases/is-purchase-allowed.test.ts | 11 ++-- .../maintain-automatic-payments.test.ts | 11 ++-- .../purchases/maintain-subscriptions.test.ts | 12 ++--- .../purchase-shopping-cart-item.test.ts | 15 ++---- .../purchases/renew-subscription.test.ts | 11 ++-- .../purchases/resume-subscription.test.ts | 52 ++----------------- .../purchases/shift-subscription.2.test.ts | 20 +++---- .../statements/create-statements.test.ts | 11 ++-- .../statements/email-statement.test.ts | 11 ++-- .../server/purchases/student-pay.test.ts | 13 ++--- src/packages/server/test/index.ts | 42 +++++++++++++++ src/packages/server/test/setup.js | 2 + 36 files changed, 168 insertions(+), 305 deletions(-) create mode 100644 src/packages/server/test/index.ts diff --git a/src/packages/backend/conat/test/setup.ts b/src/packages/backend/conat/test/setup.ts index 151eff2227..6494027a44 100644 --- a/src/packages/backend/conat/test/setup.ts +++ b/src/packages/backend/conat/test/setup.ts @@ -21,6 +21,7 @@ import { once } from "@cocalc/util/async-utils"; import { until } from "@cocalc/util/async-utils"; import { randomId } from "@cocalc/conat/names"; import { isEqual } from "lodash"; +import { setConatServer } from "@cocalc/backend/data"; export { wait, delay, once }; @@ -150,6 +151,7 @@ export async function before( } server = await createServer(); + setConatServer(server.address()); client = connect(); persistServer = createPersistServer({ client }); setConatClient({ diff --git a/src/packages/file-server/btrfs/test/setup.ts b/src/packages/file-server/btrfs/test/setup.ts index a5e98e8fdf..af4e8a0d21 100644 --- a/src/packages/file-server/btrfs/test/setup.ts +++ b/src/packages/file-server/btrfs/test/setup.ts @@ -54,6 +54,7 @@ export async function before() { mount: join(tempDir, "mnt"), rustic: join(tempDir, "rustic"), }); + return fs; } export async function after() { diff --git a/src/packages/server/compute/control.test.ts b/src/packages/server/compute/control.test.ts index 82449e24e0..08291f266f 100644 --- a/src/packages/server/compute/control.test.ts +++ b/src/packages/server/compute/control.test.ts @@ -1,4 +1,3 @@ -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import { uuid } from "@cocalc/util/misc"; import createAccount from "@cocalc/server/accounts/create-account"; import createProject from "@cocalc/server/projects/create"; @@ -6,13 +5,9 @@ import createServer from "./create-server"; import * as control from "./control"; import { delay } from "awaiting"; -beforeAll(async () => { - await initEphemeralDatabase(); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); +import { before, after } from "@cocalc/server/test"; +beforeAll(before, 15000); +afterAll(after); describe("creates account, project and a test compute server, then control it", () => { const account_id = uuid(); @@ -26,6 +21,9 @@ describe("creates account, project and a test compute server, then control it", lastName: "One", account_id, }); + }); + + it("create project", async () => { // Only User One: project_id = await createProject({ account_id, diff --git a/src/packages/server/compute/create-server.test.ts b/src/packages/server/compute/create-server.test.ts index 86870f8605..6f7acf44fc 100644 --- a/src/packages/server/compute/create-server.test.ts +++ b/src/packages/server/compute/create-server.test.ts @@ -1,18 +1,13 @@ -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import getServers from "./get-servers"; import { uuid } from "@cocalc/util/misc"; import createAccount from "@cocalc/server/accounts/create-account"; import createProject from "@cocalc/server/projects/create"; import createServer from "./create-server"; import { CLOUDS_BY_NAME } from "@cocalc/util/db-schema/compute-servers"; +import { before, after } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase(); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); +beforeAll(before, 15000); +afterAll(after); describe("creates account, project and then compute servers in various ways", () => { const account_id = uuid(); diff --git a/src/packages/server/compute/database-cache.test.ts b/src/packages/server/compute/database-cache.test.ts index 6202b04a2d..7e551d30d4 100644 --- a/src/packages/server/compute/database-cache.test.ts +++ b/src/packages/server/compute/database-cache.test.ts @@ -1,14 +1,9 @@ -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import { createDatabaseCachedResource, createTTLCache } from "./database-cache"; import { delay } from "awaiting"; +import { before, after } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase(); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); +beforeAll(before, 15000); +afterAll(after); // keep short so that unit testing is fast... but long enough // that things don't break on github actions. diff --git a/src/packages/server/compute/get-servers.test.ts b/src/packages/server/compute/get-servers.test.ts index a4b60a217c..2bfcbdf78c 100644 --- a/src/packages/server/compute/get-servers.test.ts +++ b/src/packages/server/compute/get-servers.test.ts @@ -1,18 +1,13 @@ -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import getServers, { getServer } from "./get-servers"; import { uuid } from "@cocalc/util/misc"; import createAccount from "@cocalc/server/accounts/create-account"; import createProject from "@cocalc/server/projects/create"; import addUserToProject from "@cocalc/server/projects/add-user-to-project"; import createServer from "./create-server"; +import { before, after } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase(); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); +beforeAll(before, 15000); +afterAll(after); describe("calls get compute servers with various inputs with a new account with no data (everything should return [])", () => { it("throws 'account_id is not a valid uuid' if account_id not specified", async () => { diff --git a/src/packages/server/compute/maintenance/purchases/close.test.ts b/src/packages/server/compute/maintenance/purchases/close.test.ts index b413b1b852..98f810f4d7 100644 --- a/src/packages/server/compute/maintenance/purchases/close.test.ts +++ b/src/packages/server/compute/maintenance/purchases/close.test.ts @@ -2,7 +2,6 @@ Test functions for closing purchases in various ways. */ -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import createAccount from "@cocalc/server/accounts/create-account"; import { setTestNetworkUsage } from "@cocalc/server/compute/control"; import createServer from "@cocalc/server/compute/create-server"; @@ -17,14 +16,10 @@ import { closePurchase, } from "./close"; import { getPurchase } from "./util"; +import { before, after } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase(); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); +beforeAll(before, 15000); +afterAll(after); describe("creates account, project, test compute server, and purchase, then close the purchase, and confirm it worked properly", () => { const account_id = uuid(); diff --git a/src/packages/server/compute/maintenance/purchases/manage-purchases.test.ts b/src/packages/server/compute/maintenance/purchases/manage-purchases.test.ts index 5259ea5bc8..905f2d5a25 100644 --- a/src/packages/server/compute/maintenance/purchases/manage-purchases.test.ts +++ b/src/packages/server/compute/maintenance/purchases/manage-purchases.test.ts @@ -2,7 +2,6 @@ Test managing purchases */ -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import createAccount from "@cocalc/server/accounts/create-account"; import { setTestNetworkUsage } from "@cocalc/server/compute/control"; import createServer from "@cocalc/server/compute/create-server"; @@ -21,18 +20,16 @@ import managePurchases, { outstandingPurchases, } from "./manage-purchases"; import { getPurchase } from "./util"; +import { getPool, before, after, initEphemeralDatabase } from "@cocalc/server/test"; + +beforeAll(before, 15000); +afterAll(after); + // we put a small delay in some cases due to using a database query pool. // This might need to be adjusted for CI infrastructure. const DELAY = 250; -beforeAll(async () => { - await initEphemeralDatabase(); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); describe("confirm managing of purchases works", () => { const account_id = uuid(); diff --git a/src/packages/server/compute/maintenance/purchases/ongoing-purchases.test.ts b/src/packages/server/compute/maintenance/purchases/ongoing-purchases.test.ts index 360cb22a0e..628fc9ad73 100644 --- a/src/packages/server/compute/maintenance/purchases/ongoing-purchases.test.ts +++ b/src/packages/server/compute/maintenance/purchases/ongoing-purchases.test.ts @@ -8,7 +8,6 @@ in the database, in order to run the test. */ import ongoingPurchases from "./ongoing-purchases"; -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import { uuid } from "@cocalc/util/misc"; import createAccount from "@cocalc/server/accounts/create-account"; import createProject from "@cocalc/server/projects/create"; @@ -20,14 +19,11 @@ import { MAX_PURCHASE_LENGTH_MS, } from "./manage-purchases"; import createPurchase from "@cocalc/server/purchases/create-purchase"; +import { getPool, before, after } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase(); -}, 15000); +beforeAll(before, 15000); +afterAll(after); -afterAll(async () => { - await getPool().end(); -}); async function getUpdatePurchase(id: number) { const pool = getPool(); diff --git a/src/packages/server/compute/maintenance/purchases/util.test.ts b/src/packages/server/compute/maintenance/purchases/util.test.ts index f054579035..098af47223 100644 --- a/src/packages/server/compute/maintenance/purchases/util.test.ts +++ b/src/packages/server/compute/maintenance/purchases/util.test.ts @@ -1,18 +1,14 @@ -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import { uuid } from "@cocalc/util/misc"; import createAccount from "@cocalc/server/accounts/create-account"; import createProject from "@cocalc/server/projects/create"; import createServer from "@cocalc/server/compute/create-server"; import { getServer } from "@cocalc/server/compute/get-servers"; import { setPurchaseId } from "./util"; +import { before, after } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase(); -}, 15000); +beforeAll(before, 15000); +afterAll(after); -afterAll(async () => { - await getPool().end(); -}); describe("creates compute server then sets the purchase id and confirms it", () => { const account_id = uuid(); diff --git a/src/packages/server/conat/file-server/index.ts b/src/packages/server/conat/file-server/index.ts index 564041334a..58d9a55669 100644 --- a/src/packages/server/conat/file-server/index.ts +++ b/src/packages/server/conat/file-server/index.ts @@ -136,7 +136,7 @@ async function cp({ let fs: Filesystem | null = null; let server: any = null; -export async function init() { +export async function init(_fs?) { if (server != null) { return; } @@ -150,12 +150,14 @@ export async function init() { await mkdir(mountPoint, { recursive: true }); } - fs = await filesystem({ - image: join(image, "btrfs.img"), - size: "25G", - mount: mountPoint, - rustic: rusticRepo, - }); + fs = + _fs ?? + (await filesystem({ + image: join(image, "btrfs.img"), + size: "25G", + mount: mountPoint, + rustic: rusticRepo, + })); server = await createFileServer({ client: conat(), diff --git a/src/packages/server/conat/index.ts b/src/packages/server/conat/index.ts index 48d8874d04..a5207d149b 100644 --- a/src/packages/server/conat/index.ts +++ b/src/packages/server/conat/index.ts @@ -5,11 +5,7 @@ import { init as initLLM } from "./llm"; import { loadConatConfiguration } from "./configuration"; import { createTimeService } from "@cocalc/conat/service/time"; export { initConatPersist } from "./persist"; -import { - conatApiCount, - projects, - conatProjectRunnerCount, -} from "@cocalc/backend/data"; +import { conatApiCount, conatProjectRunnerCount } from "@cocalc/backend/data"; import { localPathFileserver } from "@cocalc/backend/conat/files/local-path"; import { init as initProjectRunner } from "./project/run"; import { init as initProjectRunnerLoadBalancer } from "./project/load-balancer"; @@ -45,12 +41,6 @@ export async function initConatApi() { export async function initConatFileserver() { await loadConatConfiguration(); - const i = projects.indexOf("/[project_id]"); - if (i == -1) { - throw Error( - `projects must be a template containing /[project_id] -- ${projects}`, - ); - } logger.debug("initFileserver"); localPathFileserver(); initFileserver(); diff --git a/src/packages/server/conat/project/load-balancer.ts b/src/packages/server/conat/project/load-balancer.ts index 6c2dbed152..b6ea6126c0 100644 --- a/src/packages/server/conat/project/load-balancer.ts +++ b/src/packages/server/conat/project/load-balancer.ts @@ -34,10 +34,7 @@ async function getConfig({ project_id }) { "SELECT settings FROM projects WHERE project_id=$1", [project_id], ); - if (rows.length == 0) { - throw Error(`no project with id ${project_id}`); - } - if (rows[0].settings?.admin) { + if (rows[0]?.settings?.admin) { return { admin: true, disk: "25G" }; } else { // some defaults, mainly for testing diff --git a/src/packages/server/licenses/add-to-project.test.ts b/src/packages/server/licenses/add-to-project.test.ts index 1c19b76764..756c3f2945 100644 --- a/src/packages/server/licenses/add-to-project.test.ts +++ b/src/packages/server/licenses/add-to-project.test.ts @@ -1,15 +1,10 @@ import { uuid } from "@cocalc/util/misc"; -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import createProject from "@cocalc/server/projects/create"; import addLicenseToProject from "./add-to-project"; +import { before, after, getPool } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase({}); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); +beforeAll(before, 15000); +afterAll(after); describe("test various cases of adding a license to a project", () => { let project_id = uuid(); @@ -29,7 +24,7 @@ describe("test various cases of adding a license to a project", () => { const pool = getPool(); const { rows } = await pool.query( "SELECT site_license FROM projects WHERE project_id=$1", - [project_id] + [project_id], ); expect(rows[0].site_license).toEqual({ [license_id]: {} }); }); @@ -39,7 +34,7 @@ describe("test various cases of adding a license to a project", () => { const pool = getPool(); const { rows } = await pool.query( "SELECT site_license FROM projects WHERE project_id=$1", - [project_id] + [project_id], ); expect(rows[0].site_license).toEqual({ [license_id]: {} }); }); @@ -50,7 +45,7 @@ describe("test various cases of adding a license to a project", () => { const pool = getPool(); const { rows } = await pool.query( "SELECT site_license FROM projects WHERE project_id=$1", - [project_id] + [project_id], ); expect(rows[0].site_license).toEqual({ [license_id]: {}, @@ -62,12 +57,12 @@ describe("test various cases of adding a license to a project", () => { const pool = getPool(); await pool.query( "UPDATE projects SET site_license='{}' WHERE project_id=$1", - [project_id] + [project_id], ); await addLicenseToProject({ project_id, license_id }); const { rows } = await pool.query( "SELECT site_license FROM projects WHERE project_id=$1", - [project_id] + [project_id], ); expect(rows[0].site_license).toEqual({ [license_id]: {} }); }); diff --git a/src/packages/server/llm/test/models.test.ts b/src/packages/server/llm/test/models.test.ts index 2128534142..37f91c48eb 100644 --- a/src/packages/server/llm/test/models.test.ts +++ b/src/packages/server/llm/test/models.test.ts @@ -1,6 +1,5 @@ // import { log } from "console"; -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import { AnthropicModel, LanguageModelCore, @@ -28,19 +27,18 @@ import { evaluateMistral } from "../mistral"; import { evaluateOpenAILC } from "../openai-lc"; import { evaluateUserDefinedLLM } from "../user-defined"; import { enableModels, setupAPIKeys, test_llm } from "./shared"; +import { before, after, getPool } from "@cocalc/server/test"; // sometimes (flaky case) they take more than 10s to even start a response const LLM_TIMEOUT = 15_000; beforeAll(async () => { - await initEphemeralDatabase(); + await before(); await setupAPIKeys(); await enableModels(); }, 15000); -afterAll(async () => { - await getPool().end(); -}); +afterAll(after); const QUERY = { input: "What's 99 + 1?", diff --git a/src/packages/server/package.json b/src/packages/server/package.json index 0f982c8d81..7ea93a000b 100644 --- a/src/packages/server/package.json +++ b/src/packages/server/package.json @@ -26,12 +26,11 @@ "./projects/connection": "./dist/projects/connection/index.js", "./projects/*": "./dist/projects/*.js", "./settings": "./dist/settings/index.js", - "./settings/*": "./dist/settings/*.js" + "./settings/*": "./dist/settings/*.js", + "./test": "./dist/test/index.js", + "./test/*": "./dist/test/*.js" }, - "keywords": [ - "utilities", - "cocalc" - ], + "keywords": ["utilities", "cocalc"], "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf node_modules dist", diff --git a/src/packages/server/projects/control/stop-idle-projects.test.ts b/src/packages/server/projects/control/stop-idle-projects.test.ts index 69926aa51f..26f598d0cb 100644 --- a/src/packages/server/projects/control/stop-idle-projects.test.ts +++ b/src/packages/server/projects/control/stop-idle-projects.test.ts @@ -4,19 +4,15 @@ */ import createProject from "@cocalc/server/projects/create"; -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; +import getPool from "@cocalc/database/pool"; import { isValidUUID } from "@cocalc/util/misc"; import { test } from "./stop-idle-projects"; const { stopIdleProjects } = test; import { delay } from "awaiting"; +import { before, after } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase(); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); +beforeAll(before, 15000); +afterAll(after); describe("creates a project, set various parameters, and runs idle project function, it and confirm that things work as intended", () => { let project_id; diff --git a/src/packages/server/purchases/closing-date.test.ts b/src/packages/server/purchases/closing-date.test.ts index cb88f3e42f..68be8122a6 100644 --- a/src/packages/server/purchases/closing-date.test.ts +++ b/src/packages/server/purchases/closing-date.test.ts @@ -10,16 +10,11 @@ import { setClosingDay, } from "./closing-date"; import { uuid } from "@cocalc/util/misc"; -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import createAccount from "@cocalc/server/accounts/create-account"; +import { before, after } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase(); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); +beforeAll(before, 15000); +afterAll(after); describe("basic consistency checks for closing date functions", () => { const account_id = uuid(); diff --git a/src/packages/server/purchases/edit-license.test.ts b/src/packages/server/purchases/edit-license.test.ts index 4e1a9183a7..dfd4fb6799 100644 --- a/src/packages/server/purchases/edit-license.test.ts +++ b/src/packages/server/purchases/edit-license.test.ts @@ -5,10 +5,7 @@ import dayjs from "dayjs"; -import getPool, { - getPoolClient, - initEphemeralDatabase, -} from "@cocalc/database/pool"; +import { getPoolClient } from "@cocalc/database/pool"; import createAccount from "@cocalc/server/accounts/create-account"; import getLicense from "@cocalc/server/licenses/get-license"; import createLicense from "@cocalc/server/licenses/purchase/create-license"; @@ -21,14 +18,10 @@ import editLicenseOwner from "./edit-license-owner"; import getSubscriptions from "./get-subscriptions"; import purchaseShoppingCartItem from "./purchase-shopping-cart-item"; import { license0 } from "./test-data"; +import { before, after, getPool } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase(); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); +beforeAll(before, 15000); +afterAll(after); describe("create a license and then edit it in various ways", () => { const account_id = uuid(); diff --git a/src/packages/server/purchases/get-balance.test.ts b/src/packages/server/purchases/get-balance.test.ts index a21de53a5d..a67a7dcc23 100644 --- a/src/packages/server/purchases/get-balance.test.ts +++ b/src/packages/server/purchases/get-balance.test.ts @@ -6,16 +6,11 @@ import getBalance from "./get-balance"; import createPurchase from "./create-purchase"; import { uuid } from "@cocalc/util/misc"; -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import dayjs from "dayjs"; +import { before, after } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase(); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); +beforeAll(before, 15000); +afterAll(after); describe("test computing balance under various conditions", () => { const account_id = uuid(); diff --git a/src/packages/server/purchases/get-min-balance.test.ts b/src/packages/server/purchases/get-min-balance.test.ts index 4c509c11ee..61e50d6ee8 100644 --- a/src/packages/server/purchases/get-min-balance.test.ts +++ b/src/packages/server/purchases/get-min-balance.test.ts @@ -1,15 +1,10 @@ import { uuid } from "@cocalc/util/misc"; -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import { createTestAccount } from "./test-data"; import getMinBalance from "./get-min-balance"; +import { before, after, getPool } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase({}); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); +beforeAll(before, 15000); +afterAll(after); describe("test that getMinBalance works", () => { const account_id = uuid(); diff --git a/src/packages/server/purchases/get-purchases.test.ts b/src/packages/server/purchases/get-purchases.test.ts index e63317cdd0..e81d061766 100644 --- a/src/packages/server/purchases/get-purchases.test.ts +++ b/src/packages/server/purchases/get-purchases.test.ts @@ -7,16 +7,11 @@ import createAccount from "@cocalc/server/accounts/create-account"; import createPurchase from "./create-purchase"; import getPurchases from "./get-purchases"; import { uuid } from "@cocalc/util/misc"; -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import dayjs from "dayjs"; +import { before, after, getPool } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase(); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); +beforeAll(before, 15000); +afterAll(after); describe("creates and get purchases using various options", () => { const account_id = uuid(); diff --git a/src/packages/server/purchases/get-service-cost.test.ts b/src/packages/server/purchases/get-service-cost.test.ts index 23039b80ee..170d4d908d 100644 --- a/src/packages/server/purchases/get-service-cost.test.ts +++ b/src/packages/server/purchases/get-service-cost.test.ts @@ -1,13 +1,8 @@ -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import getServiceCost from "./get-service-cost"; +import { before, after } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase({}); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); +beforeAll(before, 15000); +afterAll(after); describe("get some service costs", () => { it("get service cost of openai-gpt-3.5-turbo", async () => { @@ -21,7 +16,7 @@ describe("get some service costs", () => { but it WILL changes over time as we change our rates and openai does as well, so we just test that the keys are there and the values are positive but relatively small. - + NOTE: This is the cost to us, but we don't actually charge users now for this. */ expect(cost.completion_tokens).toBeGreaterThan(0); @@ -74,7 +69,7 @@ describe("get some service costs", () => { it("throws error on invalid service", async () => { await expect( - async () => await getServiceCost("nonsense" as any) + async () => await getServiceCost("nonsense" as any), ).rejects.toThrow(); }); }); diff --git a/src/packages/server/purchases/get-spend-rate.test.ts b/src/packages/server/purchases/get-spend-rate.test.ts index 66aac5ffe5..2ef610cf65 100644 --- a/src/packages/server/purchases/get-spend-rate.test.ts +++ b/src/packages/server/purchases/get-spend-rate.test.ts @@ -1,16 +1,11 @@ -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import getSpendRate from "./get-spend-rate"; import { uuid } from "@cocalc/util/misc"; import createAccount from "@cocalc/server/accounts/create-account"; import createPurchase from "./create-purchase"; +import { before, after, getPool } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase({}); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); +beforeAll(before, 15000); +afterAll(after); describe("get the spend rate of a user under various circumstances", () => { const account_id = uuid(); diff --git a/src/packages/server/purchases/is-purchase-allowed.test.ts b/src/packages/server/purchases/is-purchase-allowed.test.ts index cea57fad2a..2a5426f617 100644 --- a/src/packages/server/purchases/is-purchase-allowed.test.ts +++ b/src/packages/server/purchases/is-purchase-allowed.test.ts @@ -3,7 +3,6 @@ * License: MS-RSL – see LICENSE.md for details */ -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import { getServerSettings } from "@cocalc/database/settings/server-settings"; import { MAX_COST } from "@cocalc/util/db-schema/purchases"; import { uuid } from "@cocalc/util/misc"; @@ -14,14 +13,10 @@ import { isPurchaseAllowed, } from "./is-purchase-allowed"; import { getPurchaseQuota, setPurchaseQuota } from "./purchase-quotas"; +import { before, after } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase(); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); +beforeAll(before, 15000); +afterAll(after); describe("test checking whether or not purchase is allowed under various conditions", () => { const account_id = uuid(); diff --git a/src/packages/server/purchases/maintain-automatic-payments.test.ts b/src/packages/server/purchases/maintain-automatic-payments.test.ts index 63ff071af8..86c84126b1 100644 --- a/src/packages/server/purchases/maintain-automatic-payments.test.ts +++ b/src/packages/server/purchases/maintain-automatic-payments.test.ts @@ -5,7 +5,6 @@ // test the automatic payments maintenance loop -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import maintainAutomaticPayments, { setMockCollectPayment, } from "./maintain-automatic-payments"; @@ -13,18 +12,18 @@ import { uuid } from "@cocalc/util/misc"; import { createTestAccount } from "./test-data"; import createCredit from "./create-credit"; import { getServerSettings } from "@cocalc/database/settings"; +import { before, after, getPool } from "@cocalc/server/test"; -const collect: { account_id: string; amount: number }[] = []; beforeAll(async () => { + await before(); setMockCollectPayment(async ({ account_id, amount }) => { collect.push({ account_id, amount }); }); - await initEphemeralDatabase({}); }, 15000); -afterAll(async () => { - await getPool().end(); -}); +afterAll(after); + +const collect: { account_id: string; amount: number }[] = []; describe("testing automatic payments in several situations", () => { const account_id1 = uuid(); diff --git a/src/packages/server/purchases/maintain-subscriptions.test.ts b/src/packages/server/purchases/maintain-subscriptions.test.ts index 9f9faa2242..c5c9a098c0 100644 --- a/src/packages/server/purchases/maintain-subscriptions.test.ts +++ b/src/packages/server/purchases/maintain-subscriptions.test.ts @@ -1,13 +1,9 @@ import maintainSubscriptions from "./maintain-subscriptions"; -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; +import { initEphemeralDatabase } from "@cocalc/database/pool"; +import { before, after } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase({}); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); +beforeAll(before, 15000); +afterAll(after); describe("test maintainSubscriptions", () => { it("run maintainSubscriptions once and it doesn't crash", async () => { diff --git a/src/packages/server/purchases/purchase-shopping-cart-item.test.ts b/src/packages/server/purchases/purchase-shopping-cart-item.test.ts index 4df853c77e..b40938031d 100644 --- a/src/packages/server/purchases/purchase-shopping-cart-item.test.ts +++ b/src/packages/server/purchases/purchase-shopping-cart-item.test.ts @@ -6,10 +6,7 @@ import createAccount from "@cocalc/server/accounts/create-account"; import getLicense from "@cocalc/server/licenses/get-license"; import { uuid } from "@cocalc/util/misc"; -import getPool, { - initEphemeralDatabase, - getPoolClient, -} from "@cocalc/database/pool"; +import { getPoolClient } from "@cocalc/database/pool"; import purchaseShoppingCartItem from "./purchase-shopping-cart-item"; import { computeCost } from "@cocalc/util/licenses/store/compute-cost"; import { getClosingDay, setClosingDay } from "./closing-date"; @@ -19,14 +16,10 @@ import dayjs from "dayjs"; import cancelSubscription from "./cancel-subscription"; import resumeSubscription from "./resume-subscription"; import createPurchase from "./create-purchase"; +import { before, after, getPool } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase(); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); +beforeAll(before, 15000); +afterAll(after); describe("create a subscription license and edit it and confirm the subscription cost changes", () => { // this is a shopping cart item, which I basically copied from the database... diff --git a/src/packages/server/purchases/renew-subscription.test.ts b/src/packages/server/purchases/renew-subscription.test.ts index 232824f1ef..b6c4292658 100644 --- a/src/packages/server/purchases/renew-subscription.test.ts +++ b/src/packages/server/purchases/renew-subscription.test.ts @@ -6,21 +6,16 @@ // test renew-subscriptions import { test } from "./renew-subscription"; -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import { uuid } from "@cocalc/util/misc"; import renewSubscription, { getSubscription } from "./renew-subscription"; import { createTestAccount, createTestSubscription } from "./test-data"; import dayjs from "dayjs"; import createCredit from "./create-credit"; import getBalance from "./get-balance"; +import { before, after } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase({}); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); +beforeAll(before, 15000); +afterAll(after); describe("create a subscription, then renew it", () => { const account_id = uuid(); diff --git a/src/packages/server/purchases/resume-subscription.test.ts b/src/packages/server/purchases/resume-subscription.test.ts index b26800e4c8..2e1831c249 100644 --- a/src/packages/server/purchases/resume-subscription.test.ts +++ b/src/packages/server/purchases/resume-subscription.test.ts @@ -5,7 +5,6 @@ // test resuming a canceled subscription -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import { uuid } from "@cocalc/util/misc"; import { createTestAccount, createTestSubscription } from "./test-data"; //import dayjs from "dayjs"; @@ -14,14 +13,10 @@ import cancelSubscription from "./cancel-subscription"; import { getSubscription } from "./renew-subscription"; import getLicense from "@cocalc/server/licenses/get-license"; import getBalance from "./get-balance"; +import { before, after } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase({}); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); +beforeAll(before, 15000); +afterAll(after); describe("create a subscription, cancel it, then resume it", () => { const account_id = uuid(); @@ -70,45 +65,4 @@ describe("create a subscription, cancel it, then resume it", () => { expect(license.expires).toBe(license2.expires); expect(license2.expires).toBe(sub.current_period_end.valueOf()); }); - - /* - - - it("cancels again but delete all of our money, but renew does NOT fail since it doesn't require a payment.", async () => { - await cancelSubscription({ - account_id, - subscription_id, - }); - const pool = getPool(); - await pool.query("DELETE FROM purchases WHERE account_id=$1", [account_id]); - expect.assertions(1); - try { - await resumeSubscription({ account_id, subscription_id }); - } catch (e) { - expect(e.message).toMatch("Please pay"); - } - }); - - it("cancels again then change date so subscription is expired, so renew does fail due to lack of money", async () => { - await cancelSubscription({ - account_id, - subscription_id, - }); - const pool = getPool(); - await pool.query( - "update subscriptions set current_period_end=NOW() - '1 month', current_period_start=NOW()-'2 months' WHERE id=$1", - [subscription_id], - ); - await pool.query( - "update site_licenses set expire=NOW() - '1 month' where id=$1", - [license_id], - ); - expect.assertions(1); - try { - await resumeSubscription({ account_id, subscription_id }); - } catch (e) { - expect(e.message).toMatch("Please pay"); - } - }); - */ }); diff --git a/src/packages/server/purchases/shift-subscription.2.test.ts b/src/packages/server/purchases/shift-subscription.2.test.ts index 02240d2d45..998e842fa6 100644 --- a/src/packages/server/purchases/shift-subscription.2.test.ts +++ b/src/packages/server/purchases/shift-subscription.2.test.ts @@ -3,24 +3,17 @@ * License: MS-RSL – see LICENSE.md for details */ -import getPool, { - getPoolClient, - initEphemeralDatabase, -} from "@cocalc/database/pool"; +import { getPoolClient } from "@cocalc/database/pool"; import { uuid } from "@cocalc/util/misc"; import { createTestAccount, createTestSubscription } from "./test-data"; import { getSubscription } from "./renew-subscription"; import getLicense from "@cocalc/server/licenses/get-license"; import { test } from "./shift-subscriptions"; import { setClosingDay } from "./closing-date"; +import { before, after } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase({}); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); +beforeAll(before, 15000); +afterAll(after); describe("test shiftSubscriptionToEndOnDay -- involves actual subscriptions", () => { const account_id = uuid(); @@ -31,9 +24,8 @@ describe("test shiftSubscriptionToEndOnDay -- involves actual subscriptions", () await createTestAccount(account_id); await setClosingDay(account_id, 3); - ({ subscription_id, license_id } = await createTestSubscription( - account_id - )); + ({ subscription_id, license_id } = + await createTestSubscription(account_id)); }); // it("confirms that the newly created subscription has a current period end day of 3", async () => { diff --git a/src/packages/server/purchases/statements/create-statements.test.ts b/src/packages/server/purchases/statements/create-statements.test.ts index ae22dc0dfe..68e990f737 100644 --- a/src/packages/server/purchases/statements/create-statements.test.ts +++ b/src/packages/server/purchases/statements/create-statements.test.ts @@ -3,7 +3,6 @@ * License: MS-RSL – see LICENSE.md for details */ -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import { createTestAccount } from "@cocalc/server/purchases/test-data"; import createPurchase from "@cocalc/server/purchases/create-purchase"; import { createStatements, _TEST_ } from "./create-statements"; @@ -13,14 +12,10 @@ import getStatements from "./get-statements"; import getPurchases from "../get-purchases"; import dayjs from "dayjs"; import { closeAndContinuePurchase } from "../project-quotas"; +import { before, after, getPool } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase(); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); +beforeAll(before, 15000); +afterAll(after); describe("creates an account, then creates purchases and statements", () => { const account_id = uuid(); diff --git a/src/packages/server/purchases/statements/email-statement.test.ts b/src/packages/server/purchases/statements/email-statement.test.ts index 9923cd7da1..c1ba0c6ce8 100644 --- a/src/packages/server/purchases/statements/email-statement.test.ts +++ b/src/packages/server/purchases/statements/email-statement.test.ts @@ -4,21 +4,16 @@ */ import emailStatement from "./email-statement"; -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import { createTestAccount } from "@cocalc/server/purchases/test-data"; import createPurchase from "@cocalc/server/purchases/create-purchase"; import { createStatements } from "./create-statements"; import { uuid } from "@cocalc/util/misc"; import { delay } from "awaiting"; import getStatements from "./get-statements"; +import { before, after, getPool } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase(); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); +beforeAll(before, 15000); +afterAll(after); describe("creates an account, then creates statements and corresponding emails and test that everything matches up", () => { const account_id = uuid(); diff --git a/src/packages/server/purchases/student-pay.test.ts b/src/packages/server/purchases/student-pay.test.ts index 729ef2d123..db9ff94e4d 100644 --- a/src/packages/server/purchases/student-pay.test.ts +++ b/src/packages/server/purchases/student-pay.test.ts @@ -1,19 +1,14 @@ import { uuid } from "@cocalc/util/misc"; -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import { createTestAccount } from "./test-data"; import studentPay from "./student-pay"; import createProject from "@cocalc/server/projects/create"; import createCredit from "./create-credit"; import dayjs from "dayjs"; import { delay } from "awaiting"; +import { before, after, getPool } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase({}); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); +beforeAll(before, 15000); +afterAll(after); describe("test studentPay behaves at it should in various scenarios", () => { const account_id = uuid(); @@ -90,7 +85,7 @@ describe("test studentPay behaves at it should in various scenarios", () => { } }); - let purchase_id_from_student_pay : undefined | number = 0; + let purchase_id_from_student_pay: undefined | number = 0; it("add a lot of money, so it finally works -- check that the license is applied to the project", async () => { await createCredit({ account_id, amount: 1000 }); const { purchase_id } = await studentPay({ account_id, project_id }); diff --git a/src/packages/server/test/index.ts b/src/packages/server/test/index.ts new file mode 100644 index 0000000000..8f96bd5c4d --- /dev/null +++ b/src/packages/server/test/index.ts @@ -0,0 +1,42 @@ +/* +Setup an ephemeral environment in process for running tests. This includes a conat socket.io server, +file server, etc. + +TODO: it would be nice to use pglite as an *option* here so there is no need to run a separate database +server. We still need full postgres though, so we can test the ancient versions we use in production, +since pglite is only very recent postgres. +*/ + +import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; +import { + before as conatTestInit, + after as conatTestClose, +} from "@cocalc/backend/conat/test/setup"; +import { localPathFileserver } from "@cocalc/backend/conat/files/local-path"; +import { init as initFileserver } from "@cocalc/server/conat/file-server"; +import { + before as fileserverTestInit, + after as fileserverTestClose, +} from "@cocalc/file-server/btrfs/test/setup"; + +export { getPool, initEphemeralDatabase }; + +export async function before() { + await initEphemeralDatabase(); + + // run a conat socketio server + await conatTestInit(); + + // run server that can provides an enchanced fs module for files on the local filesystem + await localPathFileserver(); + + const ephemeralFilesystem = await fileserverTestInit(); + // server that provides a btrfs managed filesystem + await initFileserver(ephemeralFilesystem); +} + +export async function after() { + await getPool().end(); + await fileserverTestClose(); + await conatTestClose(); +} diff --git a/src/packages/server/test/setup.js b/src/packages/server/test/setup.js index 68b402d5f5..de13a3f1de 100644 --- a/src/packages/server/test/setup.js +++ b/src/packages/server/test/setup.js @@ -7,3 +7,5 @@ process.env.PGDATABASE = "smc_ephemeral_testing_database"; process.env.COCALC_TEST_MODE = true; process.env.COCALC_MODE = "single-user"; + +delete process.env.CONAT_SERVER; From 16a8d147e1eb590551fff136c27cd8330847a769 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 12 Aug 2025 19:41:05 +0000 Subject: [PATCH 275/798] improving unit testing --- src/packages/backend/execute-code.test.ts | 6 +-- src/packages/server/package.json | 2 +- .../server/purchases/closing-date.test.ts | 4 +- .../server/purchases/edit-license.test.ts | 4 +- .../server/purchases/get-balance.test.ts | 4 +- .../server/purchases/get-min-balance.test.ts | 4 +- .../server/purchases/get-purchases.test.ts | 4 +- .../server/purchases/get-service-cost.test.ts | 4 +- .../server/purchases/get-spend-rate.test.ts | 4 +- .../purchases/is-purchase-allowed.test.ts | 4 +- .../maintain-automatic-payments.test.ts | 2 +- .../purchases/maintain-subscriptions.test.ts | 4 +- .../purchase-shopping-cart-item.test.ts | 4 +- .../purchases/renew-subscription.test.ts | 4 +- .../purchases/resume-subscription.test.ts | 4 +- .../purchases/shift-subscription.2.test.ts | 4 +- .../statements/create-statements.test.ts | 4 +- .../statements/email-statement.test.ts | 4 +- src/packages/server/test/index.ts | 51 ++++++++++++++----- 19 files changed, 89 insertions(+), 32 deletions(-) diff --git a/src/packages/backend/execute-code.test.ts b/src/packages/backend/execute-code.test.ts index 8cca0e920d..2d7378751e 100644 --- a/src/packages/backend/execute-code.test.ts +++ b/src/packages/backend/execute-code.test.ts @@ -40,7 +40,7 @@ describe("tests involving bash mode", () => { it("reports missing executable in non-bash mode", async () => { try { await executeCode({ - command: "this_does_not_exist", + command: "/usr/bin/this_does_not_exist", args: ["nothing"], bash: false, }); @@ -52,7 +52,7 @@ describe("tests involving bash mode", () => { it("reports missing executable in non-bash mode when ignoring on exit", async () => { try { await executeCode({ - command: "this_does_not_exist", + command: "/usr/bin/this_does_not_exist", args: ["nothing"], err_on_exit: false, bash: false, @@ -376,7 +376,7 @@ describe("await", () => { it("deal with unknown executables", async () => { const c = await executeCode({ - command: "random123unknown99", + command: "/usr/bin/random123unknown99", err_on_exit: false, async_call: true, }); diff --git a/src/packages/server/package.json b/src/packages/server/package.json index 7ea93a000b..b05a37be93 100644 --- a/src/packages/server/package.json +++ b/src/packages/server/package.json @@ -36,7 +36,7 @@ "clean": "rm -rf node_modules dist", "build": "../node_modules/.bin/tsc --build", "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput ", - "test": "TZ=UTC jest --forceExit --runInBand", + "test": "TZ=UTC jest --forceExit --maxWorkers=1", "depcheck": "pnpx depcheck", "prepublishOnly": "test" }, diff --git a/src/packages/server/purchases/closing-date.test.ts b/src/packages/server/purchases/closing-date.test.ts index 68be8122a6..2877b61e44 100644 --- a/src/packages/server/purchases/closing-date.test.ts +++ b/src/packages/server/purchases/closing-date.test.ts @@ -13,7 +13,9 @@ import { uuid } from "@cocalc/util/misc"; import createAccount from "@cocalc/server/accounts/create-account"; import { before, after } from "@cocalc/server/test"; -beforeAll(before, 15000); +beforeAll(async () => { + await before({ noConat: true }); +}, 15000); afterAll(after); describe("basic consistency checks for closing date functions", () => { diff --git a/src/packages/server/purchases/edit-license.test.ts b/src/packages/server/purchases/edit-license.test.ts index dfd4fb6799..4d133311d8 100644 --- a/src/packages/server/purchases/edit-license.test.ts +++ b/src/packages/server/purchases/edit-license.test.ts @@ -20,7 +20,9 @@ import purchaseShoppingCartItem from "./purchase-shopping-cart-item"; import { license0 } from "./test-data"; import { before, after, getPool } from "@cocalc/server/test"; -beforeAll(before, 15000); +beforeAll(async () => { + await before({ noConat: true }); +}, 15000); afterAll(after); describe("create a license and then edit it in various ways", () => { diff --git a/src/packages/server/purchases/get-balance.test.ts b/src/packages/server/purchases/get-balance.test.ts index a67a7dcc23..2c76adc7f4 100644 --- a/src/packages/server/purchases/get-balance.test.ts +++ b/src/packages/server/purchases/get-balance.test.ts @@ -9,7 +9,9 @@ import { uuid } from "@cocalc/util/misc"; import dayjs from "dayjs"; import { before, after } from "@cocalc/server/test"; -beforeAll(before, 15000); +beforeAll(async () => { + await before({ noConat: true }); +}, 15000); afterAll(after); describe("test computing balance under various conditions", () => { diff --git a/src/packages/server/purchases/get-min-balance.test.ts b/src/packages/server/purchases/get-min-balance.test.ts index 61e50d6ee8..0a804c839f 100644 --- a/src/packages/server/purchases/get-min-balance.test.ts +++ b/src/packages/server/purchases/get-min-balance.test.ts @@ -3,7 +3,9 @@ import { createTestAccount } from "./test-data"; import getMinBalance from "./get-min-balance"; import { before, after, getPool } from "@cocalc/server/test"; -beforeAll(before, 15000); +beforeAll(async () => { + await before({ noConat: true }); +}, 15000); afterAll(after); describe("test that getMinBalance works", () => { diff --git a/src/packages/server/purchases/get-purchases.test.ts b/src/packages/server/purchases/get-purchases.test.ts index e81d061766..4504037fa4 100644 --- a/src/packages/server/purchases/get-purchases.test.ts +++ b/src/packages/server/purchases/get-purchases.test.ts @@ -10,7 +10,9 @@ import { uuid } from "@cocalc/util/misc"; import dayjs from "dayjs"; import { before, after, getPool } from "@cocalc/server/test"; -beforeAll(before, 15000); +beforeAll(async () => { + await before({ noConat: true }); +}, 15000); afterAll(after); describe("creates and get purchases using various options", () => { diff --git a/src/packages/server/purchases/get-service-cost.test.ts b/src/packages/server/purchases/get-service-cost.test.ts index 170d4d908d..3974b6bbc1 100644 --- a/src/packages/server/purchases/get-service-cost.test.ts +++ b/src/packages/server/purchases/get-service-cost.test.ts @@ -1,7 +1,9 @@ import getServiceCost from "./get-service-cost"; import { before, after } from "@cocalc/server/test"; -beforeAll(before, 15000); +beforeAll(async () => { + await before({ noConat: true }); +}, 15000); afterAll(after); describe("get some service costs", () => { diff --git a/src/packages/server/purchases/get-spend-rate.test.ts b/src/packages/server/purchases/get-spend-rate.test.ts index 2ef610cf65..231d71e9cc 100644 --- a/src/packages/server/purchases/get-spend-rate.test.ts +++ b/src/packages/server/purchases/get-spend-rate.test.ts @@ -4,7 +4,9 @@ import createAccount from "@cocalc/server/accounts/create-account"; import createPurchase from "./create-purchase"; import { before, after, getPool } from "@cocalc/server/test"; -beforeAll(before, 15000); +beforeAll(async () => { + await before({ noConat: true }); +}, 15000); afterAll(after); describe("get the spend rate of a user under various circumstances", () => { diff --git a/src/packages/server/purchases/is-purchase-allowed.test.ts b/src/packages/server/purchases/is-purchase-allowed.test.ts index 2a5426f617..ab69f5714c 100644 --- a/src/packages/server/purchases/is-purchase-allowed.test.ts +++ b/src/packages/server/purchases/is-purchase-allowed.test.ts @@ -15,7 +15,9 @@ import { import { getPurchaseQuota, setPurchaseQuota } from "./purchase-quotas"; import { before, after } from "@cocalc/server/test"; -beforeAll(before, 15000); +beforeAll(async () => { + await before({ noConat: true }); +}, 15000); afterAll(after); describe("test checking whether or not purchase is allowed under various conditions", () => { diff --git a/src/packages/server/purchases/maintain-automatic-payments.test.ts b/src/packages/server/purchases/maintain-automatic-payments.test.ts index 86c84126b1..e339541bd0 100644 --- a/src/packages/server/purchases/maintain-automatic-payments.test.ts +++ b/src/packages/server/purchases/maintain-automatic-payments.test.ts @@ -15,7 +15,7 @@ import { getServerSettings } from "@cocalc/database/settings"; import { before, after, getPool } from "@cocalc/server/test"; beforeAll(async () => { - await before(); + await before({ noConat: true }); setMockCollectPayment(async ({ account_id, amount }) => { collect.push({ account_id, amount }); }); diff --git a/src/packages/server/purchases/maintain-subscriptions.test.ts b/src/packages/server/purchases/maintain-subscriptions.test.ts index c5c9a098c0..a8f68d1bbf 100644 --- a/src/packages/server/purchases/maintain-subscriptions.test.ts +++ b/src/packages/server/purchases/maintain-subscriptions.test.ts @@ -2,7 +2,9 @@ import maintainSubscriptions from "./maintain-subscriptions"; import { initEphemeralDatabase } from "@cocalc/database/pool"; import { before, after } from "@cocalc/server/test"; -beforeAll(before, 15000); +beforeAll(async () => { + await before({ noConat: true }); +}, 15000); afterAll(after); describe("test maintainSubscriptions", () => { diff --git a/src/packages/server/purchases/purchase-shopping-cart-item.test.ts b/src/packages/server/purchases/purchase-shopping-cart-item.test.ts index b40938031d..7770fe31da 100644 --- a/src/packages/server/purchases/purchase-shopping-cart-item.test.ts +++ b/src/packages/server/purchases/purchase-shopping-cart-item.test.ts @@ -18,7 +18,9 @@ import resumeSubscription from "./resume-subscription"; import createPurchase from "./create-purchase"; import { before, after, getPool } from "@cocalc/server/test"; -beforeAll(before, 15000); +beforeAll(async () => { + await before({ noConat: true }); +}, 15000); afterAll(after); describe("create a subscription license and edit it and confirm the subscription cost changes", () => { diff --git a/src/packages/server/purchases/renew-subscription.test.ts b/src/packages/server/purchases/renew-subscription.test.ts index b6c4292658..a76390fd54 100644 --- a/src/packages/server/purchases/renew-subscription.test.ts +++ b/src/packages/server/purchases/renew-subscription.test.ts @@ -14,7 +14,9 @@ import createCredit from "./create-credit"; import getBalance from "./get-balance"; import { before, after } from "@cocalc/server/test"; -beforeAll(before, 15000); +beforeAll(async () => { + await before({ noConat: true }); +}, 15000); afterAll(after); describe("create a subscription, then renew it", () => { diff --git a/src/packages/server/purchases/resume-subscription.test.ts b/src/packages/server/purchases/resume-subscription.test.ts index 2e1831c249..f2cc88526e 100644 --- a/src/packages/server/purchases/resume-subscription.test.ts +++ b/src/packages/server/purchases/resume-subscription.test.ts @@ -15,7 +15,9 @@ import getLicense from "@cocalc/server/licenses/get-license"; import getBalance from "./get-balance"; import { before, after } from "@cocalc/server/test"; -beforeAll(before, 15000); +beforeAll(async () => { + await before({ noConat: true }); +}, 15000); afterAll(after); describe("create a subscription, cancel it, then resume it", () => { diff --git a/src/packages/server/purchases/shift-subscription.2.test.ts b/src/packages/server/purchases/shift-subscription.2.test.ts index 998e842fa6..c2815e2fa5 100644 --- a/src/packages/server/purchases/shift-subscription.2.test.ts +++ b/src/packages/server/purchases/shift-subscription.2.test.ts @@ -12,7 +12,9 @@ import { test } from "./shift-subscriptions"; import { setClosingDay } from "./closing-date"; import { before, after } from "@cocalc/server/test"; -beforeAll(before, 15000); +beforeAll(async () => { + await before({ noConat: true }); +}, 15000); afterAll(after); describe("test shiftSubscriptionToEndOnDay -- involves actual subscriptions", () => { diff --git a/src/packages/server/purchases/statements/create-statements.test.ts b/src/packages/server/purchases/statements/create-statements.test.ts index 68e990f737..65705e0df2 100644 --- a/src/packages/server/purchases/statements/create-statements.test.ts +++ b/src/packages/server/purchases/statements/create-statements.test.ts @@ -14,7 +14,9 @@ import dayjs from "dayjs"; import { closeAndContinuePurchase } from "../project-quotas"; import { before, after, getPool } from "@cocalc/server/test"; -beforeAll(before, 15000); +beforeAll(async () => { + await before({ noConat: true }); +}, 15000); afterAll(after); describe("creates an account, then creates purchases and statements", () => { diff --git a/src/packages/server/purchases/statements/email-statement.test.ts b/src/packages/server/purchases/statements/email-statement.test.ts index c1ba0c6ce8..b7dbe9decc 100644 --- a/src/packages/server/purchases/statements/email-statement.test.ts +++ b/src/packages/server/purchases/statements/email-statement.test.ts @@ -12,7 +12,9 @@ import { delay } from "awaiting"; import getStatements from "./get-statements"; import { before, after, getPool } from "@cocalc/server/test"; -beforeAll(before, 15000); +beforeAll(async () => { + await before({ noConat: true }); +}, 15000); afterAll(after); describe("creates an account, then creates statements and corresponding emails and test that everything matches up", () => { diff --git a/src/packages/server/test/index.ts b/src/packages/server/test/index.ts index 8f96bd5c4d..0f45a0a8f5 100644 --- a/src/packages/server/test/index.ts +++ b/src/packages/server/test/index.ts @@ -18,25 +18,52 @@ import { before as fileserverTestInit, after as fileserverTestClose, } from "@cocalc/file-server/btrfs/test/setup"; +import { delay } from "awaiting"; export { getPool, initEphemeralDatabase }; -export async function before() { - await initEphemeralDatabase(); +let opts: any = {}; +export async function before({ + noConat, + noFileserver, + noDatabase, +}: { noConat?: boolean; noFileserver?: boolean; noDatabase?: boolean } = {}) { + opts = { + noConat, + noFileserver, + noDatabase, + }; + if (!noDatabase) { + await initEphemeralDatabase(); + } - // run a conat socketio server - await conatTestInit(); + if (!noConat) { + // run a conat socketio server + await conatTestInit(); + } - // run server that can provides an enchanced fs module for files on the local filesystem - await localPathFileserver(); + if (!noFileserver && !noConat) { + // run server that can provides an enchanced fs module for files on the local filesystem + await localPathFileserver(); - const ephemeralFilesystem = await fileserverTestInit(); - // server that provides a btrfs managed filesystem - await initFileserver(ephemeralFilesystem); + const ephemeralFilesystem = await fileserverTestInit(); + // server that provides a btrfs managed filesystem + await initFileserver(ephemeralFilesystem); + } } export async function after() { - await getPool().end(); - await fileserverTestClose(); - await conatTestClose(); + const { noConat, noFileserver, noDatabase } = opts; + if (!noDatabase) { + await getPool().end(); + } + + if (!noFileserver && !noConat) { + await fileserverTestClose(); + await delay(1000); + } + + if (!noConat) { + await conatTestClose(); + } } From ee6cce6cc591bdeb717b040014e8457f62d9a942 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 12 Aug 2025 22:10:21 +0000 Subject: [PATCH 276/798] making it possible to integration test actually fully running projects and using the conat api to control them... all in process --- src/packages/backend/conat/test/setup.ts | 10 +- src/packages/conat/project/api/index.ts | 1 + .../conat/project/api/project-client.ts | 66 +++++++++++++ src/packages/file-server/btrfs/filesystem.ts | 17 +++- src/packages/file-server/btrfs/test/setup.ts | 25 +---- src/packages/file-server/btrfs/util.ts | 19 ++++ src/packages/frontend/conat/client.ts | 41 +------- .../server/compute/database-cache.test.ts | 4 +- .../maintenance/purchases/util.test.ts | 5 +- .../server/conat/project/project.test.ts | 77 +++++++++++++++ src/packages/server/conat/project/run.ts | 3 +- src/packages/server/projects/control/base.ts | 27 ++++-- src/packages/server/projects/control/index.ts | 97 ++----------------- .../server/projects/control/kubernetes.ts | 38 -------- .../server/projects/control/kucalc.ts | 41 -------- .../server/projects/control/multi-user.ts | 57 ----------- .../server/projects/control/single-user.ts | 67 ------------- src/packages/server/projects/control/util.ts | 6 +- .../server/purchases/student-pay.test.ts | 2 +- src/packages/server/purchases/student-pay.ts | 16 ++- src/packages/server/test/index.ts | 5 + 21 files changed, 238 insertions(+), 386 deletions(-) create mode 100644 src/packages/conat/project/api/project-client.ts create mode 100644 src/packages/server/conat/project/project.test.ts delete mode 100644 src/packages/server/projects/control/kubernetes.ts delete mode 100644 src/packages/server/projects/control/kucalc.ts delete mode 100644 src/packages/server/projects/control/multi-user.ts delete mode 100644 src/packages/server/projects/control/single-user.ts diff --git a/src/packages/backend/conat/test/setup.ts b/src/packages/backend/conat/test/setup.ts index 6494027a44..e36ba0ee05 100644 --- a/src/packages/backend/conat/test/setup.ts +++ b/src/packages/backend/conat/test/setup.ts @@ -301,7 +301,15 @@ export async function waitForConsistentState( export async function after() { persistServer?.close(); - await rm(tempDir, { force: true, recursive: true }); + while (true) { + try { + await rm(tempDir, { force: true, recursive: true }); + break; + } catch (err) { + console.log(err); + await delay(1000); + } + } try { server?.close(); } catch {} diff --git a/src/packages/conat/project/api/index.ts b/src/packages/conat/project/api/index.ts index 96febf9172..848b9aa09a 100644 --- a/src/packages/conat/project/api/index.ts +++ b/src/packages/conat/project/api/index.ts @@ -3,6 +3,7 @@ import { type Editor, editor } from "./editor"; import { type Jupyter, jupyter } from "./jupyter"; import { type Sync, sync } from "./sync"; import { handleErrorMessage } from "@cocalc/conat/util"; +export { projectApiClient } from "./project-client"; export interface ProjectApi { system: System; diff --git a/src/packages/conat/project/api/project-client.ts b/src/packages/conat/project/api/project-client.ts new file mode 100644 index 0000000000..a200040661 --- /dev/null +++ b/src/packages/conat/project/api/project-client.ts @@ -0,0 +1,66 @@ +/* +Create a client for the project's api. Anything that can publish to *.project-project_id... can use this. +*/ + +import { projectSubject } from "@cocalc/conat/names"; +import { type Client, connect } from "@cocalc/conat/core/client"; +import { isValidUUID } from "@cocalc/util/misc"; +import { type ProjectApi, initProjectApi } from "./index"; + +const DEFAULT_TIMEOUT = 15000; + +export function projectApiClient({ + project_id, + compute_server_id = 0, + client = connect(), + timeout = DEFAULT_TIMEOUT, +}: { + project_id: string; + compute_server_id?: number; + client?: Client; + timeout?: number; +}): ProjectApi { + if (!isValidUUID(project_id)) { + throw Error(`project_id = '${project_id}' must be a valid uuid`); + } + const callProjectApi = async ({ name, args }) => { + return await callProject({ + client, + project_id, + compute_server_id, + timeout, + service: "api", + name, + args, + }); + }; + return initProjectApi(callProjectApi); +} + +async function callProject({ + client, + service = "api", + project_id, + compute_server_id, + name, + args = [], + timeout = DEFAULT_TIMEOUT, +}: { + client: Client; + service?: string; + project_id: string; + compute_server_id?: number; + name: string; + args: any[]; + timeout?: number; +}) { + const subject = projectSubject({ project_id, compute_server_id, service }); + const resp = await client.request( + subject, + { name, args }, + // we use waitForInterest because often the project hasn't + // quite fully started. + { timeout, waitForInterest: true }, + ); + return resp.data; +} diff --git a/src/packages/file-server/btrfs/filesystem.ts b/src/packages/file-server/btrfs/filesystem.ts index 7701a93afc..25e258530c 100644 --- a/src/packages/file-server/btrfs/filesystem.ts +++ b/src/packages/file-server/btrfs/filesystem.ts @@ -12,7 +12,7 @@ a = require('@cocalc/file-server/btrfs'); fs = await a.filesystem({image:'/tmp/b */ import refCache from "@cocalc/util/refcache"; -import { mkdirp, btrfs, sudo } from "./util"; +import { mkdirp, btrfs, sudo, ensureMoreLoopbackDevices } from "./util"; import { Subvolumes } from "./subvolumes"; import { mkdir } from "fs/promises"; import { exists } from "@cocalc/backend/misc/async-utils-node"; @@ -132,13 +132,22 @@ export class Filesystem { this.opts.mount, ); { - const { stderr, exit_code } = await sudo({ + const { exit_code: failed } = await sudo({ command: "mount", args, err_on_exit: false, }); - if (exit_code) { - return { stderr, exit_code }; + if (failed) { + // try again with more loopback devices + await ensureMoreLoopbackDevices(); + const { stderr, exit_code } = await sudo({ + command: "mount", + args, + err_on_exit: false, + }); + if (exit_code) { + return { stderr, exit_code }; + } } } const { stderr, exit_code } = await sudo({ diff --git a/src/packages/file-server/btrfs/test/setup.ts b/src/packages/file-server/btrfs/test/setup.ts index af4e8a0d21..72dc8939ff 100644 --- a/src/packages/file-server/btrfs/test/setup.ts +++ b/src/packages/file-server/btrfs/test/setup.ts @@ -2,11 +2,11 @@ import { filesystem, type Filesystem, } from "@cocalc/file-server/btrfs/filesystem"; -import { chmod, mkdtemp, mkdir, rm, stat } from "node:fs/promises"; +import { chmod, mkdtemp, mkdir, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "path"; import { until } from "@cocalc/util/async-utils"; -import { sudo } from "../util"; +import { ensureMoreLoopbackDevices, sudo } from "../util"; export { sudo }; export { delay } from "awaiting"; @@ -15,25 +15,6 @@ let tempDir; const TEMP_PREFIX = "cocalc-test-btrfs-"; -async function ensureMoreLoops() { - // to run tests, this is helpful - //for i in $(seq 8 63); do sudo mknod -m660 /dev/loop$i b 7 $i; sudo chown root:disk /dev/loop$i; done - for (let i = 0; i < 64; i++) { - try { - await stat(`/dev/loop${i}`); - continue; - } catch {} - try { - // also try/catch this because ensureMoreLoops happens in parallel many times at once... - await sudo({ - command: "mknod", - args: ["-m660", `/dev/loop${i}`, "b", "7", `${i}`], - }); - } catch {} - await sudo({ command: "chown", args: ["root:disk", `/dev/loop${i}`] }); - } -} - export async function before() { try { const command = `umount ${join(tmpdir(), TEMP_PREFIX)}*/mnt`; @@ -41,7 +22,7 @@ export async function before() { // TODO: this could impact runs in parallel await sudo({ command, bash: true }); } catch {} - await ensureMoreLoops(); + await ensureMoreLoopbackDevices(); tempDir = await mkdtemp(join(tmpdir(), TEMP_PREFIX)); // Set world read/write/execute await chmod(tempDir, 0o777); diff --git a/src/packages/file-server/btrfs/util.ts b/src/packages/file-server/btrfs/util.ts index ee71b79eac..6decbab3d5 100644 --- a/src/packages/file-server/btrfs/util.ts +++ b/src/packages/file-server/btrfs/util.ts @@ -63,3 +63,22 @@ export function parseBupTime(s: string): Date { Number(seconds), ); } + +export async function ensureMoreLoopbackDevices() { + // to run tests, this is helpful + //for i in $(seq 8 63); do sudo mknod -m660 /dev/loop$i b 7 $i; sudo chown root:disk /dev/loop$i; done + for (let i = 0; i < 64; i++) { + try { + await stat(`/dev/loop${i}`); + continue; + } catch {} + try { + // also try/catch this because ensureMoreLoops happens in parallel many times at once... + await sudo({ + command: "mknod", + args: ["-m660", `/dev/loop${i}`, "b", "7", `${i}`], + }); + } catch {} + await sudo({ command: "chown", args: ["root:disk", `/dev/loop${i}`] }); + } +} diff --git a/src/packages/frontend/conat/client.ts b/src/packages/frontend/conat/client.ts index 8897d55545..f2302d0ca9 100644 --- a/src/packages/frontend/conat/client.ts +++ b/src/packages/frontend/conat/client.ts @@ -9,7 +9,7 @@ import { randomId, inboxPrefix } from "@cocalc/conat/names"; import { projectSubject } from "@cocalc/conat/names"; import { parseQueryWithOptions } from "@cocalc/sync/table/util"; import { type HubApi, initHubApi } from "@cocalc/conat/hub/api"; -import { type ProjectApi, initProjectApi } from "@cocalc/conat/project/api"; +import { type ProjectApi, projectApiClient } from "@cocalc/conat/project/api"; import { isValidUUID } from "@cocalc/util/misc"; import { PubSub } from "@cocalc/conat/sync/pubsub"; import type { ChatOptions } from "@cocalc/util/types/llm"; @@ -353,44 +353,7 @@ export class ConatClient extends EventEmitter { compute_server_id = actions.getComputeServerId(); } } - const callProjectApi = async ({ name, args }) => { - return await this.callProject({ - project_id, - compute_server_id, - timeout, - service: "api", - name, - args, - }); - }; - return initProjectApi(callProjectApi); - }; - - private callProject = async ({ - service = "api", - project_id, - compute_server_id, - name, - args = [], - timeout = DEFAULT_TIMEOUT, - }: { - service?: string; - project_id: string; - compute_server_id?: number; - name: string; - args: any[]; - timeout?: number; - }) => { - const cn = this.conat(); - const subject = projectSubject({ project_id, compute_server_id, service }); - const resp = await cn.request( - subject, - { name, args }, - // we use waitForInterest because often the project hasn't - // quite fully started. - { timeout, waitForInterest: true }, - ); - return resp.data; + return projectApiClient({ project_id, compute_server_id, timeout }); }; synctable: ConatSyncTableFunction = async ( diff --git a/src/packages/server/compute/database-cache.test.ts b/src/packages/server/compute/database-cache.test.ts index 7e551d30d4..bf67c6540d 100644 --- a/src/packages/server/compute/database-cache.test.ts +++ b/src/packages/server/compute/database-cache.test.ts @@ -2,7 +2,9 @@ import { createDatabaseCachedResource, createTTLCache } from "./database-cache"; import { delay } from "awaiting"; import { before, after } from "@cocalc/server/test"; -beforeAll(before, 15000); +beforeAll(async () => { + await before({ noConat: true }); +}, 15000); afterAll(after); // keep short so that unit testing is fast... but long enough diff --git a/src/packages/server/compute/maintenance/purchases/util.test.ts b/src/packages/server/compute/maintenance/purchases/util.test.ts index 098af47223..7c3477a2d3 100644 --- a/src/packages/server/compute/maintenance/purchases/util.test.ts +++ b/src/packages/server/compute/maintenance/purchases/util.test.ts @@ -6,10 +6,11 @@ import { getServer } from "@cocalc/server/compute/get-servers"; import { setPurchaseId } from "./util"; import { before, after } from "@cocalc/server/test"; -beforeAll(before, 15000); +beforeAll(async () => { + await before({ noConat: true }); +}, 15000); afterAll(after); - describe("creates compute server then sets the purchase id and confirms it", () => { const account_id = uuid(); let project_id; diff --git a/src/packages/server/conat/project/project.test.ts b/src/packages/server/conat/project/project.test.ts new file mode 100644 index 0000000000..17be10e4f8 --- /dev/null +++ b/src/packages/server/conat/project/project.test.ts @@ -0,0 +1,77 @@ +import { uuid } from "@cocalc/util/misc"; +import createAccount from "@cocalc/server/accounts/create-account"; +import createProject from "@cocalc/server/projects/create"; +import { projectApiClient } from "@cocalc/conat/project/api"; +import { getProject } from "@cocalc/server/projects/control"; +import { before, after } from "@cocalc/server/test"; + +beforeAll(before); +afterAll(after); + +describe("create account, project, then start and stop project", () => { + const account_id = uuid(); + let project_id; + + it("create an account and a license so we can edit it", async () => { + await createAccount({ + email: "", + password: "xyz", + firstName: "Test", + lastName: "User", + account_id, + }); + }); + + it("creates a project", async () => { + project_id = await createProject({ + account_id, + title: "My First Project", + noPool: true, + start: false, + }); + }); + + let project; + it("get state of project", async () => { + project = getProject(project_id); + const { state } = await project.state(); + expect(state).toEqual("opened"); + + // cached + expect(getProject(project_id)).toBe(project); + }); + + let projectStartTime; + it("start the project", async () => { + projectStartTime = Date.now(); + await project.start(); + const { state } = await project.state(); + expect(state).toEqual("running"); + const startupTime = Date.now() - projectStartTime; + // this better be fast (on unloaded system it is about 100ms) + expect(startupTime).toBeLessThan(2000); + }); + + it("run a command in the project to confirm everything is properly working, available and the project started and connected to conat", async () => { + const api = projectApiClient({ project_id }); + const { stdout, stderr, exit_code } = await api.system.exec({ + command: "bash", + args: ["-c", "echo $((2+3))"], + }); + expect({ stdout, stderr, exit_code }).toEqual({ + stdout: "5\n", + stderr: "", + exit_code: 0, + }); + + const firstOutputTime = Date.now() - projectStartTime; + // this better be fast (on unloaded system is less than 1 second) + expect(firstOutputTime).toBeLessThan(5000); + }); + + it("stop the project", async () => { + await project.stop(); + const { state } = await project.state(); + expect(state).toEqual("opened"); + }); +}); diff --git a/src/packages/server/conat/project/run.ts b/src/packages/server/conat/project/run.ts index 398ad67761..f8b174453b 100644 --- a/src/packages/server/conat/project/run.ts +++ b/src/packages/server/conat/project/run.ts @@ -203,7 +203,8 @@ async function start({ args.push(script, "--init", "project_init.sh"); //logEnv(env); - console.log(`${cmd} ${args.join(" ")}`); + // console.log(`${cmd} ${args.join(" ")}`); + logger.debug(`${cmd} ${args.join(" ")}`); const child = spawn(cmd, args, { env, uid, diff --git a/src/packages/server/projects/control/base.ts b/src/packages/server/projects/control/base.ts index ab8e21971a..b621f18d63 100644 --- a/src/packages/server/projects/control/base.ts +++ b/src/packages/server/projects/control/base.ts @@ -4,7 +4,7 @@ */ /* -Project control abstract base class. +Project control class. The hub uses this to get information about a project and do some basic tasks. There are different implementations for different ways in which cocalc @@ -52,11 +52,16 @@ export type Action = "open" | "start" | "stop" | "restart"; // collected. These objects don't use much memory, but blocking garbage collection // would be bad. const projectCache: { [project_id: string]: WeakRef } = {}; -export function getProject(project_id: string): BaseProject | undefined { - return projectCache[project_id]?.deref(); +export function getProject(project_id: string): BaseProject { + let project = projectCache[project_id]?.deref(); + if (project == null) { + project = new BaseProject(project_id); + projectCache[project_id] = new WeakRef(project); + } + return project!; } -export abstract class BaseProject extends EventEmitter { +export class BaseProject extends EventEmitter { public readonly project_id: string; public is_ready: boolean = false; public is_freed: boolean = false; @@ -137,11 +142,17 @@ export abstract class BaseProject extends EventEmitter { // Get the state of the project -- state is just whether or not // it is runnig, stopping, starting. It's not much info. - abstract state(): Promise; + state = async (): Promise => { + // rename everywhere to status? state is a field, and status + // is the whole object + const runner = this.projectRunner(); + return await runner.status({ project_id: this.project_id }); + }; - // Get the status of the project -- status is MUCH more information - // about the project, including ports of various services. - abstract status(): Promise; + status = async (): Promise => { + // deprecated? + return {} as ProjectStatus; + }; start = async (): Promise => { const runner = this.projectRunner(); diff --git a/src/packages/server/projects/control/index.ts b/src/packages/server/projects/control/index.ts index b71eaf3348..88ec83fcb7 100644 --- a/src/packages/server/projects/control/index.ts +++ b/src/packages/server/projects/control/index.ts @@ -1,62 +1,13 @@ import getLogger from "@cocalc/backend/logger"; import { db } from "@cocalc/database"; -import connectToProject from "@cocalc/server/projects/connection"; -import { BaseProject } from "./base"; -import kubernetes from "./kubernetes"; -import kucalc from "./kucalc"; -import multiUser from "./multi-user"; -import singleUser from "./single-user"; -import getPool from "@cocalc/database/pool"; - -export const COCALC_MODES = [ - "single-user", - "multi-user", - "kucalc", - "kubernetes", -] as const; - -export type CocalcMode = (typeof COCALC_MODES)[number]; +import { BaseProject, getProject } from "./base"; +export { getProject }; export type ProjectControlFunction = (project_id: string) => BaseProject; -// NOTE: you can't *change* the mode -- caching just caches what you first set. -let cached: ProjectControlFunction | undefined = undefined; - -export default function init(mode?: CocalcMode): ProjectControlFunction { +export default function init(): ProjectControlFunction { const logger = getLogger("project-control"); - logger.debug("init", mode); - if (cached !== undefined) { - logger.info("using cached project control client"); - return cached; - } - if (!mode) { - mode = process.env.COCALC_MODE as CocalcMode; - } - if (!mode) { - throw Error( - "you can only call projects/control with no mode argument AFTER it has been initialized by the hub or if you set the COCALC_MODE env var", - ); - } - logger.info("creating project control client"); - - let getProject; - switch (mode) { - case "single-user": - getProject = singleUser; - break; - case "multi-user": - getProject = multiUser; - break; - case "kucalc": - getProject = kucalc; - break; - case "kubernetes": - getProject = kubernetes; - break; - default: - throw Error(`invalid mode "${mode}"`); - } - logger.info(`project controller created with mode ${mode}`); + logger.debug("init"); const database = db(); database.projectControl = getProject; @@ -64,46 +15,12 @@ export default function init(mode?: CocalcMode): ProjectControlFunction { // that the there is a connection to the corresponding project, so that // the project can respond. database.ensure_connection_to_project = async ( - project_id: string, + _project_id: string, cb?: Function, ): Promise => { - const dbg = (...args) => { - logger.debug("ensure_connection_to_project: ", project_id, ...args); - }; - const pool = getPool(); - const { rows } = await pool.query( - "SELECT state->'state' AS state FROM projects WHERE project_id=$1", - [project_id], - ); - const state = rows[0]?.state; - if (state != "running") { - dbg("NOT connecting because state is not 'running', state=", state); - return; - } - dbg("connecting"); - try { - await connectToProject(project_id); - cb?.(); - } catch (err) { - dbg("WARNING: unable to make a connection", err); - cb?.(err); - } + console.log("database.ensure_connection_to_project -- DEPRECATED"); + cb?.(); }; - cached = getProject; return getProject; } - -export const getProject: ProjectControlFunction = (project_id: string) => { - if (cached == null) { - if (process.env["COCALC_MODE"]) { - return init(process.env["COCALC_MODE"] as CocalcMode)(project_id); - } - throw Error( - `must call init first or set the environment variable COCALC_MODE to one of ${COCALC_MODES.join( - ", ", - )}`, - ); - } - return cached(project_id); -}; diff --git a/src/packages/server/projects/control/kubernetes.ts b/src/packages/server/projects/control/kubernetes.ts deleted file mode 100644 index 12840bed6b..0000000000 --- a/src/packages/server/projects/control/kubernetes.ts +++ /dev/null @@ -1,38 +0,0 @@ -/* -cocalc-kubernetes support. - -TODO/CRITICAL: I deleted this from target.ts, so be sure to make this.host be actually right! - - if (project._kubernetes) { - // this is ugly -- need to determine host in case of kubernetes, since - // host as set in the project object is old/wrong. - const status = await callback2(project.status); - if (!status.ip) { - throw Error("must wait for project to start"); - } - host = status.ip; - } - - - -*/ - -import { BaseProject, ProjectStatus, ProjectState, getProject } from "./base"; -import getLogger from "@cocalc/backend/logger"; -const logger = getLogger("project-control-kubernetes"); - -class Project extends BaseProject { - async state(): Promise { - logger.debug("state ", this.project_id); - throw Error("implement me"); - } - - async status(): Promise { - logger.debug("status ", this.project_id); - throw Error("implement me"); - } -} - -export default function get(project_id: string): Project { - return (getProject(project_id) as Project) ?? new Project(project_id); -} diff --git a/src/packages/server/projects/control/kucalc.ts b/src/packages/server/projects/control/kucalc.ts deleted file mode 100644 index f6c750402b..0000000000 --- a/src/packages/server/projects/control/kucalc.ts +++ /dev/null @@ -1,41 +0,0 @@ -/* -Compute client for use in Kubernetes cluster by the hub. - -This **modifies the database** to get "something out there (manage-actions) to" -start and stop the project, copy files between projects, etc. -*/ - -import { BaseProject, ProjectStatus, ProjectState, getProject } from "./base"; -import { db } from "@cocalc/database"; -import { callback2 } from "@cocalc/util/async-utils"; - -class Project extends BaseProject { - constructor(project_id: string) { - super(project_id); - } - - private async get(columns: string[]): Promise<{ [field: string]: any }> { - return await callback2(db().get_project, { - project_id: this.project_id, - columns, - }); - } - - async state(): Promise { - return (await this.get(["state"]))?.state ?? {}; - } - - async status(): Promise { - const status = (await this.get(["status"]))?.status ?? {}; - // In KuCalc the ports for various services are hardcoded constants, - // and not actually storted in the database, so we put them here. - // This is also hardcoded in kucalc's addons/project/image/init/init.sh (!) - status["hub-server.port"] = 6000; - status["browser-server.port"] = 6001; - return status; - } -} - -export default function get(project_id: string): Project { - return (getProject(project_id) as Project) ?? new Project(project_id); -} diff --git a/src/packages/server/projects/control/multi-user.ts b/src/packages/server/projects/control/multi-user.ts deleted file mode 100644 index 47cb756a2a..0000000000 --- a/src/packages/server/projects/control/multi-user.ts +++ /dev/null @@ -1,57 +0,0 @@ -/* -multi-user: a multi-user Linux system where the hub runs as root, -so can create and delete user accounts, etc. - -There is some security and isolation between projects, coming from -different operating system users. - -This is mainly used for cocalc-docker, which is a deployment of -CoCalc running in a single docker container, with one hub running -as root. - -This **executes some basic shell commands** (e.g., useradd, rsync) -to start and stop the project, copy files between projects, etc. - -This code is very similar to single-user.ts, except with some -small modifications due to having to create and delete Linux users. -*/ - -import { getState, getStatus, homePath } from "./util"; -import { BaseProject, getProject, ProjectStatus, ProjectState } from "./base"; -import getLogger from "@cocalc/backend/logger"; - -const logger = getLogger("project-control:multi-user"); - -class Project extends BaseProject { - private HOME?: string; - - constructor(project_id: string) { - super(project_id); - } - - async state(): Promise { - if (this.stateChanging != null) { - return this.stateChanging; - } - this.HOME ??= await homePath(this.project_id); - const state = await getState(this.HOME); - logger.debug(`got state of ${this.project_id} = ${JSON.stringify(state)}`); - this.saveStateToDatabase(state); - return state; - } - - async status(): Promise { - this.HOME ??= await homePath(this.project_id); - const status = await getStatus(this.HOME); - // TODO: don't include secret token in log message. - logger.debug( - `got status of ${this.project_id} = ${JSON.stringify(status)}`, - ); - this.saveStatusToDatabase(status); - return status; - } -} - -export default function get(project_id: string): Project { - return (getProject(project_id) as Project) ?? new Project(project_id); -} diff --git a/src/packages/server/projects/control/single-user.ts b/src/packages/server/projects/control/single-user.ts deleted file mode 100644 index a1f5a1cb0e..0000000000 --- a/src/packages/server/projects/control/single-user.ts +++ /dev/null @@ -1,67 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2022 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -This is meant to run on a multi-user system, but where the hub -runs as a single user and all projects also run as that same -user, but with there own HOME directories. There is thus no -security or isolation at all between projects. There is still -a notion of multiple cocalc projects and cocalc users. - -This is useful for: - - development of cocalc from inside of a CoCalc project - - non-collaborative use of cocalc on your own - laptop, e.g., when you're on an airplane. - - -DEVELOPMENT: - - -~/cocalc/src/packages/server/projects/control$ COCALC_MODE='single-user' node -Welcome to Node.js v20.19.1. -Type ".help" for more information. -> a = require('@cocalc/server/projects/control'); -> p = a.getProject('8a840733-93b6-415c-83d4-7e5712a6266b') -> await p.start() -*/ - -import getLogger from "@cocalc/backend/logger"; -import { BaseProject, ProjectState, ProjectStatus, getProject } from "./base"; -import { getState, getStatus, homePath } from "./util"; - -const logger = getLogger("project-control:single-user"); - -class Project extends BaseProject { - private HOME?: string; - - constructor(project_id: string) { - super(project_id); - } - - async state(): Promise { - if (this.stateChanging != null) { - return this.stateChanging; - } - this.HOME ??= await homePath(this.project_id); - const state = await getState(this.HOME); - this.saveStateToDatabase(state); - return state; - } - - async status(): Promise { - this.HOME ??= await homePath(this.project_id); - const status = await getStatus(this.HOME); - // TODO: don't include secret token in log message. - logger.debug( - `got status of ${this.project_id} = ${JSON.stringify(status)}`, - ); - await this.saveStatusToDatabase(status); - return status; - } -} - -export default function get(project_id: string): Project { - return (getProject(project_id) as Project) ?? new Project(project_id); -} diff --git a/src/packages/server/projects/control/util.ts b/src/packages/server/projects/control/util.ts index 2392291479..67cefd4ab1 100644 --- a/src/packages/server/projects/control/util.ts +++ b/src/packages/server/projects/control/util.ts @@ -246,6 +246,7 @@ export async function getEnvironment( } export async function getState(HOME: string): Promise { + throw Error("getState: deprecated -- redo using conat!"); // [ ] TODO: deprecate logger.debug(`getState("${HOME}"): DEPRECATED`); return { @@ -326,9 +327,6 @@ export async function restartProjectIfRunning(project_id: string) { const project = getProject(project_id); const { state } = await project.state(); if (state == "starting" || state == "running") { - // don't await this -- it could take a long time and isn't necessary to wait for. - (async () => { - await project.restart(); - })(); + await project.restart(); } } diff --git a/src/packages/server/purchases/student-pay.test.ts b/src/packages/server/purchases/student-pay.test.ts index db9ff94e4d..d5014cee04 100644 --- a/src/packages/server/purchases/student-pay.test.ts +++ b/src/packages/server/purchases/student-pay.test.ts @@ -7,7 +7,7 @@ import dayjs from "dayjs"; import { delay } from "awaiting"; import { before, after, getPool } from "@cocalc/server/test"; -beforeAll(before, 15000); +beforeAll(before); afterAll(after); describe("test studentPay behaves at it should in various scenarios", () => { diff --git a/src/packages/server/purchases/student-pay.ts b/src/packages/server/purchases/student-pay.ts index 5cc834088c..f6df38d34c 100644 --- a/src/packages/server/purchases/student-pay.ts +++ b/src/packages/server/purchases/student-pay.ts @@ -108,16 +108,12 @@ export default async function studentPay({ // Add license to the project. await addLicenseToProject({ project_id, license_id, client }); - // nonblocking restart if running - not part of transaction, could take a while, - // and no need to block everything else on this. - (async () => { - try { - await restartProjectIfRunning(project_id); - } catch (err) { - // non-fatal, since it's just a convenience. - logger.debug("WARNING -- issue restarting project ", err); - } - })(); + try { + await restartProjectIfRunning(project_id); + } catch (err) { + // non-fatal, since it's just a convenience. + logger.debug("WARNING -- issue restarting project ", err); + } if (purchaseInfo.start == null || purchaseInfo.end == null) { throw Error("start and end must be set"); diff --git a/src/packages/server/test/index.ts b/src/packages/server/test/index.ts index 0f45a0a8f5..69011a4cc7 100644 --- a/src/packages/server/test/index.ts +++ b/src/packages/server/test/index.ts @@ -18,6 +18,8 @@ import { before as fileserverTestInit, after as fileserverTestClose, } from "@cocalc/file-server/btrfs/test/setup"; +import { init as initProjectRunner } from "@cocalc/server/conat/project/run"; +import { init as initProjectRunnerLoadBalancer } from "@cocalc/server/conat/project/load-balancer"; import { delay } from "awaiting"; export { getPool, initEphemeralDatabase }; @@ -49,6 +51,9 @@ export async function before({ const ephemeralFilesystem = await fileserverTestInit(); // server that provides a btrfs managed filesystem await initFileserver(ephemeralFilesystem); + + await initProjectRunner(); + await initProjectRunnerLoadBalancer(); } } From 18e4798389318ae39090b274d0215c676750ce3d Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 12 Aug 2025 22:45:32 +0000 Subject: [PATCH 277/798] deprecate mode; subtle issue with transactions and setting project state --- src/packages/hub/hub.ts | 23 +++++++------------ .../purchases/manage-purchases.test.ts | 1 - .../server/conat/project/project.test.ts | 5 ++++ src/packages/server/llm/test/models.test.ts | 2 +- src/packages/server/purchases/student-pay.ts | 18 +++++++++------ src/packages/server/test/setup.js | 2 -- 6 files changed, 25 insertions(+), 26 deletions(-) diff --git a/src/packages/hub/hub.ts b/src/packages/hub/hub.ts index 500b48ad3d..ff8e38c5b1 100644 --- a/src/packages/hub/hub.ts +++ b/src/packages/hub/hub.ts @@ -10,7 +10,7 @@ import { callback } from "awaiting"; import blocked from "blocked"; import { spawn } from "child_process"; -import { program as commander, Option } from "commander"; +import { program as commander } from "commander"; import basePath from "@cocalc/backend/base-path"; import { pghost as DEFAULT_DB_HOST, @@ -25,9 +25,7 @@ import { init_passport } from "@cocalc/server/hub/auth"; import { initialOnPremSetup } from "@cocalc/server/initial-onprem-setup"; import initHandleMentions from "@cocalc/server/mentions/handle"; import initMessageMaintenance from "@cocalc/server/messages/maintenance"; -import initProjectControl, { - COCALC_MODES, -} from "@cocalc/server/projects/control"; +import initProjectControl from "@cocalc/server/projects/control"; import initIdleTimeout from "@cocalc/server/projects/control/stop-idle-projects"; import initNewProjectPoolMaintenanceLoop from "@cocalc/server/projects/pool/maintain"; import initPurchasesMaintenanceLoop from "@cocalc/server/purchases/maintenance"; @@ -163,7 +161,7 @@ async function startServer(): Promise { // Project control logger.info("initializing project control..."); - const projectControl = initProjectControl(program.mode); + const projectControl = initProjectControl(); // used for nextjs hot module reloading dev server process.env["COCALC_MODE"] = program.mode; @@ -309,13 +307,10 @@ async function main(): Promise { commander .name("cocalc-hub-server") .usage("options") - .addOption( - new Option( - "--mode [string]", - `REQUIRED mode in which to run CoCalc (${COCALC_MODES.join( - ", ", - )}) - or set COCALC_MODE env var`, - ).choices(COCALC_MODES as any as string[]), + .option( + "--mode ", + `REQUIRED mode in which to run CoCalc or set COCALC_MODE env var`, + "", ) .option( "--all", @@ -430,9 +425,7 @@ async function main(): Promise { program.mode = process.env.COCALC_MODE; if (!program.mode) { throw Error( - `the --mode option must be specified or the COCALC_MODE env var set to one of ${COCALC_MODES.join( - ", ", - )}`, + `the --mode option must be specified or the COCALC_MODE env var`, ); process.exit(1); } diff --git a/src/packages/server/compute/maintenance/purchases/manage-purchases.test.ts b/src/packages/server/compute/maintenance/purchases/manage-purchases.test.ts index 905f2d5a25..1f3589c78d 100644 --- a/src/packages/server/compute/maintenance/purchases/manage-purchases.test.ts +++ b/src/packages/server/compute/maintenance/purchases/manage-purchases.test.ts @@ -27,7 +27,6 @@ afterAll(after); // we put a small delay in some cases due to using a database query pool. -// This might need to be adjusted for CI infrastructure. const DELAY = 250; diff --git a/src/packages/server/conat/project/project.test.ts b/src/packages/server/conat/project/project.test.ts index 17be10e4f8..fa706ea81b 100644 --- a/src/packages/server/conat/project/project.test.ts +++ b/src/packages/server/conat/project/project.test.ts @@ -3,6 +3,7 @@ import createAccount from "@cocalc/server/accounts/create-account"; import createProject from "@cocalc/server/projects/create"; import { projectApiClient } from "@cocalc/conat/project/api"; import { getProject } from "@cocalc/server/projects/control"; +import { restartProjectIfRunning } from "@cocalc/server/projects/control/util"; import { before, after } from "@cocalc/server/test"; beforeAll(before); @@ -31,6 +32,10 @@ describe("create account, project, then start and stop project", () => { }); }); + it("restart if running (it is not)", async () => { + await restartProjectIfRunning(project_id); + }); + let project; it("get state of project", async () => { project = getProject(project_id); diff --git a/src/packages/server/llm/test/models.test.ts b/src/packages/server/llm/test/models.test.ts index 37f91c48eb..863e8fea78 100644 --- a/src/packages/server/llm/test/models.test.ts +++ b/src/packages/server/llm/test/models.test.ts @@ -33,7 +33,7 @@ import { before, after, getPool } from "@cocalc/server/test"; const LLM_TIMEOUT = 15_000; beforeAll(async () => { - await before(); + await before({ noConat: true }); await setupAPIKeys(); await enableModels(); }, 15000); diff --git a/src/packages/server/purchases/student-pay.ts b/src/packages/server/purchases/student-pay.ts index f6df38d34c..0670d45e88 100644 --- a/src/packages/server/purchases/student-pay.ts +++ b/src/packages/server/purchases/student-pay.ts @@ -108,13 +108,6 @@ export default async function studentPay({ // Add license to the project. await addLicenseToProject({ project_id, license_id, client }); - try { - await restartProjectIfRunning(project_id); - } catch (err) { - // non-fatal, since it's just a convenience. - logger.debug("WARNING -- issue restarting project ", err); - } - if (purchaseInfo.start == null || purchaseInfo.end == null) { throw Error("start and end must be set"); } @@ -157,6 +150,17 @@ export default async function studentPay({ // end atomic transaction client.release(); } + + // Do NOT try to restart the project until outside of the transaction! + // Otherwise it deadlocks the database, since this changes the state + // of the project but not as part of the transaction!!!!! That's + // why this code is down here and not up there. + try { + await restartProjectIfRunning(project_id); + } catch (err) { + // non-fatal, since it's just a convenience. + logger.debug("WARNING -- issue restarting project ", err); + } } export function getCost( diff --git a/src/packages/server/test/setup.js b/src/packages/server/test/setup.js index de13a3f1de..a38883f781 100644 --- a/src/packages/server/test/setup.js +++ b/src/packages/server/test/setup.js @@ -6,6 +6,4 @@ process.env.PGDATABASE = "smc_ephemeral_testing_database"; // checked for in some code to behave differently while running unit tests. process.env.COCALC_TEST_MODE = true; -process.env.COCALC_MODE = "single-user"; - delete process.env.CONAT_SERVER; From 3f2642ddef300a5125a38fb67a0e8badeff89ed8 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 13 Aug 2025 02:36:26 +0000 Subject: [PATCH 278/798] more sync unit testing, paying attention to file delete/change behavior --- .../conat/project/{ => test}/project.test.ts | 2 +- .../server/conat/project/test/sync.test.ts | 149 ++++++++++++++++++ src/packages/server/test/index.ts | 5 +- src/packages/sync/editor/generic/sync-doc.ts | 132 +++++++++++----- 4 files changed, 250 insertions(+), 38 deletions(-) rename src/packages/server/conat/project/{ => test}/project.test.ts (97%) create mode 100644 src/packages/server/conat/project/test/sync.test.ts diff --git a/src/packages/server/conat/project/project.test.ts b/src/packages/server/conat/project/test/project.test.ts similarity index 97% rename from src/packages/server/conat/project/project.test.ts rename to src/packages/server/conat/project/test/project.test.ts index fa706ea81b..3f8f59966a 100644 --- a/src/packages/server/conat/project/project.test.ts +++ b/src/packages/server/conat/project/test/project.test.ts @@ -13,7 +13,7 @@ describe("create account, project, then start and stop project", () => { const account_id = uuid(); let project_id; - it("create an account and a license so we can edit it", async () => { + it("create an account and a project so we can control it", async () => { await createAccount({ email: "", password: "xyz", diff --git a/src/packages/server/conat/project/test/sync.test.ts b/src/packages/server/conat/project/test/sync.test.ts new file mode 100644 index 0000000000..f8bbd50251 --- /dev/null +++ b/src/packages/server/conat/project/test/sync.test.ts @@ -0,0 +1,149 @@ +import { uuid } from "@cocalc/util/misc"; +import createAccount from "@cocalc/server/accounts/create-account"; +import createProject from "@cocalc/server/projects/create"; +import { getProject } from "@cocalc/server/projects/control"; +import { before, after, client, connect, wait } from "@cocalc/server/test"; +import { addCollaborator } from "@cocalc/server/projects/collaborators"; +import { once } from "@cocalc/util/async-utils"; +import { delay } from "awaiting"; + +beforeAll(before); +afterAll(after); + +describe("basic collab editing of a file *on disk* in a project -- verifying interaction between filesystem and editor", () => { + const account_id1 = uuid(), + account_id2 = uuid(); + let project_id; + let project; + let fs; + + it("create accounts and a project", async () => { + await createAccount({ + email: "", + password: "xyz", + firstName: "User", + lastName: "One", + account_id: account_id1, + }); + + await createAccount({ + email: "", + password: "xyz", + firstName: "User", + lastName: "Two", + account_id: account_id2, + }); + + project_id = await createProject({ + account_id: account_id1, + title: "Collab Project", + start: false, + noPool: true, + }); + project = getProject(project_id); + + fs = client.fs({ project_id }); + + await addCollaborator({ + account_id: account_id1, + opts: { account_id: account_id2, project_id }, + }); + }); + + it("write a file that we will then open and edit (we have not started the project yet)", async () => { + await fs.writeFile("a.txt", "hello"); + await fs.writeFile("b.txt", "hello"); + expect((await project.state()).state).toBe("opened"); + }); + + let syncstring, syncstring2; + it("open 'a.txt' for sync editing", async () => { + const opts = { + project_id, + path: "a.txt", + // we use a much shorter "ignoreOnSaveInterval" so testing is fast. + ignoreOnSaveInterval: 100, + watchDebounce: 1, + deletedThreshold: 100, + watchRecreateWait: 100, + deletedCheckInterval: 50, + }; + syncstring = client.sync.string(opts); + // a second completely separate client: + syncstring2 = connect().sync.string(opts); + await Promise.all([syncstring.init(), syncstring2.init()]); + expect(syncstring).not.toBe(syncstring2); + + expect(syncstring.to_str()).toEqual("hello"); + // the first version of the document should NOT be blank + expect(syncstring.versions().length).toEqual(1); + + expect(syncstring2.to_str()).toEqual("hello"); + expect(syncstring2.versions().length).toEqual(1); + }); + + it("change the file and save to disk, then read from filesystem", async () => { + syncstring.from_str("hello world"); + await syncstring.save_to_disk(); + expect((await fs.readFile("a.txt")).toString()).toEqual("hello world"); + }); + + it("change the file on disk and observe s updates", async () => { + const change = once(syncstring, "change"); + // wait so changes to the file on disk won't be ignored: + await delay(syncstring.opts.ignoreOnSaveInterval + 50); + await fs.writeFile("a.txt", "Hello World!"); + await change; + expect(syncstring.to_str()).toEqual("Hello World!"); + }); + + it("overwrite a.txt with the older b.txt and see that this update also triggers a change even though b.txt is older -- the point is that the time is *different*", async () => { + await delay(syncstring.opts.ignoreOnSaveInterval + 50); + const change = once(syncstring, "change"); + await fs.cp("b.txt", "a.txt", { preserveTimestamps: true }); + await change; + expect(syncstring.to_str()).toEqual("hello"); + const a_stat = await fs.stat("a.txt"); + const b_stat = await fs.stat("b.txt"); + expect(a_stat.atime).toEqual(b_stat.atime); + expect(a_stat.mtime).toEqual(b_stat.mtime); + }); + + it("delete 'a.txt' from disk and observe a 'deleted' event is emitted", async () => { + await delay(250); // TODO: not good! + const deleted = once(syncstring, "deleted"); + const deleted2 = once(syncstring2, "deleted"); + await fs.unlink("a.txt"); + await deleted; + await deleted2; + // good we got the event -- we can ignore it; but doc is now blank + expect(syncstring.to_str()).toEqual(""); + expect(syncstring.isDeleted).toEqual(true); + expect(syncstring2.to_str()).toEqual(""); + expect(syncstring2.isDeleted).toEqual(true); + }); + + // this fails! + it("put a really old file at a.txt and it comes back from being deleted", async () => { + const change = once(syncstring, "change"); + await fs.writeFile("old.txt", "i am old"); + await fs.utimes( + "old.txt", + (Date.now() - 100_000) / 1000, + (Date.now() - 100_000) / 1000, + ); + await fs.cp("old.txt", "a.txt", { preserveTimestamps: true }); + await change; + expect(syncstring.to_str()).toEqual("i am old"); + // [ ] TODO: it's very disconceting that isDeleted stays true for + // one of these! + // await wait({ + // until: () => { + // console.log([syncstring.isDeleted, syncstring2.isDeleted]); + // return !syncstring.isDeleted && !syncstring2.isDeleted; + // }, + // }); + // expect(syncstring.isDeleted).toEqual(false); + // expect(syncstring2.isDeleted).toEqual(false); + }); +}); diff --git a/src/packages/server/test/index.ts b/src/packages/server/test/index.ts index 69011a4cc7..b154ebbdf1 100644 --- a/src/packages/server/test/index.ts +++ b/src/packages/server/test/index.ts @@ -11,6 +11,9 @@ import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import { before as conatTestInit, after as conatTestClose, + connect, + client, + wait, } from "@cocalc/backend/conat/test/setup"; import { localPathFileserver } from "@cocalc/backend/conat/files/local-path"; import { init as initFileserver } from "@cocalc/server/conat/file-server"; @@ -22,7 +25,7 @@ import { init as initProjectRunner } from "@cocalc/server/conat/project/run"; import { init as initProjectRunnerLoadBalancer } from "@cocalc/server/conat/project/load-balancer"; import { delay } from "awaiting"; -export { getPool, initEphemeralDatabase }; +export { client, connect, getPool, initEphemeralDatabase, wait }; let opts: any = {}; export async function before({ diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 22bd8bedfd..d9642da370 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -34,10 +34,13 @@ const WATCH_RECREATE_WAIT = 3000; // all clients ignore file changes from when a save starts until this // amount of time later, so they avoid loading a file just because it was -// saved by themself or another client. This is very important for -// large files. -const IGNORE_ON_SAVE_INTERVAL = 10000; +// saved by themself or another client. This is especially important for +// large files that can take a long time to save. +const IGNORE_ON_SAVE_INTERVAL = 7500; +// reading file when it changes on disk is deboucned this much, e.g., +// if the file keeps changing you won't see those changes until it +// stops changing for this long. const WATCH_DEBOUNCE = 250; import { @@ -136,10 +139,15 @@ export interface SyncOpts0 { // optional timeout for how long to wait from when a file is // deleted until emiting a 'deleted' event. deletedThreshold?: number; + deletedCheckInterval?: number; // how long to wait before trying to recreate a watch -- this mainly // matters in cases when the file is deleted and the client ignores // the 'deleted' event. watchRecreateWait?: number; + + // instead of the default IGNORE_ON_SAVE_INTERVAL + ignoreOnSaveInterval?: number; + watchDebounce?: number; } export interface SyncOpts extends SyncOpts0 { @@ -161,6 +169,7 @@ const logger = getLogger("sync-doc"); logger.debug("init"); export class SyncDoc extends EventEmitter { + public readonly opts: SyncOpts; public readonly project_id: string; // project_id that contains the doc public readonly path: string; // path of the file corresponding to the doc private string_id: string; @@ -240,11 +249,12 @@ export class SyncDoc extends EventEmitter { private noAutosave?: boolean; - private deletedThreshold?: number; - private watchRecreateWait?: number; + private readFileDebounced: Function; + public isDeleted: boolean = false; constructor(opts: SyncOpts) { super(); + this.opts = opts; if (opts.string_id === undefined) { this.string_id = schema.client_db.sha1(opts.project_id, opts.path); @@ -252,6 +262,7 @@ export class SyncDoc extends EventEmitter { this.string_id = opts.string_id; } + // TODO: it might be better to just use this.opts.field everywhere...? for (const field of [ "project_id", "path", @@ -267,14 +278,27 @@ export class SyncDoc extends EventEmitter { "ephemeral", "fs", "noAutosave", - "deletedThreshold", - "watchRecreateWait", ]) { if (opts[field] != undefined) { this[field] = opts[field]; } } + this.readFileDebounced = asyncDebounce( + async () => { + try { + this.emit("handle-file-change"); + await this.readFile(); + await this.stat(); + } catch {} + }, + this.opts.watchDebounce ?? WATCH_DEBOUNCE, + { + leading: false, + trailing: true, + }, + ); + this.client.once("closed", this.close); this.legacy = new LegacyHistory({ @@ -327,7 +351,7 @@ export class SyncDoc extends EventEmitter { this SyncDoc. */ private initialized = false; - private init = async () => { + init = reuseInFlight(async () => { if (this.initialized) { throw Error("init can only be called once"); } @@ -366,7 +390,7 @@ export class SyncDoc extends EventEmitter { // Success -- everything initialized with no issues. this.set_state("ready"); this.emit_change(); // from nothing to something. - }; + }); // Return id of ACTIVE remote compute server, if one is connected and pinging, or 0 // if none is connected. This is used by Jupyter to determine who @@ -1168,7 +1192,7 @@ export class SyncDoc extends EventEmitter { this.init_cursors(), this.init_evaluator(), this.init_ipywidgets(), - this.initFileWatcher(), + this.initFileWatcherFirstTime(), ]); this.assert_not_closed( "initAll -- successful init patch_list, cursors, evaluator, and ipywidgets", @@ -2250,15 +2274,22 @@ export class SyncDoc extends EventEmitter { let contents; try { contents = await this.fs.readFile(this.path, "utf8"); + // console.log(this.client.client.id, "read from disk --isDeleted = false"); + if (this.isDeleted) { + this.isDeleted = false; + } this.valueOnDisk = contents; dbg("file exists"); size = contents.length; this.from_str(contents); } catch (err) { if (err.code == "ENOENT") { + // console.log(this.client.client.id, "reset doc and set isDeleted=true"); + this.isDeleted = true; dbg("file no longer exists -- setting to blank"); size = 0; this.from_str(""); + this.commit(); } else { throw err; } @@ -2283,6 +2314,7 @@ export class SyncDoc extends EventEmitter { const prevStats = this.stats; this.stats = (await this.fs.stat(this.path)) as Stats; if (prevStats?.mode != this.stats.mode) { + // used by clients to track read-only state. this.emit("metadata-change"); } return this.stats; @@ -2336,7 +2368,7 @@ export class SyncDoc extends EventEmitter { // NOTE: this does not take into account saving by another client // anymore; it used to, but that made things much more complicated. hash_of_saved_version = (): number | undefined => { - if (!this.isReady() || this.valueOnDisk == null) { + if (!this.isReady() || this.valueOnDisk == null || this.isDeleted) { return; } return hash_string(this.valueOnDisk); @@ -2411,7 +2443,7 @@ export class SyncDoc extends EventEmitter { private valueOnDisk: string | undefined = undefined; private hasUnsavedChanges = (): boolean => { - return this.valueOnDisk != this.to_str(); + return this.valueOnDisk != this.to_str() || this.isDeleted; }; writeFile = async () => { @@ -2434,7 +2466,9 @@ export class SyncDoc extends EventEmitter { // so no clients waste resources loading in response to us saving // to disk. try { - await this.fileWatcher?.ignore(IGNORE_ON_SAVE_INTERVAL); + await this.fileWatcher?.ignore( + this.opts.ignoreOnSaveInterval ?? IGNORE_ON_SAVE_INTERVAL, + ); } catch { // not a big problem if we can't ignore (e.g., this happens potentially // after deleting the file or if file doesn't exist) @@ -2729,27 +2763,43 @@ export class SyncDoc extends EventEmitter { } }, 60000); - private readFileDebounced = asyncDebounce( - async () => { - try { - this.emit("handle-file-change"); - await this.readFile(); - await this.stat(); - } catch {} - }, - WATCH_DEBOUNCE, - { - leading: false, - trailing: true, - }, - ); + private initFileWatcherFirstTime = () => { + // set this going, but don't await it. + (async () => { + await until( + async () => { + if (this.isClosed()) return true; + try { + await this.initFileWatcher(); + return true; + } catch { + return false; + } + }, + { min: this.opts?.watchRecreateWait ?? WATCH_RECREATE_WAIT }, + ); + })(); + }; private fileWatcher?: any; private initFileWatcher = async () => { // use this.fs interface to watch path for changes -- we try once: try { this.fileWatcher = await this.fs.watch(this.path, { unique: true }); - } catch {} + if (this.isDeleted) { + await this.readFile(); + } + } catch (err) { + // console.log("error creating watcher", err); + if (err.code == "ENOENT") { + // the file was deleted -- check if this stays deleted long enough to count + await this.signalIfFileDeleted(); + } + // throwing this error just causes initFileWatcher to get + // initialized again soon (a few seconds), again attemping a watch, + // unless it is the first time initializing the document. + throw err; + } if (this.isClosed()) return; // not closed -- so if above succeeds we start watching. @@ -2772,12 +2822,13 @@ export class SyncDoc extends EventEmitter { } } // check if file was deleted - this.closeIfFileDeleted(); + this.signalIfFileDeleted(); this.fileWatcher?.close(); delete this.fileWatcher; } + if (this.isClosed()) return; // start a new watcher since file descriptor probably changed or maybe file deleted - await delay(this.watchRecreateWait ?? WATCH_RECREATE_WAIT); + await delay(this.opts?.watchRecreateWait ?? WATCH_RECREATE_WAIT); await until( async () => { if (this.isClosed()) return true; @@ -2788,7 +2839,7 @@ export class SyncDoc extends EventEmitter { return false; } }, - { min: this.watchRecreateWait ?? WATCH_RECREATE_WAIT }, + { min: this.opts?.watchRecreateWait ?? WATCH_RECREATE_WAIT }, ); })(); }; @@ -2814,14 +2865,14 @@ export class SyncDoc extends EventEmitter { } }; - private closeIfFileDeleted = async () => { + private signalIfFileDeleted = async (): Promise => { if (this.isClosed()) return; const start = Date.now(); - const threshold = this.deletedThreshold ?? DELETED_THRESHOLD; - while (true) { + const threshold = this.opts.deletedThreshold ?? DELETED_THRESHOLD; + while (!this.isClosed()) { try { if (await this.fileExists()) { - // file definitely exists right now. + // file definitely exists right now -- NOT deleted. return; } // file definitely does NOT exist right now. @@ -2835,10 +2886,19 @@ export class SyncDoc extends EventEmitter { // it does not exist above // file still doesn't exist -- consider it deleted -- browsers // should close the tab and possibly notify user. + this.from_str(""); + this.commit(); + // console.log("emit deleted and set isDeleted=true"); + this.isDeleted = true; this.emit("deleted"); return; } - await delay(Math.min(DELETED_CHECK_INTERVAL, threshold - elapsed)); + await delay( + Math.min( + this.opts.deletedCheckInterval ?? DELETED_CHECK_INTERVAL, + threshold - elapsed, + ), + ); } }; } From de2b19e8486efe652561f89b741b92b1f7b8ed98 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 13 Aug 2025 15:22:28 +0000 Subject: [PATCH 279/798] include ip address with project status --- src/packages/conat/project/runner/state.ts | 1 + src/packages/server/conat/project/run.ts | 4 +++- src/packages/server/conat/project/test/sync.test.ts | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/packages/conat/project/runner/state.ts b/src/packages/conat/project/runner/state.ts index f2f3f7ab5b..aa677748c6 100644 --- a/src/packages/conat/project/runner/state.ts +++ b/src/packages/conat/project/runner/state.ts @@ -10,6 +10,7 @@ export type ProjectState = "running" | "opened" | "stopping" | "starting"; export interface ProjectStatus { server?: string; state: ProjectState; + ip?: string; // the ip address when running } export default async function state({ client }) { diff --git a/src/packages/server/conat/project/run.ts b/src/packages/server/conat/project/run.ts index f8b174453b..b968d66c99 100644 --- a/src/packages/server/conat/project/run.ts +++ b/src/packages/server/conat/project/run.ts @@ -248,7 +248,9 @@ async function status({ project_id }) { state = "running"; } setProjectState({ project_id, state }); - return { state }; + // [ ] TODO: ip -- need to figure out the networking story for running projects + // The following will only work on a single machine with global network address space + return { state, ip: "127.0.0.1" }; } export async function init(count: number = 1) { diff --git a/src/packages/server/conat/project/test/sync.test.ts b/src/packages/server/conat/project/test/sync.test.ts index f8bbd50251..b9a7f2d025 100644 --- a/src/packages/server/conat/project/test/sync.test.ts +++ b/src/packages/server/conat/project/test/sync.test.ts @@ -2,7 +2,7 @@ import { uuid } from "@cocalc/util/misc"; import createAccount from "@cocalc/server/accounts/create-account"; import createProject from "@cocalc/server/projects/create"; import { getProject } from "@cocalc/server/projects/control"; -import { before, after, client, connect, wait } from "@cocalc/server/test"; +import { before, after, client, connect } from "@cocalc/server/test"; import { addCollaborator } from "@cocalc/server/projects/collaborators"; import { once } from "@cocalc/util/async-utils"; import { delay } from "awaiting"; From d486297527c5d3f011eb2da7d48ebfcb9d7016df Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 13 Aug 2025 12:34:20 -0700 Subject: [PATCH 280/798] eliminate COCALC_DOCKER env var; also serve on non-priv ports (working toward it running not as root) --- src/packages/backend/logger.ts | 2 -- src/packages/hub/package.json | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/packages/backend/logger.ts b/src/packages/backend/logger.ts index 6d1dcea6a3..4b89029caa 100644 --- a/src/packages/backend/logger.ts +++ b/src/packages/backend/logger.ts @@ -71,8 +71,6 @@ function myFormat(...args): string { function defaultTransports(): { console?: boolean; file?: string } { if (process.env.SMC_TEST) { return {}; - } else if (process.env.COCALC_DOCKER) { - return { file: "/var/log/hub/log" }; } else if (process.env.NODE_ENV == "production") { return { console: true }; } else { diff --git a/src/packages/hub/package.json b/src/packages/hub/package.json index f7247e7597..21ed4a8504 100644 --- a/src/packages/hub/package.json +++ b/src/packages/hub/package.json @@ -60,9 +60,9 @@ "hub-project-dev": "pnpm build && NODE_OPTIONS='--inspect' pnpm hub-project-dev-nobuild", "hub-project-prod-nobuild": "unset DATA COCALC_ROOT && export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && export PGHOST=${PGHOST:=`realpath $INIT_CWD/../../data/postgres/socket`} && PGUSER='smc' NODE_ENV=production NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=single-user --all --hostname=0.0.0.0", "hub-project-prod-ssl": "unset DATA COCALC_ROOT && export CONAT_SERVER=https://localhost:$PORT && export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && export PGHOST=${PGHOST:=`realpath $INIT_CWD/../../data/postgres/socket`} && PGUSER='smc' NODE_ENV=production NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=single-user --all --hostname=0.0.0.0 --https-key=$INIT_CWD/../../data/secrets/cert/key.pem --https-cert=$INIT_CWD/../../data/secrets/cert/cert.pem", - "hub-docker-dev": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && COCALC_DOCKER=true NODE_ENV=development PROJECTS=/projects/[project_id] PORT=443 NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps --trace-warnings --inspect' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0 --https-key=/projects/conf/cert/key.pem --https-cert=/projects/conf/cert/cert.pem", - "hub-docker-prod": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && COCALC_DOCKER=true NODE_ENV=production PROJECTS=/projects/[project_id] PORT=443 NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0 --https-key=/projects/conf/cert/key.pem --https-cert=/projects/conf/cert/cert.pem", - "hub-docker-prod-nossl": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && COCALC_DOCKER=true NODE_ENV=production PROJECTS=/projects/[project_id] PORT=80 NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0", + "hub-docker-dev": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && NODE_ENV=development PROJECTS=/projects/[project_id] PORT=4043 NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps --trace-warnings --inspect' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0 --https-key=/projects/conf/cert/key.pem --https-cert=/projects/conf/cert/cert.pem", + "hub-docker-prod": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && NODE_ENV=production PROJECTS=/projects/[project_id] PORT=4043 NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0 --https-key=/projects/conf/cert/key.pem --https-cert=/projects/conf/cert/cert.pem", + "hub-docker-prod-nossl": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && NODE_ENV=production PROJECTS=/projects/[project_id] PORT=8080 NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0", "tsc": "tsc --watch --pretty --preserveWatchOutput", "test": "jest dist/", "depcheck": "pnpx depcheck | grep -Ev '\\.coffee|coffee$'", From f325899035c95f980af03b3eb211ed7f9485a9ee Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 13 Aug 2025 12:47:14 -0700 Subject: [PATCH 281/798] working on cocalc-docker --- src/packages/hub/package.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/packages/hub/package.json b/src/packages/hub/package.json index 21ed4a8504..c4accb5831 100644 --- a/src/packages/hub/package.json +++ b/src/packages/hub/package.json @@ -60,9 +60,9 @@ "hub-project-dev": "pnpm build && NODE_OPTIONS='--inspect' pnpm hub-project-dev-nobuild", "hub-project-prod-nobuild": "unset DATA COCALC_ROOT && export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && export PGHOST=${PGHOST:=`realpath $INIT_CWD/../../data/postgres/socket`} && PGUSER='smc' NODE_ENV=production NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=single-user --all --hostname=0.0.0.0", "hub-project-prod-ssl": "unset DATA COCALC_ROOT && export CONAT_SERVER=https://localhost:$PORT && export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && export PGHOST=${PGHOST:=`realpath $INIT_CWD/../../data/postgres/socket`} && PGUSER='smc' NODE_ENV=production NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=single-user --all --hostname=0.0.0.0 --https-key=$INIT_CWD/../../data/secrets/cert/key.pem --https-cert=$INIT_CWD/../../data/secrets/cert/cert.pem", - "hub-docker-dev": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && NODE_ENV=development PROJECTS=/projects/[project_id] PORT=4043 NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps --trace-warnings --inspect' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0 --https-key=/projects/conf/cert/key.pem --https-cert=/projects/conf/cert/cert.pem", - "hub-docker-prod": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && NODE_ENV=production PROJECTS=/projects/[project_id] PORT=4043 NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0 --https-key=/projects/conf/cert/key.pem --https-cert=/projects/conf/cert/cert.pem", - "hub-docker-prod-nossl": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && NODE_ENV=production PROJECTS=/projects/[project_id] PORT=8080 NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0", + "hub-docker-dev": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && NODE_ENV=development PORT=4043 NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps --trace-warnings --inspect' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0 --https-key=/data/conf/cert/key.pem --https-cert=/data/conf/cert/cert.pem", + "hub-docker-prod": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && NODE_ENV=production PORT=4043 NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0 --https-key=/data/conf/cert/key.pem --https-cert=/data/conf/cert/cert.pem", + "hub-docker-prod-nossl": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && NODE_ENV=production PORT=8080 NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0", "tsc": "tsc --watch --pretty --preserveWatchOutput", "test": "jest dist/", "depcheck": "pnpx depcheck | grep -Ev '\\.coffee|coffee$'", From 683fba7a0a7ece1323a4be2d89adb4bb142c176b Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 13 Aug 2025 14:29:29 -0700 Subject: [PATCH 282/798] do not specify port for docker --- src/packages/hub/package.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/packages/hub/package.json b/src/packages/hub/package.json index c4accb5831..24f258311f 100644 --- a/src/packages/hub/package.json +++ b/src/packages/hub/package.json @@ -60,9 +60,9 @@ "hub-project-dev": "pnpm build && NODE_OPTIONS='--inspect' pnpm hub-project-dev-nobuild", "hub-project-prod-nobuild": "unset DATA COCALC_ROOT && export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && export PGHOST=${PGHOST:=`realpath $INIT_CWD/../../data/postgres/socket`} && PGUSER='smc' NODE_ENV=production NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=single-user --all --hostname=0.0.0.0", "hub-project-prod-ssl": "unset DATA COCALC_ROOT && export CONAT_SERVER=https://localhost:$PORT && export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && export PGHOST=${PGHOST:=`realpath $INIT_CWD/../../data/postgres/socket`} && PGUSER='smc' NODE_ENV=production NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=single-user --all --hostname=0.0.0.0 --https-key=$INIT_CWD/../../data/secrets/cert/key.pem --https-cert=$INIT_CWD/../../data/secrets/cert/cert.pem", - "hub-docker-dev": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && NODE_ENV=development PORT=4043 NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps --trace-warnings --inspect' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0 --https-key=/data/conf/cert/key.pem --https-cert=/data/conf/cert/cert.pem", - "hub-docker-prod": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && NODE_ENV=production PORT=4043 NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0 --https-key=/data/conf/cert/key.pem --https-cert=/data/conf/cert/cert.pem", - "hub-docker-prod-nossl": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && NODE_ENV=production PORT=8080 NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0", + "hub-docker-dev": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && NODE_ENV=development NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps --trace-warnings --inspect' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0 --https-key=/data/conf/cert/key.pem --https-cert=/data/conf/cert/cert.pem", + "hub-docker-prod": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && NODE_ENV=production NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0 --https-key=/data/conf/cert/key.pem --https-cert=/data/conf/cert/cert.pem", + "hub-docker-prod-nossl": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && NODE_ENV=production NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0", "tsc": "tsc --watch --pretty --preserveWatchOutput", "test": "jest dist/", "depcheck": "pnpx depcheck | grep -Ev '\\.coffee|coffee$'", From 51654f2fde179877695dcd66997161af2e2c4185 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 13 Aug 2025 14:55:43 -0700 Subject: [PATCH 283/798] use the path to the nsjail we build from source during the install --- src/packages/server/conat/project/run.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/packages/server/conat/project/run.ts b/src/packages/server/conat/project/run.ts index b968d66c99..0c9abdfa4c 100644 --- a/src/packages/server/conat/project/run.ts +++ b/src/packages/server/conat/project/run.ts @@ -51,6 +51,7 @@ import { client as fileserverClient, type Fileserver, } from "@cocalc/server/conat/file-server"; +import { nsjail } from "@cocalc/backend/sandbox/install"; // for development it may be useful to just disabling using nsjail namespaces // entirely -- change this to true to do so. @@ -197,7 +198,7 @@ async function start({ args.push(...limits(config)); args.push("--"); args.push(process.execPath); - cmd = "nsjail"; + cmd = nsjail; } args.push(script, "--init", "project_init.sh"); From 8d32d38be4c9d421e9e5b07784c6fc0ef65d5912 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 14 Aug 2025 04:07:42 +0000 Subject: [PATCH 284/798] find test was sometimes failing due to not enough time --- src/packages/backend/sandbox/find.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/packages/backend/sandbox/find.test.ts b/src/packages/backend/sandbox/find.test.ts index 66574f5cac..fa265b5b49 100644 --- a/src/packages/backend/sandbox/find.test.ts +++ b/src/packages/backend/sandbox/find.test.ts @@ -3,6 +3,7 @@ import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; + let tempDir; beforeAll(async () => { tempDir = await mkdtemp(join(tmpdir(), "cocalc")); @@ -11,6 +12,8 @@ afterAll(async () => { await rm(tempDir, { force: true, recursive: true }); }); + +jest.setTimeout(15000); describe("find files", () => { it("directory starts empty", async () => { const { stdout, truncated } = await find(tempDir, { From 65ed4e806b7989de14b73b7588b2fb5a8ad8ef38 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 14 Aug 2025 20:13:25 +0000 Subject: [PATCH 285/798] fix major state bug in use-fs in the frontend that broke a lot regarding directory listings --- src/packages/frontend/project/directory-selector.tsx | 8 ++++++-- src/packages/frontend/project/listing/use-fs.ts | 11 +++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/packages/frontend/project/directory-selector.tsx b/src/packages/frontend/project/directory-selector.tsx index 1463cbbbe3..180d383e72 100644 --- a/src/packages/frontend/project/directory-selector.tsx +++ b/src/packages/frontend/project/directory-selector.tsx @@ -69,7 +69,7 @@ export default function DirectorySelector({ closable = true, }: Props) { const frameContext = useFrameContext(); // optionally used to define project_id and startingPath, when in a frame - if (project_id == null) project_id = frameContext.project_id; + project_id ??= frameContext.project_id; const fallbackComputeServerId = useTypedRedux( { project_id }, "compute_server_id", @@ -377,10 +377,14 @@ function Subdirs(props) { toggleSelection, } = props; const fs = useFs({ project_id, computeServerId }); + const cacheId = getCacheId({ + project_id, + compute_server_id: computeServerId, + }); const { files, error, refresh } = useFiles({ fs, path, - cacheId: getCacheId({ project_id, compute_server_id: computeServerId }), + cacheId, }); if (error) { return ; diff --git a/src/packages/frontend/project/listing/use-fs.ts b/src/packages/frontend/project/listing/use-fs.ts index 1bf01ab762..6b2f330f75 100644 --- a/src/packages/frontend/project/listing/use-fs.ts +++ b/src/packages/frontend/project/listing/use-fs.ts @@ -3,7 +3,7 @@ Hook for getting a FilesystemClient. */ import { webapp_client } from "@cocalc/frontend/webapp-client"; import { type FilesystemClient } from "@cocalc/conat/files/fs"; -import { useState } from "react"; +import { useMemo } from "react"; // this will probably get more complicated temporarily when we // are transitioning between filesystems (hence why we return null in @@ -17,13 +17,12 @@ export default function useFs({ compute_server_id?: number; computeServerId?: number; }): FilesystemClient | null { - const [fs] = useState(() => - webapp_client.conat_client - .conat() - .fs({ + return useMemo( + () => + webapp_client.conat_client.conat().fs({ project_id, compute_server_id: compute_server_id ?? computeServerId, }), + [project_id, compute_server_id, computeServerId], ); - return fs; } From be068c68874d36ac1e2195b2041cbec19f17fce6 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 14 Aug 2025 22:21:05 +0000 Subject: [PATCH 286/798] completely delete the new project pool -- it's better to just make project startup and creation ridiculously fast --- src/packages/frontend/client/project.ts | 2 - .../course/student-projects/actions.ts | 1 - src/packages/frontend/projects/actions.ts | 3 - src/packages/hub/hub.ts | 6 - .../server/conat/project/test/project.test.ts | 1 - .../server/conat/project/test/sync.test.ts | 1 - src/packages/server/projects/create.ts | 17 --- .../server/projects/pool/all-projects.ts | 44 -------- .../server/projects/pool/get-project.ts | 106 ------------------ src/packages/server/projects/pool/maintain.ts | 83 -------------- src/packages/util/db-schema/projects.ts | 3 - src/packages/util/db-schema/site-defaults.ts | 8 -- 12 files changed, 275 deletions(-) delete mode 100644 src/packages/server/projects/pool/all-projects.ts delete mode 100644 src/packages/server/projects/pool/get-project.ts delete mode 100644 src/packages/server/projects/pool/maintain.ts diff --git a/src/packages/frontend/client/project.ts b/src/packages/frontend/client/project.ts index c977d88a8f..d720022483 100644 --- a/src/packages/frontend/client/project.ts +++ b/src/packages/frontend/client/project.ts @@ -473,8 +473,6 @@ export class ProjectClient { start?: boolean; // "license_id1,license_id2,..." -- if given, create project with these licenses applied license?: string; - // never use pool - noPool?: boolean; // make exact clone of the files from this project: src_project_id?: string; }): Promise => { diff --git a/src/packages/frontend/course/student-projects/actions.ts b/src/packages/frontend/course/student-projects/actions.ts index a173dda6c0..0a534704b0 100644 --- a/src/packages/frontend/course/student-projects/actions.ts +++ b/src/packages/frontend/course/student-projects/actions.ts @@ -73,7 +73,6 @@ export class StudentProjectsActions { title: store.get("settings").get("title"), description: store.get("settings").get("description"), image: store.get("settings").get("custom_image") ?? defaultImage, - noPool: true, // student is unlikely to use the project right *now* }); } catch (err) { this.course_actions.set_error( diff --git a/src/packages/frontend/projects/actions.ts b/src/packages/frontend/projects/actions.ts index ba281d4465..3c98cc7e15 100644 --- a/src/packages/frontend/projects/actions.ts +++ b/src/packages/frontend/projects/actions.ts @@ -379,7 +379,6 @@ export class ProjectsActions extends Actions { description?: string; image?: string; // if given, sets the compute image (the ID string) start?: boolean; // immediately start on create - noPool?: boolean; // never use the pool license?: string; }): Promise { const image = await redux.getStore("customize").getDefaultComputeImage(); @@ -389,14 +388,12 @@ export class ProjectsActions extends Actions { description: string; image?: string; start: boolean; - noPool?: boolean; license?: string; } = defaults(opts, { title: "No Title", description: "No Description", image, start: false, - noPool: undefined, license: undefined, }); if (!opts2.image) { diff --git a/src/packages/hub/hub.ts b/src/packages/hub/hub.ts index ff8e38c5b1..562cb8ecea 100644 --- a/src/packages/hub/hub.ts +++ b/src/packages/hub/hub.ts @@ -27,7 +27,6 @@ import initHandleMentions from "@cocalc/server/mentions/handle"; import initMessageMaintenance from "@cocalc/server/messages/maintenance"; import initProjectControl from "@cocalc/server/projects/control"; import initIdleTimeout from "@cocalc/server/projects/control/stop-idle-projects"; -import initNewProjectPoolMaintenanceLoop from "@cocalc/server/projects/pool/maintain"; import initPurchasesMaintenanceLoop from "@cocalc/server/purchases/maintenance"; import initSalesloftMaintenance from "@cocalc/server/salesloft/init"; import { stripe_sync } from "@cocalc/server/stripe/sync"; @@ -285,11 +284,6 @@ async function startServer(): Promise { } if (program.all || program.mentions) { - // kucalc: for now we just have the hub-mentions servers - // do the new project pool maintenance, since there is only - // one hub-stats. - // On non-cocalc it'll get done by *the* hub because of program.all. - initNewProjectPoolMaintenanceLoop(); // Starts periodic maintenance on pay-as-you-go purchases, e.g., quota // upgrades of projects. initPurchasesMaintenanceLoop(); diff --git a/src/packages/server/conat/project/test/project.test.ts b/src/packages/server/conat/project/test/project.test.ts index 3f8f59966a..5ae1d7d338 100644 --- a/src/packages/server/conat/project/test/project.test.ts +++ b/src/packages/server/conat/project/test/project.test.ts @@ -27,7 +27,6 @@ describe("create account, project, then start and stop project", () => { project_id = await createProject({ account_id, title: "My First Project", - noPool: true, start: false, }); }); diff --git a/src/packages/server/conat/project/test/sync.test.ts b/src/packages/server/conat/project/test/sync.test.ts index b9a7f2d025..fd0ab0c731 100644 --- a/src/packages/server/conat/project/test/sync.test.ts +++ b/src/packages/server/conat/project/test/sync.test.ts @@ -38,7 +38,6 @@ describe("basic collab editing of a file *on disk* in a project -- verifying int account_id: account_id1, title: "Collab Project", start: false, - noPool: true, }); project = getProject(project_id); diff --git a/src/packages/server/projects/create.ts b/src/packages/server/projects/create.ts index 13fbeecdb3..c24c63bc84 100644 --- a/src/packages/server/projects/create.ts +++ b/src/packages/server/projects/create.ts @@ -9,7 +9,6 @@ import { DEFAULT_COMPUTE_IMAGE } from "@cocalc/util/db-schema/defaults"; import { isValidUUID } from "@cocalc/util/misc"; import { v4 } from "uuid"; import { associatedLicense } from "@cocalc/server/licenses/public-path"; -import getFromPool from "@cocalc/server/projects/pool/get-project"; import getLogger from "@cocalc/backend/logger"; import { getProject } from "@cocalc/server/projects/control"; import { type CreateProjectOptions } from "@cocalc/util/db-schema/projects"; @@ -34,7 +33,6 @@ export default async function createProject(opts: CreateProjectOptions) { description, image, public_path_id, - noPool, start, src_project_id, } = opts; @@ -57,21 +55,6 @@ export default async function createProject(opts: CreateProjectOptions) { } project_id = opts.project_id; } else { - // Try to get from pool if no license and no image specified (so the default) and not cloning, - // and not "noPool". NOTE: we may improve the pool to also provide some - // basic licensed projects later, and better support for images. Maybe. - if (!src_project_id && !noPool && !license && account_id != null) { - project_id = await getFromPool({ - account_id, - title, - description, - image, - }); - if (project_id != null) { - return project_id; - } - } - project_id = v4(); } diff --git a/src/packages/server/projects/pool/all-projects.ts b/src/packages/server/projects/pool/all-projects.ts deleted file mode 100644 index 93f7a0e6d7..0000000000 --- a/src/packages/server/projects/pool/all-projects.ts +++ /dev/null @@ -1,44 +0,0 @@ -import getLogger from "@cocalc/backend/logger"; -import getPool from "@cocalc/database/pool"; -import create from "@cocalc/server/projects/create"; -import { maxAge } from "./get-project"; - -const log = getLogger("server:new-project-pool:app-projects"); - -// Get the ids of all projects in the pool. -export async function getAllProjects(): Promise { - const pool = getPool(); - const { rows } = await pool.query( - `SELECT project_id FROM projects WHERE users IS NULL AND deleted IS NULL and last_edited >= NOW() - INTERVAL '${maxAge}'` - ); - return rows.map((x) => x.project_id); -} - -// Return an array of the id's of projects that were successefully created. -// For the ones that failed just log that they failed but do not throw -// an exception. Thus this never fails, but may not actually create n projects. -export async function createProjects(n: number): Promise { - const projectPromises: Promise[] = Array.from( - { length: n }, - () => - create({ - noPool: true, // obviously don't use the pool to add projects to the pool! - title: "New Project", - description: "", - }) // create a promise for each project creation - ); - - const results = await Promise.allSettled(projectPromises); // get the results of all promises - - const successfulProjects: string[] = []; - - results.forEach((result) => { - if (result.status === "fulfilled") { - successfulProjects.push(result.value); // add to successful projects array - } else { - log.warn(`Project creation failed: ${result.reason}`); // log the failure - } - }); - - return successfulProjects; -} diff --git a/src/packages/server/projects/pool/get-project.ts b/src/packages/server/projects/pool/get-project.ts deleted file mode 100644 index cf3c20c77b..0000000000 --- a/src/packages/server/projects/pool/get-project.ts +++ /dev/null @@ -1,106 +0,0 @@ -/* -Get a new project from the new project pool. - -The main idea to avoid race conditions is to use a clever sql query -to -- in one operation -- set the users field for one available project -and return the project_id; if two hubs do this at once, the database -just assigns them different projects. - -We thus use SQL query that does the following: It searches the database to -find a row in the projects table for which users and deleted are both null -and last_edited is within the last 12 hours. If it finds one such row, -it sets the jsonb field users equal to {[account_id]: {"group": "owner"}}. -It also outputs the project_id of that row. If it doesn't find that row -it outputs no rows. (Trivial for chatgpt...) - -ORDER by created so that *oldest* project used first, since it is most -likely to be started, e.g., newest project might still be starting. - - -WITH matching_row AS ( - SELECT project_id - FROM projects - WHERE users IS NULL - AND deleted IS NULL - AND last_edited >= NOW() - INTERVAL '12 hours' - ORDER BY created - LIMIT 1 -), -updated_project AS ( - UPDATE projects - SET users = {} - WHERE project_id IN (SELECT project_id FROM matching_row) - RETURNING project_id, users -) -SELECT project_id, users -FROM updated_project; - -*/ - -import getPool from "@cocalc/database/pool"; -import { maintainNewProjectPool } from "./maintain"; -import getLogger from "@cocalc/backend/logger"; - -export const maxAge = "12 hours"; - -const log = getLogger("server:new-project-pool:get-project"); - -// input is a user and the output is a project_id -// of a running project that is set to have the account_id -// as the sole owner. Returns null if there is nothing currently -// available in the pool. -export default async function getFromPool({ - account_id, - title, - description, - image, -}: { - account_id: string; - title?: string; - description?: string; - image?: string; -}): Promise { - log.debug("getting a project from the pool for ", account_id); - - const query = `WITH matching_row AS ( - SELECT project_id - FROM projects - WHERE users IS NULL - AND deleted IS NULL ${image ? " AND compute_image=$2 " : ""} - AND last_edited >= NOW() - INTERVAL '${maxAge}' - ORDER BY created asc - LIMIT 1 -), -updated_project AS ( - UPDATE projects - SET users = jsonb_build_object($1::text, jsonb_build_object('group', 'owner')) - WHERE project_id IN (SELECT project_id FROM matching_row) - RETURNING project_id, users -) -SELECT project_id, users -FROM updated_project;`; - - const pool = getPool(); - const { rows } = await pool.query( - query, - image ? [account_id, image] : [account_id] - ); - if (rows.length == 0) { - log.debug("pool is empty, so can't get anything from pool"); - return null; - } - try { - // just removed something from pool, so refresh pool: - await maintainNewProjectPool(1); - } catch (err) { - log.warn("Error maintaining pool", err); - } - const { project_id } = rows[0]; - if (title != null || description != null) { - await pool.query( - "UPDATE projects SET title=$1, description=$2 WHERE project_id=$3", - [title ?? "", description ?? "", project_id] - ); - } - return project_id; -} diff --git a/src/packages/server/projects/pool/maintain.ts b/src/packages/server/projects/pool/maintain.ts deleted file mode 100644 index e5c224bc3b..0000000000 --- a/src/packages/server/projects/pool/maintain.ts +++ /dev/null @@ -1,83 +0,0 @@ -/* -maintainNewProjectPool - -This ensures that all projects in the pool are touched periodically, -i.e., running and not about to idle timeout. - -The definition of "in the pool" is that the project is (1) not deleted, (2) has a null "users" field, -and (3) last_edited is recent to avoid stale old projects with an old image, etc. -This is a quick query due to indexes. - -This also creates new projects to ensure there are enough in the pool. - -NOTE: Race condition -- if multiple hubs simultaneously create projects, this at once it would at worst -result in the pool being temporarily too big. -*/ - -import getLogger from "@cocalc/backend/logger"; -import { getServerSettings } from "@cocalc/database/settings/server-settings"; -import { getAllProjects, createProjects } from "./all-projects"; -import { getProject } from "@cocalc/server/projects/control"; - -const log = getLogger("server:new-project-pool:maintain"); - -const MAX_CREATE = 50; // never add more than this many projects at once to the pool. - -export default function loop(periodMs = 30000) { - setInterval(async () => { - try { - await maintainNewProjectPool(); - } catch (err) { - log.warn("error in new project pool maintenance", err); - } - }, periodMs); -} - -let lastCall = Date.now(); -export async function maintainNewProjectPool(maxCreate?: number) { - const now = Date.now(); - if (now - lastCall <= 3000) { - log.debug("skipping too frequent call to maintainNewProjectPool"); - // no matter what, never do maintenance more than once every few seconds. - return; - } - lastCall = now; - const { new_project_pool } = await getServerSettings(); - if (!new_project_pool) { - log.debug("new project pool not enabled"); - return; - } - const projects = await getAllProjects(); - const cur = projects.length; - // Add projects to the pool if necessary - for (const project_id of await createProjects( - Math.min(new_project_pool - cur, maxCreate ?? MAX_CREATE) - )) { - log.debug("adding ", project_id, "to the pool"); - projects.push(project_id); - } - - // ensure all projects are running and ready to use - log.debug( - "there are currently ", - projects.length, - "projects in the pool -- ensuring all are running" - ); - await Promise.allSettled( - projects.map(async (project_id) => { - try { - await getProject(project_id).touch(); - log.debug("touched ", project_id, "so it stays running"); - return true; - } catch (error) { - log.warn( - "Something went wrong while touching the project with id:", - project_id, - ". The error message is:", - error.message - ); - return false; - } - }) - ); -} diff --git a/src/packages/util/db-schema/projects.ts b/src/packages/util/db-schema/projects.ts index 2ab9a410da..54436638e7 100644 --- a/src/packages/util/db-schema/projects.ts +++ b/src/packages/util/db-schema/projects.ts @@ -722,9 +722,6 @@ export interface CreateProjectOptions { // (optional) license id (or multiple ids separated by commas) -- if given, project will be created with this license license?: string; public_path_id?: string; // may imply use of a license - // noPool = do not allow using the pool (e.g., need this when creating projects to put in the pool); - // not a real issue since when creating for pool account_id is null, and then we wouldn't use the pool... - noPool?: boolean; // start running the moment the project is created -- uses more resources, but possibly better user experience start?: boolean; diff --git a/src/packages/util/db-schema/site-defaults.ts b/src/packages/util/db-schema/site-defaults.ts index 93ae9926e5..9cd91341bf 100644 --- a/src/packages/util/db-schema/site-defaults.ts +++ b/src/packages/util/db-schema/site-defaults.ts @@ -115,7 +115,6 @@ export type SiteSettingsKeys = | "landing_pages" | "sandbox_projects_enabled" | "sandbox_project_id" - | "new_project_pool" | "compute_servers_enabled" | "compute_servers_google-cloud_enabled" | "compute_servers_onprem_enabled" @@ -799,13 +798,6 @@ export const site_settings_conf: SiteSettings = { desc: "The `project_id` (a UUIDv4) of a sandbox project on your server for people who visit CoCalc to play around with. This is potentially dangerous, so use with care! This project MUST have 'Sandbox' enabled in project settings, so that anybody can access it.", default: "", }, - new_project_pool: { - name: "New Project Pool", - desc: "Number of new non-upgraded running projects to have at the ready to speed up the experience of creating new projects for users in interactive settings (where they are likely to immediately open the project).", - default: "0", - valid: only_nonneg_int, - show: () => true, - }, openai_enabled: { name: "OpenAI ChatGPT UI", desc: "Controls visibility of UI elements related to OpenAI ChatGPT integration. You must **also set your OpenAI API key** below for this functionality to work.", From accc5b854f79f5543f415f0ad548c40528253851 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 15 Aug 2025 00:13:00 +0000 Subject: [PATCH 287/798] fix bug in removing redundant reps from ipynb (obviously JUPYTER_MIMETYPES[type] doesn't test for inclusion of type in an array) --- .../jupyter/ipynb/import-from-ipynb.ts | 33 +++++++++++-------- src/packages/jupyter/util/misc.ts | 4 ++- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/src/packages/jupyter/ipynb/import-from-ipynb.ts b/src/packages/jupyter/ipynb/import-from-ipynb.ts index 2761e15e02..f1ecc68d99 100644 --- a/src/packages/jupyter/ipynb/import-from-ipynb.ts +++ b/src/packages/jupyter/ipynb/import-from-ipynb.ts @@ -8,7 +8,10 @@ Importing from an ipynb object (in-memory version of .ipynb file) */ import * as misc from "@cocalc/util/misc"; -import { JUPYTER_MIMETYPES } from "@cocalc/jupyter/util/misc"; +import { + JUPYTER_MIMETYPES, + JUPYTER_MIMETYPES_SET, +} from "@cocalc/jupyter/util/misc"; const DEFAULT_IPYNB = { cells: [ @@ -404,7 +407,8 @@ export class IPynbImporter { } } -export function remove_redundant_reps(data?: any) { +// mutate data removing redundant reps +export function remove_redundant_reps(data?: object) { if (data == null) { return; } @@ -414,23 +418,26 @@ export function remove_redundant_reps(data?: any) { // backend only) for the .ipynb export, but I'm not doing that right now! // This means opening and closing an ipynb file may lose information, which // no client currently cares about (?) -- maybe nbconvert does. - let keep; + let keep = ""; for (const type of JUPYTER_MIMETYPES) { if (data[type] != null) { keep = type; break; } } - if (keep != null) { - for (const type in data) { - // NOTE: we only remove multiple reps that are both in JUPYTER_MIMETYPES; - // if there is another rep that is NOT in JUPYTER_MIMETYPES, then it is - // not removed, e.g., application/vnd.jupyter.widget-view+json and - // text/plain both are types of representation of a widget. - if (JUPYTER_MIMETYPES[type] !== undefined && type !== keep) { - delete data[type]; - } + if (!keep) { + return; + } + for (const type in data) { + // NOTE: we only remove multiple reps that are both in JUPYTER_MIMETYPES; + // if there is another rep that is NOT in JUPYTER_MIMETYPES, then it is + // not removed, e.g., application/vnd.jupyter.widget-view+json and + // text/plain both are types of representation of a widget. + if ( + type != keep && + (JUPYTER_MIMETYPES_SET as Set).has(type) != null + ) { + delete data[type]; } } - return data; } diff --git a/src/packages/jupyter/util/misc.ts b/src/packages/jupyter/util/misc.ts index 2c0c7e3a55..1efd000bd1 100644 --- a/src/packages/jupyter/util/misc.ts +++ b/src/packages/jupyter/util/misc.ts @@ -18,7 +18,7 @@ import * as immutable from "immutable"; import { cmp } from "@cocalc/util/misc"; // This list is inspired by OutputArea.output_types in https://github.com/jupyter/notebook/blob/master/notebook/static/notebook/js/outputarea.js -// The order matters -- we only keep the left-most type (see import-from-ipynb.coffee) +// The order matters -- we only keep the left-most type (see import-from-ipynb.ts) // See https://jupyterlab.readthedocs.io/en/stable/user/file_formats.html#file-and-output-formats export const JUPYTER_MIMETYPES = [ @@ -36,6 +36,8 @@ export const JUPYTER_MIMETYPES = [ "text/plain", ] as const; +export const JUPYTER_MIMETYPES_SET = new Set(JUPYTER_MIMETYPES); + // with metadata.cocalc.priority >= this the kernel will be "emphasized" or "suggested" in the UI export const KERNEL_POPULAR_THRESHOLD = 10; From 276093829284a90e97c8ef02b7cd2e5cd2c70a6f Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 15 Aug 2025 00:37:04 +0000 Subject: [PATCH 288/798] jupyter: fix bug that completely broke rendering of svg in jupyter notebooks - this is also broken in prod; i'm fixing this only in this branch... --- src/packages/jupyter/control.ts | 1 - src/packages/jupyter/kernel/kernel.ts | 14 +++++++++----- src/packages/jupyter/util/misc.ts | 17 +++++++++++++++++ 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts index 1ae9e3533f..b59eb8d1bb 100644 --- a/src/packages/jupyter/control.ts +++ b/src/packages/jupyter/control.ts @@ -182,7 +182,6 @@ class MulticellOutputHandler { }; } - export function outputHandler({ path, cells }: RunOptions) { if (jupyterActions[ipynbPath(path)] == null) { throw Error(`session '${ipynbPath(path)}' not available`); diff --git a/src/packages/jupyter/kernel/kernel.ts b/src/packages/jupyter/kernel/kernel.ts index e37b30d501..564d839013 100644 --- a/src/packages/jupyter/kernel/kernel.ts +++ b/src/packages/jupyter/kernel/kernel.ts @@ -45,7 +45,10 @@ import { } from "@cocalc/jupyter/types/project-interface"; import { JupyterActions } from "@cocalc/jupyter/redux/project-actions"; import { JupyterStore } from "@cocalc/jupyter/redux/store"; -import { JUPYTER_MIMETYPES } from "@cocalc/jupyter/util/misc"; +import { + JUPYTER_MIMETYPES, + isJupyterBase64MimeType, +} from "@cocalc/jupyter/util/misc"; import { isSha1 } from "@cocalc/util/misc"; import type { SyncDB } from "@cocalc/sync/editor/db/sync"; import { retry_until_success, until } from "@cocalc/util/async-utils"; @@ -786,9 +789,10 @@ export class JupyterKernel if (this._actions == null) { throw Error("blob store not available"); } - const buf: Buffer = !type.startsWith("text/") - ? Buffer.from(data, "base64") - : Buffer.from(data); + const buf = Buffer.from( + data, + isJupyterBase64MimeType(type) ? "base64" : undefined, + ); const sha1: string = misc_node_sha1(buf); await this._actions.asyncBlobStore.set(sha1, buf); @@ -835,7 +839,6 @@ export class JupyterKernel type === "application/pdf" || type === "text/html" ) { - dbg("removing ", type); // Store all images and PDF and text/html in a binary blob store, so we don't have // to involve it in realtime sync. It tends to be large, etc. if (isSha1(content.data[type])) { @@ -845,6 +848,7 @@ export class JupyterKernel } const sha1 = await saveBlob(content.data[type], type); if (sha1) { + dbg("put content in blob store: ", { type, sha1 }); // only remove if the save actually worked -- we don't want to break output // for the user for a little optimization! if (type == "text/html") { diff --git a/src/packages/jupyter/util/misc.ts b/src/packages/jupyter/util/misc.ts index 1efd000bd1..cff641b391 100644 --- a/src/packages/jupyter/util/misc.ts +++ b/src/packages/jupyter/util/misc.ts @@ -38,6 +38,23 @@ export const JUPYTER_MIMETYPES = [ export const JUPYTER_MIMETYPES_SET = new Set(JUPYTER_MIMETYPES); +export function isJupyterBase64MimeType(type: string) { + type = type.toLowerCase(); + if (type.startsWith("text")) { + // no text ones are base64 encoded + return false; + } + if (type == "application/json" || type == "application/javascript") { + return false; + } + if (type == "image/svg+xml") { + return false; + } + + // what remains should be application/pdf and the image types + return true; +} + // with metadata.cocalc.priority >= this the kernel will be "emphasized" or "suggested" in the UI export const KERNEL_POPULAR_THRESHOLD = 10; From e4872ae3fca949483068be1b0d5f8c2781743437 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 15 Aug 2025 00:46:41 +0000 Subject: [PATCH 289/798] put a slight delay in when restarting project --- src/packages/frontend/projects/actions.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/packages/frontend/projects/actions.ts b/src/packages/frontend/projects/actions.ts index 3c98cc7e15..c653839beb 100644 --- a/src/packages/frontend/projects/actions.ts +++ b/src/packages/frontend/projects/actions.ts @@ -32,6 +32,7 @@ import { Upgrades } from "@cocalc/util/upgrades/types"; import { ProjectsState, store } from "./store"; import { load_all_projects, switch_to_project } from "./table"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { delay } from "awaiting"; import type { CourseInfo, @@ -992,7 +993,9 @@ export class ProjectsActions extends Actions { }); const state = store.get_state(project_id); if (state == "running") { - await this.stop_project(project_id); + await this.stop_project(project_id); + // [ ] TODO: this delay is a temporary workaround + await delay(1000); } await this.start_project(project_id, options); }, From a26fc28304cc9b1faef2f79a13c8323559efe421 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 15 Aug 2025 03:11:42 +0000 Subject: [PATCH 290/798] expose named server control via the conat project api --- src/packages/conat/project/api/system.ts | 14 ++++++ src/packages/project/conat/api/system.ts | 6 +++ src/packages/project/named-servers/control.ts | 44 +++++++++++++------ src/packages/project/named-servers/index.ts | 2 +- 4 files changed, 52 insertions(+), 14 deletions(-) diff --git a/src/packages/conat/project/api/system.ts b/src/packages/conat/project/api/system.ts index a2fbfa9dc1..9b513c3382 100644 --- a/src/packages/conat/project/api/system.ts +++ b/src/packages/conat/project/api/system.ts @@ -8,6 +8,7 @@ import type { ConfigurationAspect, } from "@cocalc/comm/project-configuration"; import { type ProjectJupyterApiOptions } from "@cocalc/util/jupyter/api-types"; +import type { NamedServerName } from "@cocalc/util/types/servers"; export const system = { terminate: true, @@ -33,6 +34,10 @@ export const system = { // jupyter stateless API jupyterExecute: true, + + // named servers like jupyterlab, vscode, etc. + startNamedServer: true, + statusOfNamedServer: true, }; export interface System { @@ -72,4 +77,13 @@ export interface System { }) => Promise; jupyterExecute: (opts: ProjectJupyterApiOptions) => Promise; + + startNamedServer: ( + name: NamedServerName, + ) => Promise<{ port: number; url: string }>; + statusOfNamedServer: ( + name: NamedServerName, + ) => Promise< + { state: "running"; port: number; url: string } | { state: "stopped" } + >; } diff --git a/src/packages/project/conat/api/system.ts b/src/packages/project/conat/api/system.ts index 3750ac32ee..2af212f9bd 100644 --- a/src/packages/project/conat/api/system.ts +++ b/src/packages/project/conat/api/system.ts @@ -101,3 +101,9 @@ export async function signal({ import jupyterExecute from "@cocalc/jupyter/stateless-api/execute"; export { jupyterExecute }; + +import { + start as startNamedServer, + status as statusOfNamedServer, +} from "@cocalc/project/named-servers/control"; +export { startNamedServer, statusOfNamedServer }; diff --git a/src/packages/project/named-servers/control.ts b/src/packages/project/named-servers/control.ts index ff20884db7..f044e1152a 100644 --- a/src/packages/project/named-servers/control.ts +++ b/src/packages/project/named-servers/control.ts @@ -7,24 +7,40 @@ import getPort from "get-port"; import { exec } from "node:child_process"; import { mkdir, readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; - import basePath from "@cocalc/backend/base-path"; import { data } from "@cocalc/backend/data"; import { project_id } from "@cocalc/project/data"; import { INFO } from "@cocalc/project/info-json"; import { getLogger } from "@cocalc/project/logger"; -import { NamedServerName } from "@cocalc/util/types/servers"; +import { + type NamedServerName, + NAMED_SERVER_NAMES, +} from "@cocalc/util/types/servers"; import getSpec from "./list"; const winston = getLogger("named-servers:control"); +function assertNamedServer(name: string) { + if (typeof name != "string" || !NAMED_SERVER_NAMES.includes(name as any)) { + throw Error(`the named servers are: ${NAMED_SERVER_NAMES.join(", ")}`); + } +} + +function getBase(name: NamedServerName): string { + const baseType = name === "rserver" ? "server" : "port"; + return join(basePath, `/${project_id}/${baseType}/${name}`); +} + // Returns the port or throws an exception. -export async function start(name: NamedServerName): Promise { +export async function start( + name: NamedServerName, +): Promise<{ port: number; url: string }> { + assertNamedServer(name); winston.debug(`start ${name}`); const s = await status(name); - if (s.status === "running") { + if (s.state === "running") { winston.debug(`${name} is already running`); - return s.port; + return { port: s.port, url: s.url }; } const port = await getPort({ port: preferredPort(name) }); // For servers that need a basePath, they will use this one. @@ -35,9 +51,8 @@ export async function start(name: NamedServerName): Promise { if (ip === "localhost") { ip = "127.0.0.1"; } - // TODO that baseType should come from named-server-panel:SPEC[name].usesBasePath - const baseType = name === "rserver" ? "server" : "port"; - const base = join(basePath, `/${project_id}/${baseType}/${name}`); + + const base = getBase(name); const cmd = await getCommand(name, ip, port, base); winston.debug(`will start ${name} by running "${cmd}"`); @@ -47,7 +62,7 @@ export async function start(name: NamedServerName): Promise { const child = exec(cmd, { cwd: process.env.HOME }); await writeFile(p.pid, `${child.pid}`); - return port; + return { port, url: base }; } async function getCommand( @@ -62,10 +77,13 @@ async function getCommand( return `${cmd} 1>${stdout} 2>${stderr}`; } -// Returns the status and port (if defined). +// Returns the state and port (if defined). export async function status( name: NamedServerName, -): Promise<{ status: "running"; port: number } | { status: "stopped" }> { +): Promise< + { state: "running"; port: number; url: string } | { state: "stopped" } +> { + assertNamedServer(name); const { pid, port } = await paths(name); try { const pidValue = parseInt((await readFile(pid)).toString()); @@ -78,10 +96,10 @@ export async function status( if (!Number.isInteger(portValue)) { throw Error("invalid port"); } - return { status: "running", port: portValue }; + return { state: "running", port: portValue, url: getBase(name) }; } catch (_err) { // it's not running or the port isn't valid - return { status: "stopped" }; + return { state: "stopped" }; } } diff --git a/src/packages/project/named-servers/index.ts b/src/packages/project/named-servers/index.ts index a6644d95ff..c007676f1b 100644 --- a/src/packages/project/named-servers/index.ts +++ b/src/packages/project/named-servers/index.ts @@ -13,7 +13,7 @@ const winston = getLogger("named-servers"); async function getPort(name: NamedServerName): Promise { winston.debug(`getPort("${name}")`); - return await start(name); + return (await start(name)).port; } async function handleMessage(socket, mesg): Promise { From 8c49068fa3786091c092cec0814d9b7398a930ff Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 15 Aug 2025 04:04:17 +0000 Subject: [PATCH 291/798] switch hub proxy server to use conat to get named server port --- src/packages/hub/proxy/target.ts | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/packages/hub/proxy/target.ts b/src/packages/hub/proxy/target.ts index 9a3729f88f..0608fac91b 100644 --- a/src/packages/hub/proxy/target.ts +++ b/src/packages/hub/proxy/target.ts @@ -7,7 +7,6 @@ to this target or the target project isn't running. */ import LRU from "lru-cache"; - import getLogger from "@cocalc/hub/logger"; import { database } from "@cocalc/hub/servers/database"; import { ProjectControlFunction } from "@cocalc/server/projects/control"; @@ -15,8 +14,7 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { NamedServerName } from "@cocalc/util/types/servers"; import hasAccess from "./check-for-access-to-project"; import { parseReq } from "./parse"; - -const hub_projects = require("../projects"); +import { projectApiClient } from "@cocalc/conat/project/api"; const logger = getLogger("proxy:target"); @@ -144,7 +142,7 @@ export async function getTarget({ port = parseInt(port_desc); if (!Number.isInteger(port)) { dbg("determining name=", port_desc, "server port..."); - port = await namedServerPort(project_id, port_desc, projectControl); + port = await namedServerPort(project_id, port_desc); dbg("got named server name=", port_desc, " port=", port); } } else if (type === "raw") { @@ -177,21 +175,17 @@ const namedServerPortCache = new LRU({ async function _namedServerPort( project_id: string, name: NamedServerName, - projectControl, ): Promise { const key = project_id + name; const p = namedServerPortCache.get(key); if (p) { return p; } - const project = hub_projects.new_project( - // NOT @cocalc/server/projects/control like above... - project_id, - database, - projectControl, - ); - const port = await project.named_server_port(name); - namedServerPortCache.set(key, port); + // TODO: to use nsjail instead of k8s to run projects in a scalable way, + // we are going to have also get the host ip address a port mapping + // right here. + const api = projectApiClient({ project_id }); + const { port } = await api.system.startNamedServer(name); return port; } From a329a43e23aee625c158d22ca08471ee300ccbc2 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 15 Aug 2025 04:10:53 +0000 Subject: [PATCH 292/798] remove old named_server_port code --- src/packages/hub/projects.coffee | 6 ---- src/packages/project/named-servers/index.ts | 32 ------------------- src/packages/project/package.json | 1 - .../project/servers/hub/handle-message.ts | 5 --- .../server/projects/connection/call.ts | 1 - src/packages/util/message.d.ts | 1 - src/packages/util/message.js | 13 -------- 7 files changed, 59 deletions(-) delete mode 100644 src/packages/project/named-servers/index.ts diff --git a/src/packages/hub/projects.coffee b/src/packages/hub/projects.coffee index aed8cdb7ee..7a586abbaa 100644 --- a/src/packages/hub/projects.coffee +++ b/src/packages/hub/projects.coffee @@ -86,12 +86,6 @@ class Project opts.mesg.project_id = @project_id @local_hub.call(opts) - # async function - named_server_port: (name) => - @dbg("named_server_port(name=#{name})") - resp = await callback2(@call, {mesg : message.named_server_port(name:name), timeout : 30}) - @dbg("named_server_port #{resp.port}") - return resp.port read_file: (opts) => @dbg("read_file") diff --git a/src/packages/project/named-servers/index.ts b/src/packages/project/named-servers/index.ts deleted file mode 100644 index c007676f1b..0000000000 --- a/src/packages/project/named-servers/index.ts +++ /dev/null @@ -1,32 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { getLogger } from "@cocalc/project/logger"; -import * as message from "@cocalc/util/message"; -import { NamedServerName } from "@cocalc/util/types/servers"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { start } from "./control"; - -const winston = getLogger("named-servers"); - -async function getPort(name: NamedServerName): Promise { - winston.debug(`getPort("${name}")`); - return (await start(name)).port; -} - -async function handleMessage(socket, mesg): Promise { - try { - mesg.port = await getPort(mesg.name); - } catch (err) { - socket.write_mesg("json", message.error({ id: mesg.id, error: `${err}` })); - return; - } - socket.write_mesg("json", mesg); -} - -const handle = reuseInFlight(handleMessage, { - createKey: (args) => `${args[1]?.name}`, -}); -export default handle; diff --git a/src/packages/project/package.json b/src/packages/project/package.json index 7ae58a436d..a687db3224 100644 --- a/src/packages/project/package.json +++ b/src/packages/project/package.json @@ -3,7 +3,6 @@ "version": "1.40.0", "description": "CoCalc: project daemon", "exports": { - "./named-servers": "./dist/named-servers/index.js", "./conat": "./dist/conat/index.js", "./conat/terminal": "./dist/conat/terminal/index.js", "./*": "./dist/*.js", diff --git a/src/packages/project/servers/hub/handle-message.ts b/src/packages/project/servers/hub/handle-message.ts index f9c13c23d7..4ad363faea 100644 --- a/src/packages/project/servers/hub/handle-message.ts +++ b/src/packages/project/servers/hub/handle-message.ts @@ -18,7 +18,6 @@ import { exec_shell_code } from "@cocalc/project/exec_shell_code"; import { get_kernel_data } from "@cocalc/jupyter/kernel/kernel-data"; import jupyterExecute from "@cocalc/jupyter/stateless-api/execute"; import { getLogger } from "@cocalc/project/logger"; -import handleNamedServer from "@cocalc/project/named-servers"; import { print_to_pdf } from "@cocalc/project/print_to_pdf"; import { read_file_from_project, @@ -63,10 +62,6 @@ export default async function handleMessage( socket.write_mesg("json", message.pong({ id: mesg.id })); return; - case "named_server_port": - handleNamedServer(socket, mesg); - return; - case "project_exec": // this is no longer used by web browser clients; however it *is* used by the HTTP api served // by the hub to api key users, so do NOT remove it! E.g., the latex endpoint, the compute diff --git a/src/packages/server/projects/connection/call.ts b/src/packages/server/projects/connection/call.ts index 53bdfb9bd8..b9182ccc39 100644 --- a/src/packages/server/projects/connection/call.ts +++ b/src/packages/server/projects/connection/call.ts @@ -13,7 +13,6 @@ and they include: - ping: for testing; returns a pong - heartbeat: used for maintaining the connection -- named_server_port: finding out the port used by jupyter, jupyterlab, etc. - project_exec: run shell command - read_file_from_project: reads file and stores it as a blob in the database. blob expires in 24 hours. - write_file_to_project: write abitrary file to disk in project (goes via a blob) diff --git a/src/packages/util/message.d.ts b/src/packages/util/message.d.ts index 2682106b9a..d9a494b25c 100644 --- a/src/packages/util/message.d.ts +++ b/src/packages/util/message.d.ts @@ -22,7 +22,6 @@ export const open_project: any; export const project_opened: any; export const project_exec: any; export const project_exec_output: any; -export const named_server_port: any; export const read_file_from_project: any; export const file_read_from_project: any; export const read_text_file_from_project: any; diff --git a/src/packages/util/message.js b/src/packages/util/message.js index 5aea5143a8..690e3aca51 100644 --- a/src/packages/util/message.js +++ b/src/packages/util/message.js @@ -659,19 +659,6 @@ message({ stats: undefined, }); -//##################################################################### -// Named Server -//##################################################################### - -// starts a named server in a project, e.g, 'jupyterlab', and reports the -// port it is running at -// hub <--> project -message({ - event: "named_server_port", - name: required, // 'jupyter', 'jupyterlab', 'code', 'pluto' or whatever project supports... - port: undefined, // gets set in the response - id: undefined, -}); //############################################################################ From a0605c9bfdc0a4f6461e3ef7a51447c7f484e67e Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 15 Aug 2025 13:24:25 +0000 Subject: [PATCH 293/798] no need to chown when starting project, since project runner now runs as a normal user --- src/packages/server/conat/project/run.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/packages/server/conat/project/run.ts b/src/packages/server/conat/project/run.ts index 0c9abdfa4c..d12e2e3ded 100644 --- a/src/packages/server/conat/project/run.ts +++ b/src/packages/server/conat/project/run.ts @@ -34,7 +34,6 @@ import { root } from "@cocalc/backend/data"; import { dirname, join } from "node:path"; import { userInfo } from "node:os"; import { - chown, ensureConfFilesExists, getEnvironment, homePath, @@ -142,7 +141,6 @@ async function start({ const home = await homePath(project_id); await mkdir(home, { recursive: true }); - await chown(home, uid); await ensureConfFilesExists(home, uid); const env = await getEnvironment( project_id, From 869001bb3b20cf35d7c58a16ae8a12ff66b11ece Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 15 Aug 2025 14:41:02 +0000 Subject: [PATCH 294/798] re-implement jupyter stateless api for frontend using conat - entirely got rid of database cache -- it adds complexity and confusion for little value. This does mean output disappears when page is refreshed; longterm any output should be stored in the document, not the database. - doesn't work for nextjs/share server yet. --- src/packages/conat/project/api/jupyter.ts | 6 + src/packages/conat/project/api/system.ts | 6 - .../frontend/components/run-button/index.tsx | 175 ++++++------------ .../components/run-button/kernel-info.ts | 15 +- .../frontend/components/run-button/output.tsx | 4 +- .../components/run-button/select-kernel.tsx | 1 - .../slate/elements/code-block/editable.tsx | 1 - .../slate/elements/code-block/index.tsx | 1 - .../elements/code/kernel.tsx | 10 +- .../frontend/jupyter/nbviewer/cell-input.tsx | 1 - src/packages/jupyter/stateless-api/execute.ts | 2 +- src/packages/project/conat/api/jupyter.ts | 3 + src/packages/project/conat/api/system.ts | 3 - 13 files changed, 80 insertions(+), 148 deletions(-) diff --git a/src/packages/conat/project/api/jupyter.ts b/src/packages/conat/project/api/jupyter.ts index 22f7e46211..38fc8b91b2 100644 --- a/src/packages/conat/project/api/jupyter.ts +++ b/src/packages/conat/project/api/jupyter.ts @@ -1,6 +1,7 @@ import type { NbconvertParams } from "@cocalc/util/jupyter/types"; import type { RunNotebookOptions } from "@cocalc/util/jupyter/nbgrader-types"; import type { KernelSpec } from "@cocalc/util/jupyter/types"; +import { type ProjectJupyterApiOptions } from "@cocalc/util/jupyter/api-types"; export const jupyter = { start: true, @@ -14,6 +15,9 @@ export const jupyter = { complete: true, signal: true, getConnectionFile: true, + + // jupyter stateless API + apiExecute: true, }; // In the functions below path can be either the .ipynb or the .sage-jupyter2 path, and @@ -52,4 +56,6 @@ export interface Jupyter { getConnectionFile: (opts: { path: string }) => Promise; signal: (opts: { path: string; signal: string }) => Promise; + + apiExecute: (opts: ProjectJupyterApiOptions) => Promise; } diff --git a/src/packages/conat/project/api/system.ts b/src/packages/conat/project/api/system.ts index 9b513c3382..b7d509e33d 100644 --- a/src/packages/conat/project/api/system.ts +++ b/src/packages/conat/project/api/system.ts @@ -7,7 +7,6 @@ import type { Configuration, ConfigurationAspect, } from "@cocalc/comm/project-configuration"; -import { type ProjectJupyterApiOptions } from "@cocalc/util/jupyter/api-types"; import type { NamedServerName } from "@cocalc/util/types/servers"; export const system = { @@ -32,9 +31,6 @@ export const system = { signal: true, - // jupyter stateless API - jupyterExecute: true, - // named servers like jupyterlab, vscode, etc. startNamedServer: true, statusOfNamedServer: true, @@ -76,8 +72,6 @@ export interface System { pid?: number; }) => Promise; - jupyterExecute: (opts: ProjectJupyterApiOptions) => Promise; - startNamedServer: ( name: NamedServerName, ) => Promise<{ port: number; url: string }>; diff --git a/src/packages/frontend/components/run-button/index.tsx b/src/packages/frontend/components/run-button/index.tsx index 657b01fa84..70e1a80520 100644 --- a/src/packages/frontend/components/run-button/index.tsx +++ b/src/packages/frontend/components/run-button/index.tsx @@ -13,25 +13,20 @@ import { useState, } from "react"; import TimeAgo from "react-timeago"; - -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -//import { file_associations } from "@cocalc/frontend/file-associations"; -//import OpenAIAvatar from "@cocalc/frontend/components/openai-avatar"; +import { projectApiClient } from "@cocalc/conat/project/api"; import { Icon } from "@cocalc/frontend/components/icon"; import infoToMode from "@cocalc/frontend/editors/slate/elements/code-block/info-to-mode"; import { CodeMirrorStatic } from "@cocalc/frontend/jupyter/codemirror-static"; import Logo from "@cocalc/frontend/jupyter/logo"; import "@cocalc/frontend/jupyter/output-messages/mime-types/init-nbviewer"; import { useFileContext } from "@cocalc/frontend/lib/file-context"; -import computeHash from "@cocalc/util/jupyter-api/compute-hash"; import { path_split, plural } from "@cocalc/util/misc"; import { COLORS } from "@cocalc/util/theme"; -import api from "./api"; -import { getFromCache, saveToCache } from "./cache"; import getKernel from "./get-kernel"; import { kernelDisplayName, kernelLanguage } from "./kernel-info"; import Output from "./output"; import SelectKernel from "./select-kernel"; +import LRU from "lru-cache"; // ATTN[i18n]: it's tempting to translate this, but it is a dependency of next (vouchers/notes → slate/code-block → buttons) @@ -48,11 +43,8 @@ export interface Props { runRef?: RunRef; tag?: string; size; - // automatically check for known output in database on initial load, e.g., - // yes for markdown, but not for a jupyter notebook on the share server. - auto?: boolean; - setInfo?: (info: string) => void; + timeout?: number; } // definitely never show run buttons for text formats that can't possibly be run. @@ -81,8 +73,8 @@ export default function RunButton({ runRef, tag, size, - auto, setInfo, + timeout = 30_000, }: Props) { const mode = infoToMode(info); const noRun = NO_RUN.has(mode); @@ -92,7 +84,6 @@ export default function RunButton({ project_id, path: filename, is_visible, - /*hasOpenAI, */ } = useFileContext(); const path = project_id && filename ? path_split(filename).head : undefined; const [running, setRunning] = useState(false); @@ -133,88 +124,44 @@ export default function RunButton({ const [kernelName, setKernelName] = useState(undefined); const [showPopover, setShowPopover] = useState(false); + // determine the kernel useEffect(() => { if ( noRun || - !jupyterApiEnabled || + (!project_id && !jupyterApiEnabled) || setOutput == null || - running || - !info.trim() - ) + running + ) { return; - const { output: messages, kernel: usedKernel } = getFromCache({ - input, - history, - info, - project_id, - path, - }); - if (!info) { - setKernelName(undefined); } + setOutput({ old: true }); + (async () => { + let kernel; + try { + kernel = await getKernel({ input, history, info, project_id }); + } catch (err) { + // could fail, e.g., if user not signed in. shouldn't be fatal. + console.warn(`WARNING: ${err}`); + return; + } + setKernelName(kernel); + })(); + }, [input, history, info]); + + // set output based on local cache (so unmount/mount doesn't clear output) + useEffect(() => { + const messages = getFromCache({ input, history, info }); if (messages != null) { setOutput({ messages }); - setKernelName(usedKernel); - } else { - setOutput({ old: true }); - // but we try to asynchronously get the output from the - // backend, if available - (async () => { - let kernel; - try { - kernel = await getKernel({ input, history, info, project_id }); - } catch (err) { - // could fail, e.g., if user not signed in. shouldn't be fatal. - console.warn(`WARNING: ${err}`); - return; - } - setKernelName(kernel); - if (!auto && outputMessagesRef.current == null) { - // we don't initially automatically check database since auto is false. - return; - } - - const hash = computeHash({ - input, - history, - kernel, - project_id, - path, - }); - let x; - try { - x = await getFromDatabaseCache(hash); - } catch (err) { - console.warn(`WARNING: ${err}`); - return; - } - const { output: messages, created } = x; - if (messages != null) { - saveToCache({ - input, - history, - info, - output: messages, - project_id, - path, - kernel, - }); - setOutput({ messages }); - setCreated(created); - } - })(); } - }, [input, history, info]); + }, []); if (noRun || (!jupyterApiEnabled && !project_id)) { - // run button is not enabled when no project_id given, or not info at all. + // run button is not enabled when no project_id given, or no info at all. return null; } - const run = async ({ - noCache, - forceKernel, - }: { noCache?: boolean; forceKernel?: string } = {}) => { + const run = async ({ forceKernel }: { forceKernel?: string } = {}) => { try { setRunning(true); setOutput({ running: true }); @@ -234,42 +181,33 @@ export default function RunButton({ } setKernelName(kernel); } - let resp; + let messages; try { if (!kernel) { setOutput({ error: "Select a Kernel" }); return; } - resp = await api("execute", { + let api; + if (project_id) { + api = projectApiClient({ project_id, timeout }); + } else { + throw Error("not implemented"); + } + messages = await api.jupyter.apiExecute({ input, history, kernel, - noCache, project_id, path, tag, }); + saveInCache({ input, history, info, messages }); } catch (err) { - if (resp?.error != null) { - setOutput({ error: resp.error }); - } else { - setOutput({ error: `Timeout or communication problem` }); - } + setOutput({ error: `${err}` }); return; } - if (resp.output != null) { - setOutput({ messages: resp.output }); - setCreated(resp.created); - saveToCache({ - input, - history, - info, - output: resp.output, - project_id, - path, - kernel, - }); - } + setOutput({ messages }); + setCreated(new Date()); } catch (error) { setOutput({ error }); } finally { @@ -297,7 +235,7 @@ export default function RunButton({ disabled={disabled || !kernelName} onClick={() => { setShowPopover(false); - run({ noCache: false }); + run(); }} > { setShowPopover(false); - run({ noCache: true }); + run(); }} > - {/*hasOpenAI && ( - - )*/}
); } -type GetFromCache = (hash: string) => Promise<{ - output?: object[]; - created?: Date; -}>; +const outputCache = new LRU({ max: 200 }); + +function cacheKey({ input, history, info }) { + return JSON.stringify([input, history, info]); +} + +function saveInCache({ input, history, info, messages }) { + outputCache.set(cacheKey({ input, history, info }), messages); +} -const getFromDatabaseCache: GetFromCache = reuseInFlight( - async (hash) => await api("execute", { hash }), -); +function getFromCache({ input, history, info }) { + return outputCache.get(cacheKey({ input, history, info })); +} diff --git a/src/packages/frontend/components/run-button/kernel-info.ts b/src/packages/frontend/components/run-button/kernel-info.ts index b3b9b3cff0..2cf7c05b1c 100644 --- a/src/packages/frontend/components/run-button/kernel-info.ts +++ b/src/packages/frontend/components/run-button/kernel-info.ts @@ -4,10 +4,9 @@ */ import LRU from "lru-cache"; - import type { KernelSpec } from "@cocalc/jupyter/types"; import { capitalize } from "@cocalc/util/misc"; -import api from "./api"; +import { projectApiClient } from "@cocalc/conat/project/api"; const kernelInfoCache = new LRU({ ttl: 30000, @@ -44,10 +43,14 @@ export async function getKernelInfo( throw new Error("No information, because project is not running"); } - const { kernels } = await api( - "kernels", - project_id ? { project_id } : undefined, - ); + // TODO: compute server support -- would select here + let api; + if (project_id) { + api = projectApiClient({ project_id }); + } else { + throw Error("not implemented"); + } + const kernels = await api.jupyter.kernels(); if (kernels == null) { throw Error("bug"); } diff --git a/src/packages/frontend/components/run-button/output.tsx b/src/packages/frontend/components/run-button/output.tsx index 1476001852..2bc6bf0e5e 100644 --- a/src/packages/frontend/components/run-button/output.tsx +++ b/src/packages/frontend/components/run-button/output.tsx @@ -31,7 +31,7 @@ export default function Output({ } return (
- {running && } + {running && } {output != null && (
diff --git a/src/packages/frontend/components/run-button/select-kernel.tsx b/src/packages/frontend/components/run-button/select-kernel.tsx index c107bbef9d..47bb665873 100644 --- a/src/packages/frontend/components/run-button/select-kernel.tsx +++ b/src/packages/frontend/components/run-button/select-kernel.tsx @@ -8,7 +8,6 @@ import { OptionProps } from "antd/es/select"; import { fromJS } from "immutable"; import { sortBy } from "lodash"; import { useEffect, useState } from "react"; - import Logo from "@cocalc/frontend/jupyter/logo"; import type { KernelSpec } from "@cocalc/jupyter/types"; import { diff --git a/src/packages/frontend/editors/slate/elements/code-block/editable.tsx b/src/packages/frontend/editors/slate/elements/code-block/editable.tsx index ba3c876e10..4a1ed439a8 100644 --- a/src/packages/frontend/editors/slate/elements/code-block/editable.tsx +++ b/src/packages/frontend/editors/slate/elements/code-block/editable.tsx @@ -129,7 +129,6 @@ function Element({ attributes, children, element }: RenderElementProps) { )} {!disableMarkdownCodebar && ( = ({ )} ; } diff --git a/src/packages/frontend/jupyter/nbviewer/cell-input.tsx b/src/packages/frontend/jupyter/nbviewer/cell-input.tsx index 17ea13726c..95dbfb8a5d 100644 --- a/src/packages/frontend/jupyter/nbviewer/cell-input.tsx +++ b/src/packages/frontend/jupyter/nbviewer/cell-input.tsx @@ -94,7 +94,6 @@ export default function CellInput({ )} Date: Fri, 15 Aug 2025 15:44:49 +0000 Subject: [PATCH 295/798] rewrite next api support for stateless jupyter api to use conat --- src/packages/conat/core/client.ts | 2 +- .../frontend/components/run-button/index.tsx | 17 +- .../components/run-button/kernel-info.ts | 11 +- .../next/components/landing/header.tsx | 1 - .../next/pages/api/v2/jupyter/execute.ts | 16 +- src/packages/next/pages/features/python.tsx | 2 +- .../project/servers/hub/handle-message.ts | 41 ----- src/packages/server/jupyter/abuse.ts | 75 -------- src/packages/server/jupyter/execute.ts | 167 ++---------------- src/packages/server/jupyter/kernels.ts | 19 +- src/packages/util/message.d.ts | 4 - src/packages/util/message.js | 42 ----- 12 files changed, 37 insertions(+), 360 deletions(-) delete mode 100644 src/packages/server/jupyter/abuse.ts diff --git a/src/packages/conat/core/client.ts b/src/packages/conat/core/client.ts index cdf1a9f89e..d276b84e21 100644 --- a/src/packages/conat/core/client.ts +++ b/src/packages/conat/core/client.ts @@ -483,7 +483,7 @@ export class Client extends EventEmitter { if (!options.address) { if (!process.env.CONAT_SERVER) { throw Error( - "Must specificy address or set CONAT_SERVER environment variable", + "Must specify address or set CONAT_SERVER environment variable", ); } options = { ...options, address: process.env.CONAT_SERVER }; diff --git a/src/packages/frontend/components/run-button/index.tsx b/src/packages/frontend/components/run-button/index.tsx index 70e1a80520..c7b6ba1e8b 100644 --- a/src/packages/frontend/components/run-button/index.tsx +++ b/src/packages/frontend/components/run-button/index.tsx @@ -27,6 +27,7 @@ import { kernelDisplayName, kernelLanguage } from "./kernel-info"; import Output from "./output"; import SelectKernel from "./select-kernel"; import LRU from "lru-cache"; +import nextApi from "./api"; // ATTN[i18n]: it's tempting to translate this, but it is a dependency of next (vouchers/notes → slate/code-block → buttons) @@ -187,20 +188,20 @@ export default function RunButton({ setOutput({ error: "Select a Kernel" }); return; } - let api; - if (project_id) { - api = projectApiClient({ project_id, timeout }); - } else { - throw Error("not implemented"); - } - messages = await api.jupyter.apiExecute({ + const opts = { input, history, kernel, project_id, path, tag, - }); + }; + if (project_id) { + const api = projectApiClient({ project_id, timeout }); + messages = await api.jupyter.apiExecute(opts); + } else { + ({ output: messages } = await nextApi("execute", opts)); + } saveInCache({ input, history, info, messages }); } catch (err) { setOutput({ error: `${err}` }); diff --git a/src/packages/frontend/components/run-button/kernel-info.ts b/src/packages/frontend/components/run-button/kernel-info.ts index 2cf7c05b1c..046993ab90 100644 --- a/src/packages/frontend/components/run-button/kernel-info.ts +++ b/src/packages/frontend/components/run-button/kernel-info.ts @@ -7,6 +7,7 @@ import LRU from "lru-cache"; import type { KernelSpec } from "@cocalc/jupyter/types"; import { capitalize } from "@cocalc/util/misc"; import { projectApiClient } from "@cocalc/conat/project/api"; +import nextApi from "./api"; const kernelInfoCache = new LRU({ ttl: 30000, @@ -43,14 +44,14 @@ export async function getKernelInfo( throw new Error("No information, because project is not running"); } - // TODO: compute server support -- would select here - let api; + let kernels; if (project_id) { - api = projectApiClient({ project_id }); + // TODO: compute server support -- would select here + const api = projectApiClient({ project_id }); + kernels = await api.jupyter.kernels(); } else { - throw Error("not implemented"); + ({ kernels } = await nextApi("kernels")); } - const kernels = await api.jupyter.kernels(); if (kernels == null) { throw Error("bug"); } diff --git a/src/packages/next/components/landing/header.tsx b/src/packages/next/components/landing/header.tsx index d9ec8ff645..fcc2954bac 100644 --- a/src/packages/next/components/landing/header.tsx +++ b/src/packages/next/components/landing/header.tsx @@ -6,7 +6,6 @@ import { Button, Layout, Tooltip } from "antd"; import Link from "next/link"; import { join } from "path"; - import { Icon } from "@cocalc/frontend/components/icon"; import { IS_MOBILE } from "@cocalc/frontend/feature"; import { SoftwareEnvNames } from "@cocalc/util/consts/software-envs"; diff --git a/src/packages/next/pages/api/v2/jupyter/execute.ts b/src/packages/next/pages/api/v2/jupyter/execute.ts index 674a21f4e8..276f77eb4e 100644 --- a/src/packages/next/pages/api/v2/jupyter/execute.ts +++ b/src/packages/next/pages/api/v2/jupyter/execute.ts @@ -7,14 +7,6 @@ The INPUT parameters are: - history: list of previous inputs as string (in order) that were sent to the kernel. - input: a new input -ALTERNATIVELY, can just give: - -- hash: hash of kernel/history/input - -and if output is known it is returned. Otherwise, nothing happens. -We are trusting that there aren't hash collisions for this applications, -since we're using a sha1 hash. - The OUTPUT is: - a list of messages that describe the output of the last code execution. @@ -23,7 +15,6 @@ The OUTPUT is: import { execute } from "@cocalc/server/jupyter/execute"; import getAccountId from "lib/account/get-account"; import getParams from "lib/api/get-params"; -import { analytics_cookie_name } from "@cocalc/util/misc"; export default async function handle(req, res) { try { @@ -36,20 +27,15 @@ export default async function handle(req, res) { } async function doIt(req) { - const { input, kernel, history, tag, noCache, hash, project_id, path } = - getParams(req); + const { input, kernel, history, tag, project_id, path } = getParams(req); const account_id = await getAccountId(req); - const analytics_cookie = req.cookies[analytics_cookie_name]; return await execute({ account_id, project_id, path, - analytics_cookie, input, - hash, history, kernel, tag, - noCache, }); } diff --git a/src/packages/next/pages/features/python.tsx b/src/packages/next/pages/features/python.tsx index fadf68b245..cc02606487 100644 --- a/src/packages/next/pages/features/python.tsx +++ b/src/packages/next/pages/features/python.tsx @@ -34,7 +34,7 @@ import PythonLogo from "public/features/python-logo.svg"; const component = "Python"; const title = `Run ${component} Online`; -export default function Octave({ customize }) { +export default function Python({ customize }) { return ( diff --git a/src/packages/project/servers/hub/handle-message.ts b/src/packages/project/servers/hub/handle-message.ts index 4ad363faea..35a4f94aae 100644 --- a/src/packages/project/servers/hub/handle-message.ts +++ b/src/packages/project/servers/hub/handle-message.ts @@ -15,8 +15,6 @@ import { handle_save_blob_message } from "@cocalc/project/blobs"; import { getClient } from "@cocalc/project/client"; import { project_id } from "@cocalc/project/data"; import { exec_shell_code } from "@cocalc/project/exec_shell_code"; -import { get_kernel_data } from "@cocalc/jupyter/kernel/kernel-data"; -import jupyterExecute from "@cocalc/jupyter/stateless-api/execute"; import { getLogger } from "@cocalc/project/logger"; import { print_to_pdf } from "@cocalc/project/print_to_pdf"; import { @@ -28,7 +26,6 @@ import { version } from "@cocalc/util/smc-version"; import { Message } from "./types"; import writeTextFileToProject from "./write-text-file-to-project"; import readTextFileFromProject from "./read-text-file-from-project"; -import { jupyter_execute_response } from "@cocalc/util/message"; const logger = getLogger("handle-message-from-hub"); @@ -69,44 +66,6 @@ export default async function handleMessage( exec_shell_code(socket, mesg); return; - case "jupyter_execute": - try { - const outputs = await jupyterExecute(mesg as any); - socket.write_mesg( - "json", - jupyter_execute_response({ id: mesg.id, output: outputs }), - ); - } catch (err) { - socket.write_mesg( - "json", - message.error({ - id: mesg.id, - error: `${err}`, - }), - ); - } - return; - - case "jupyter_kernels": - try { - socket.write_mesg( - "json", - message.jupyter_kernels({ - kernels: await get_kernel_data(), - id: mesg.id, - }), - ); - } catch (err) { - socket.write_mesg( - "json", - message.error({ - id: mesg.id, - error: `${err}`, - }), - ); - } - return; - // Reading and writing files to/from project and sending over socket case "read_file_from_project": read_file_from_project(socket, mesg); diff --git a/src/packages/server/jupyter/abuse.ts b/src/packages/server/jupyter/abuse.ts deleted file mode 100644 index d5cc36e8ec..0000000000 --- a/src/packages/server/jupyter/abuse.ts +++ /dev/null @@ -1,75 +0,0 @@ -/* -We initially just implement some very simple rate limitations to prevent very -blatant abuse. Everything is hardcoded and nothing is configurable via the -admin settings panel yet. -*/ - -import { isValidUUID } from "@cocalc/util/misc"; -import recentUsage from "./recent-usage"; -import getLogger from "@cocalc/backend/logger"; - -const log = getLogger("jupyter-api:abuse"); - -const PERIOD = "1 hour"; - -// This is the max amount of time in seconds user can use during the given PERIDO -// cached output doesn't count. This is ONLY usage in the public pool of servers, -// and computation in a user's own project doesn't count. -const QUOTAS = { - noAccount: 60 * 3, // 3 minutes per hour of total time for a non-signed in user. - account: 60 * 15, // 15 minutes per hour for a signed in user - global: 3600 * 5, // gobal: up to about 5 things running at once all the time -}; - -// for testing -// const QUOTAS = { -// noAccount: 10, -// account: 20, -// global: 30, -// }; - -// Throws an exception if the request should not be allowed. -export default async function checkForAbuse({ - account_id, - analytics_cookie, -}: { - account_id?: string; - analytics_cookie?: string; -}): Promise { - if (!isValidUUID(account_id) && !isValidUUID(analytics_cookie)) { - // at least some amount of tracking. - throw Error("at least one of account_id or analytics_cookie must be set"); - } - const usage = await recentUsage({ - cache: "short", - period: PERIOD, - account_id, - analytics_cookie, - }); - log.debug("recent usage by this user", { - account_id, - analytics_cookie, - usage, - }); - if (account_id) { - if (usage > QUOTAS.account) { - throw Error( - `You may use at most ${QUOTAS.account} seconds of compute time per ${PERIOD}. Please try again later or do this computation in a project.` - ); - } - } else if (usage > QUOTAS.noAccount) { - throw Error( - `You may use at most ${QUOTAS.noAccount} seconds of compute time per ${PERIOD}. Sign in to increase your quota.` - ); - } - - // Prevent more sophisticated abuse, e.g., changing analytics_cookie or account frequently, - // or just a general huge surge in usage. - const overallUsage = await recentUsage({ cache: "medium", period: PERIOD }); - log.debug("overallUsage = ", usage); - if (overallUsage > QUOTAS.global) { - throw Error( - `There is too much overall usage of code evaluation right now. Please try again later or do this computation in a project.` - ); - } -} diff --git a/src/packages/server/jupyter/execute.ts b/src/packages/server/jupyter/execute.ts index a552b5ff15..b478842476 100644 --- a/src/packages/server/jupyter/execute.ts +++ b/src/packages/server/jupyter/execute.ts @@ -2,16 +2,12 @@ Backend server side part of ChatGPT integration with CoCalc. */ -import getPool from "@cocalc/database/pool"; import getLogger from "@cocalc/backend/logger"; import { getServerSettings } from "@cocalc/database/settings/server-settings"; -import computeHash from "@cocalc/util/jupyter-api/compute-hash"; import getProject from "./global-project-pool"; -import callProject from "@cocalc/server/projects/call"; -import { jupyter_execute } from "@cocalc/util/message"; import isCollaborator from "@cocalc/server/projects/is-collaborator"; -import checkForAbuse from "./abuse"; -import { expire_time } from "@cocalc/util/relative-time"; +import { projectApiClient } from "@cocalc/conat/project/api"; +import { conat } from "@cocalc/backend/conat"; const log = getLogger("jupyter-api:execute"); @@ -42,49 +38,36 @@ interface Options { input?: string; // new input that user types kernel?: string; history?: string[]; - hash?: string; account_id?: string; - analytics_cookie?: string; tag?: string; - noCache?: boolean; project_id?: string; path?: string; + timeout?: number; } export async function execute({ - hash, input, kernel, account_id, - analytics_cookie, history, tag, - noCache, project_id, path, + timeout = 30_000, }: Options): Promise<{ output: object[]; created: Date; } | null> { - // TODO -- await checkForAbuse({ account_id, analytics_cookie }); - log.debug("execute", { input, kernel, history, - hash, account_id, - analytics_cookie, tag, project_id, path, }); - // If hash is given, we only check if output is in database, and - // if so return it. Otherwise, return nothing. - if (hash != null && !noCache) { - return await getFromDatabase(hash); - } if (input == null) { throw Error("input or hash must not be null"); } @@ -94,32 +77,14 @@ export async function execute({ const created = new Date(); - hash = computeHash({ history, input, kernel, project_id, path }); - - if (!noCache) { - // Check if we already have this execution history in the database: - const savedOutput = await getFromDatabase(hash); - if (savedOutput != null) { - log.debug("got saved output"); - return savedOutput; - } - log.debug("have to compute output"); - } - // Execute the code. - let request_account_id, request_project_id, pool, limits; + let request_project_id, pool, limits; if (project_id == null) { - const { jupyter_api_enabled, jupyter_account_id } = - await getServerSettings(); + const { jupyter_api_enabled } = await getServerSettings(); if (!jupyter_api_enabled) { throw Error("Jupyter API is not enabled on this server."); } - // we only worry about abuse against the general public pool, not - // when used in a user's own project - await checkForAbuse({ account_id, analytics_cookie }); - - request_account_id = jupyter_account_id; request_project_id = await getProject(); pool = GLOBAL_POOL; @@ -135,12 +100,16 @@ export async function execute({ if (!(await isCollaborator({ project_id, account_id }))) { throw Error("permission denied -- user must be collaborator on project"); } - request_account_id = account_id; pool = PROJECT_POOL; limits = PROJECT_LIMITS; } - const mesg = jupyter_execute({ + const api = projectApiClient({ + project_id: request_project_id, + timeout, + client: conat(), + }); + const output = await api.jupyter.apiExecute({ input, history, kernel, @@ -148,116 +117,6 @@ export async function execute({ pool, limits, }); - const resp = await callProject({ - account_id: request_account_id, - project_id: request_project_id, - mesg, - }); - if (resp.error) { - throw Error(resp.error); - } - const { output } = resp; - // this is HUGE and should not be logged! - // log.debug("output", output); - const total_time_s = (Date.now() - created.valueOf()) / 1000; - saveResponse({ - created, - input, - output, - kernel, - account_id, - project_id, - path, - analytics_cookie, - history, - tag, - total_time_s, - hash, - noCache, - }); - return { output, created }; -} -// We just assume that hash conflicts don't happen for our purposes here. It's a cryptographic hash function. -async function getFromDatabase( - hash: string, -): Promise<{ output: object[]; created: Date } | null> { - const pool = getPool(); - try { - const { rows } = await pool.query( - `SELECT id, output, created FROM jupyter_api_cache WHERE hash=$1`, - [hash], - ); - if (rows.length == 0) { - return null; - } - // cache hit -- we also update last_active (nonblocking, nonfatal) - (async () => { - try { - await pool.query( - "UPDATE jupyter_api_cache SET last_active=NOW(), expire=NOW() + '1 month'::INTERVAL WHERE id=$1", - [rows[0].id], - ); - } catch (err) { - log.warn("Failed updating cache last_active", err); - } - })(); - return rows[0]; - } catch (err) { - log.warn("Failed to query database cache", err); - return null; - } -} - -// Save mainly for analytics, metering, and to generally see how (or if) -// people use chatgpt in cocalc. -// Also, we could dedup identical inputs (?). -async function saveResponse({ - created, - input, - output, - kernel, - account_id, - project_id, - path, - analytics_cookie, - history, - tag, - total_time_s, - hash, - noCache, -}) { - const pool = getPool(); - if (noCache) { - await pool.query("DELETE FROM jupyter_api_cache WHERE hash=$1", [hash]); - } - // expire in one month – for the log, this must be more than "PERIOD" in abuse.ts - const expire = expire_time(30 * 24 * 60 * 60); - try { - await Promise.all([ - pool.query( - `INSERT INTO jupyter_api_log(created,account_id,project_id,path,analytics_cookie,tag,hash,total_time_s,kernel,history,input,expire) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)`, - [ - created, - account_id, - project_id, - path, - analytics_cookie, - tag, - hash, - total_time_s, - kernel, - history, - input, - expire, - ], - ), - pool.query( - `INSERT INTO jupyter_api_cache(created,hash,output,last_active,expire) VALUES($1,$2,$3,$4,$5)`, - [created, hash, output, created, expire], - ), - ]); - } catch (err) { - log.warn("Failed to save Jupyter execute log entry to database:", err); - } + return { output, created }; } diff --git a/src/packages/server/jupyter/kernels.ts b/src/packages/server/jupyter/kernels.ts index 827f320a8f..30012f3988 100644 --- a/src/packages/server/jupyter/kernels.ts +++ b/src/packages/server/jupyter/kernels.ts @@ -5,10 +5,10 @@ Backend server side part of ChatGPT integration with CoCalc. import getLogger from "@cocalc/backend/logger"; import { getServerSettings } from "@cocalc/database/settings/server-settings"; import getProject from "./global-project-pool"; -import callProject from "@cocalc/server/projects/call"; -import { jupyter_kernels } from "@cocalc/util/message"; import LRU from "lru-cache"; import isCollaborator from "@cocalc/server/projects/is-collaborator"; +import { projectApiClient } from "@cocalc/conat/project/api"; +import { conat } from "@cocalc/backend/conat"; const cache = new LRU({ ttl: 30000, @@ -58,15 +58,8 @@ export default async function getKernels({ project_id = await getProject(); account_id = jupyter_account_id; } - const mesg = jupyter_kernels({}); - const resp = await callProject({ - account_id, - project_id, - mesg, - }); - if (resp.error) { - throw Error(resp.error); - } - cache.set(key, resp.kernels); - return resp.kernels; + const api = projectApiClient({ project_id, client: conat() }); + const kernels = await api.jupyter.kernels(); + cache.set(key, kernels); + return kernels; } diff --git a/src/packages/util/message.d.ts b/src/packages/util/message.d.ts index d9a494b25c..c3baf892e8 100644 --- a/src/packages/util/message.d.ts +++ b/src/packages/util/message.d.ts @@ -97,8 +97,4 @@ export const openai_embeddings_save: any; export const openai_embeddings_save_response: any; export const openai_embeddings_remove: any; export const openai_embeddings_remove_response: any; -export const jupyter_execute: any; -export const jupyter_execute_response: any; -export const jupyter_kernels: any; -export const jupyter_start_pool: any; export const signal_sent: any; diff --git a/src/packages/util/message.js b/src/packages/util/message.js index 690e3aca51..eb9e861b5e 100644 --- a/src/packages/util/message.js +++ b/src/packages/util/message.js @@ -1983,45 +1983,3 @@ message({ id: undefined, ids: required, // uuid's of removed data }); - -API( - message({ - event: "jupyter_execute", - id: undefined, - hash: undefined, // give either hash *or* kernel, input, history, etc. - kernel: undefined, // jupyter kernel - input: undefined, // input code to execute - history: undefined, // optional history of this conversation as a list of input strings. Do not include output - project_id: undefined, // project it should run in. - path: undefined, // optional path where execution happens - tag: undefined, - pool: undefined, // {size?: number; timeout_s?: number;} - limits: undefined, // see packages/jupyter/nbgrader/jupyter-run.ts - }), -); - -message({ - event: "jupyter_execute_response", - id: undefined, - output: required, // the response - total_time_s: undefined, - time: undefined, -}); - -API( - message({ - event: "jupyter_kernels", - id: undefined, - project_id: undefined, - kernels: undefined, // response is same message but with this filled in with array of data giving available kernels - }), -); - -API( - message({ - event: "jupyter_start_pool", - id: undefined, - project_id: undefined, - kernels: undefined, // response is same message but with this filled in with array of data giving available kernels - }), -); From 5604bb1bc479a2e475ecd7383586d62cea700ce6 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 15 Aug 2025 15:52:15 +0000 Subject: [PATCH 296/798] delayed render option for progress bar --- src/packages/frontend/components/loading.tsx | 8 ++++---- src/packages/frontend/components/progress-estimate.tsx | 8 +++++++- src/packages/frontend/components/run-button/output.tsx | 4 +++- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/packages/frontend/components/loading.tsx b/src/packages/frontend/components/loading.tsx index 5b1578de9b..0abfe44ff6 100644 --- a/src/packages/frontend/components/loading.tsx +++ b/src/packages/frontend/components/loading.tsx @@ -6,7 +6,8 @@ import { CSSProperties } from "react"; import { useIntl } from "react-intl"; import FakeProgress from "@cocalc/frontend/components/fake-progress"; -import { TypedMap, useDelayedRender } from "@cocalc/frontend/app-framework"; +import { TypedMap } from "@cocalc/frontend/app-framework"; +import useDelayedRender from "@cocalc/frontend/app-framework/delayed-render-hook"; import { labels } from "@cocalc/frontend/i18n"; import { Icon } from "./icon"; @@ -45,9 +46,8 @@ export function Loading({ }: Props) { const intl = useIntl(); - const render = useDelayedRender(delay ?? 0); - if (!render) { - return <>; + if (!useDelayedRender(delay ?? 0)) { + return null; } return ( diff --git a/src/packages/frontend/components/progress-estimate.tsx b/src/packages/frontend/components/progress-estimate.tsx index 92a6ee9aea..49c01e3231 100644 --- a/src/packages/frontend/components/progress-estimate.tsx +++ b/src/packages/frontend/components/progress-estimate.tsx @@ -5,13 +5,15 @@ take. import { CSSProperties, useEffect, useState } from "react"; import { Progress } from "antd"; import useIsMountedRef from "@cocalc/frontend/app-framework/is-mounted-hook"; +import useDelayedRender from "@cocalc/frontend/app-framework/delayed-render-hook"; interface Props { seconds: number; style?: CSSProperties; + delay?: number; } -export default function ProgressEstimate({ seconds, style }: Props) { +export default function ProgressEstimate({ seconds, style, delay }: Props) { const isMountedRef = useIsMountedRef(); const [progress, setProgress] = useState(0); @@ -26,6 +28,10 @@ export default function ProgressEstimate({ seconds, style }: Props) { return () => clearInterval(interval); }, [progress, seconds]); + if (!useDelayedRender(delay ?? 0)) { + return null; + } + return ( - {running && } + {running && ( + + )} {output != null && (
Date: Fri, 15 Aug 2025 16:01:57 +0000 Subject: [PATCH 297/798] delete all the database caching and logging related to jupyter eval - caching is confusing and not helpful - logging: we never look or use it for anything anyways (and it would be much better to have this in the user's scope) --- .../crm-editor/tables/jupyter-api.ts | 52 ------ .../frame-editors/crm-editor/tables/tables.ts | 1 - src/packages/server/jupyter/recent-usage.ts | 66 -------- src/packages/util/db-schema/jupyter.ts | 157 ------------------ 4 files changed, 276 deletions(-) delete mode 100644 src/packages/frontend/frame-editors/crm-editor/tables/jupyter-api.ts delete mode 100644 src/packages/server/jupyter/recent-usage.ts delete mode 100644 src/packages/util/db-schema/jupyter.ts diff --git a/src/packages/frontend/frame-editors/crm-editor/tables/jupyter-api.ts b/src/packages/frontend/frame-editors/crm-editor/tables/jupyter-api.ts deleted file mode 100644 index 7f49405144..0000000000 --- a/src/packages/frontend/frame-editors/crm-editor/tables/jupyter-api.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { register } from "./tables"; - -register({ - name: "crm_jupyter_api_cache", - - title: "Jupyter Cache", - - icon: "jupyter", - - query: { - crm_jupyter_api_cache: [ - { - id: null, - hash: null, - created: null, - last_active: null, - output: null, - }, - ], - }, - allowCreate: false, - changes: false, -}); - -register({ - name: "crm_jupyter_api_log", - - title: "Jupyter Log", - - icon: "jupyter", - - query: { - crm_jupyter_api_log: [ - { - id: null, - created: null, - hash: null, - account_id: null, - analytics_cookie: null, - project_id: null, - path: null, - kernel: null, - history: null, - input: null, - tag: null, - total_time_s: null, - }, - ], - }, - allowCreate: false, - changes: false, -}); diff --git a/src/packages/frontend/frame-editors/crm-editor/tables/tables.ts b/src/packages/frontend/frame-editors/crm-editor/tables/tables.ts index 46e30868c4..c859e27d92 100644 --- a/src/packages/frontend/frame-editors/crm-editor/tables/tables.ts +++ b/src/packages/frontend/frame-editors/crm-editor/tables/tables.ts @@ -36,7 +36,6 @@ import "./syncstrings"; import "./vouchers"; import "./openai"; import "./analytics"; -import "./jupyter-api"; import "./retention"; interface TableDescription { diff --git a/src/packages/server/jupyter/recent-usage.ts b/src/packages/server/jupyter/recent-usage.ts deleted file mode 100644 index 3e15e9c02b..0000000000 --- a/src/packages/server/jupyter/recent-usage.ts +++ /dev/null @@ -1,66 +0,0 @@ -/* -Return recent usage information about usage of the Jupyter API to carry -out non-cached computations. -*/ - -import getPool from "@cocalc/database/pool"; - -type QueryArgs = { - period: string; - account_id?: string; - analytics_cookie?: string; - cache?: "short" | "medium" | "long"; -}; - -export default async function recentUsage({ - period, - account_id, - analytics_cookie, - cache, -}: QueryArgs): Promise { - let queryArgs; - - if (account_id) { - queryArgs = buildAccountIdQuery(period, account_id); - } else if (analytics_cookie) { - queryArgs = buildAnalyticsCookieQuery(period, analytics_cookie); - } else { - queryArgs = buildOverallUsageQuery(period); - } - - return getUsageForQuery(queryArgs[0], queryArgs[1], cache); -} - -async function getUsageForQuery( - query: string, - args: any[], - cache?: QueryArgs["cache"] -): Promise { - const pool = getPool(cache); - const { rows } = await pool.query(query, args); - return parseInt(rows[0]?.["usage"] ?? 0); -} - -function buildAccountIdQuery( - period: string, - account_id: string -): [string, any[]] { - const query = `SELECT SUM(total_time_s) AS usage FROM jupyter_api_log WHERE created >= NOW() - INTERVAL '${period}' AND account_id=$1 AND project_id IS NULL AND path IS NULL`; - const args = [account_id]; - return [query, args]; -} - -function buildAnalyticsCookieQuery( - period: string, - analytics_cookie: string -): [string, any[]] { - const query = `SELECT SUM(total_time_s) AS usage FROM jupyter_api_log WHERE created >= NOW() - INTERVAL '${period}' AND analytics_cookie=$1 AND project_id IS NULL AND path IS NULL`; - const args = [analytics_cookie]; - return [query, args]; -} - -function buildOverallUsageQuery(period: string): [string, any[]] { - const query = `SELECT SUM(total_time_s) AS usage FROM jupyter_api_log WHERE created >= NOW() - INTERVAL '${period}' AND project_id IS NULL AND path IS NULL`; - const args = []; - return [query, args]; -} diff --git a/src/packages/util/db-schema/jupyter.ts b/src/packages/util/db-schema/jupyter.ts deleted file mode 100644 index 038cea486f..0000000000 --- a/src/packages/util/db-schema/jupyter.ts +++ /dev/null @@ -1,157 +0,0 @@ -import { Table } from "./types"; -import { CREATED_BY, ID, CREATED } from "./crm"; -import { SCHEMA as schema } from "./index"; - -// The jupyter api log has one entry each time a computation -// actually gets performed. Nothing is logged when a request -// is satisfied using the cache. - -Table({ - name: "jupyter_api_log", - fields: { - id: ID, - created: CREATED, - analytics_cookie: { - title: "Analytics Cookie", - type: "string", - desc: "The analytics cookie for the user that asked this question.", - }, - account_id: CREATED_BY, - hash: { - type: "string", - desc: "Hash of the input history, input, kernel, project_id, and path.", - }, - total_time_s: { - type: "number", - desc: "Total amount of time the API call took in seconds.", - }, - tag: { - type: "string", - desc: "A string that the client can include that is useful for analytics later", - }, - project_id: { - desc: "Optional project that is used for this evaluation.", - type: "uuid", - render: { type: "project_link" }, - }, - path: { - desc: "Optional path that is used for this evaluation.", - type: "string", - }, - kernel: { - type: "string", - }, - history: { - title: "History", - type: "array", - pg_type: "TEXT[]", - desc: "The previous inputs", - render: { - type: "json", - }, - }, - input: { - title: "Input", - type: "string", - desc: "Input text that was sent to kernel", - render: { - type: "code", - }, - }, - expire: { - type: "timestamp", - desc: "expire a log entry after 1 month", - }, - }, - rules: { - desc: "Jupyter Kernel Execution Log", - primary_key: "id", - pg_indexes: ["created", "hash"], - }, -}); - -Table({ - name: "crm_jupyter_api_log", - rules: { - virtual: "jupyter_api_log", - primary_key: "id", - user_query: { - get: { - pg_where: [], - admin: true, - fields: { - id: null, - created: null, - hash: null, - account_id: null, - analytics_cookie: null, - project_id: null, - path: null, - kernel: null, - history: null, - input: null, - tag: null, - total_time_s: null, - }, - }, - }, - }, - fields: schema.jupyter_api_log.fields, -}); - -Table({ - name: "jupyter_api_cache", - fields: { - id: ID, - hash: { - type: "string", - desc: "Hash of the input history, input, kernel, project_id, and path.", - }, - created: CREATED, - last_active: { - type: "timestamp", - desc: "When this cache entry was last requested", - }, - output: { - title: "Output", - type: "array", - pg_type: "JSONB[]", - desc: "Output from running the computation", - render: { - type: "json", - }, - }, - expire: { - type: "timestamp", - desc: "this is last_active + 1 month", - }, - }, - rules: { - desc: "Jupyter Kernel Execution Log", - primary_key: "id", - pg_indexes: ["created", "hash"], - }, -}); - -Table({ - name: "crm_jupyter_api_cache", - rules: { - virtual: "jupyter_api_cache", - primary_key: "id", - user_query: { - get: { - pg_where: [], - admin: true, - fields: { - id: null, - hash: null, - created: null, - last_active: null, - count: null, - output: null, - }, - }, - }, - }, - fields: schema.jupyter_api_cache.fields, -}); From 490e807bec5e920130c8d6301fce92ef884b47dc Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 15 Aug 2025 17:02:13 +0000 Subject: [PATCH 298/798] when zod schema fails due to bad email error wasn't shown --- change apiPost to always show zod errors --- src/packages/next/components/auth/sign-up.tsx | 45 ++++++++++--------- src/packages/next/lib/api/post.ts | 8 +++- 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/src/packages/next/components/auth/sign-up.tsx b/src/packages/next/components/auth/sign-up.tsx index f701d62564..288b5d7c5e 100644 --- a/src/packages/next/components/auth/sign-up.tsx +++ b/src/packages/next/components/auth/sign-up.tsx @@ -157,7 +157,7 @@ function SignUp0({ reCaptchaToken = await executeRecaptcha("signup"); } - const result = await apiPost("/auth/sign-up", { + const opts = { terms: true, email, password, @@ -168,7 +168,8 @@ function SignUp0({ publicPathId, tags: Array.from(tags), signupReason, - }); + }; + const result = await apiPost("/auth/sign-up", opts); if (result.issues && len(result.issues) > 0) { setIssues(result.issues); } else { @@ -415,22 +416,22 @@ function SignUp0({ what, )}` : requiresToken2 && !registrationToken - ? "Enter the secret registration token" - : !email - ? "How will you sign in?" - : !isValidEmailAddress(email) - ? "Enter a valid email address above" - : requiredSSO != null - ? "You must sign up via SSO" - : !password || password.length < 6 - ? "Choose password with at least 6 characters" - : !firstName?.trim() - ? "Enter your first name above" - : !lastName?.trim() - ? "Enter your last name above" - : signingUp - ? "" - : "Sign Up!"} + ? "Enter the secret registration token" + : !email + ? "How will you sign in?" + : !isValidEmailAddress(email) + ? "Enter a valid email address above" + : requiredSSO != null + ? "You must sign up via SSO" + : !password || password.length < 6 + ? "Choose password with at least 6 characters" + : !firstName?.trim() + ? "Enter your first name above" + : !lastName?.trim() + ? "Enter your last name above" + : signingUp + ? "" + : "Sign Up!"} {signingUp && ( Signing Up... @@ -480,10 +481,10 @@ function EmailOrSSO(props: EmailOrSSOProps) { {hideSSO ? "Sign up using your single sign-on provider" : strategies.length > 0 && emailSignup - ? "Sign up using either your email address or a single sign-on provider." - : emailSignup - ? "Enter the email address you will use to sign in." - : "Sign up using a single sign-on provider."} + ? "Sign up using either your email address or a single sign-on provider." + : emailSignup + ? "Enter the email address you will use to sign in." + : "Sign up using a single sign-on provider."}

{renderSSO()} diff --git a/src/packages/next/lib/api/post.ts b/src/packages/next/lib/api/post.ts index 03e872fbd7..0c31c9d310 100644 --- a/src/packages/next/lib/api/post.ts +++ b/src/packages/next/lib/api/post.ts @@ -7,7 +7,7 @@ const VERSION = "v2"; export default async function apiPost( path: string, data?: object, - cache_s: number = 0 // if given, cache results for this many seconds to avoid overfetching + cache_s: number = 0, // if given, cache results for this many seconds to avoid overfetching ): Promise { let cache, key; if (cache_s) { @@ -30,6 +30,11 @@ export default async function apiPost( // if error is set in response, then just throw exception (this greatly simplifies client code). throw Error(result.error); } + if (result.errors) { + // This happens with zod schema errors, e.g., try creating an account with email a@b.c, + // which violates the schema for email in zod. + throw Error(JSON.stringify(result.errors)); + } } catch (err) { if (response.statusText == "Not Found") { throw Error(`The API endpoint ${path} does not exist`); @@ -39,6 +44,7 @@ export default async function apiPost( if (cache_s) { cache.set(key, result); } + return result; } From 2c29d9e746bae1420c8cc7cdbe4e0e61557c39a5 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 15 Aug 2025 17:04:49 +0000 Subject: [PATCH 299/798] fix dep issue due to using conat in next api --- src/packages/next/package.json | 3 +- src/packages/pnpm-lock.yaml | 184 ++++++++++++++++++--------------- 2 files changed, 101 insertions(+), 86 deletions(-) diff --git a/src/packages/next/package.json b/src/packages/next/package.json index 76964040da..2b7524d128 100644 --- a/src/packages/next/package.json +++ b/src/packages/next/package.json @@ -37,7 +37,7 @@ "start-project": "unset PGHOST PGUSER COCALC_ROOT; export PORT=5000 BASE_PATH=/$COCALC_PROJECT_ID/port/5000; echo https://cocalc.com$BASE_PATH; pnpm start", "test": "NODE_ENV='dev' pnpm exec jest", "test-api": "NODE_ENV='production' pnpm exec jest ./lib/api/framework.test.ts", - "depcheck": "pnpx depcheck --ignores @openapitools/openapi-generator-cli,eslint-config-next,locales,components,lib,public,pages,software-inventory,pg", + "depcheck": "pnpx depcheck --ignores @openapitools/openapi-generator-cli,eslint-config-next,locales,components,lib,public,pages,software-inventory,pg,bufferutil", "prepublishOnly": "pnpm test", "patch-openapi": "sed '/^interface NrfOasData {/s/^interface/export interface/' node_modules/next-rest-framework/dist/index.d.ts > node_modules/next-rest-framework/dist/index.d.ts.temp && mv node_modules/next-rest-framework/dist/index.d.ts.temp node_modules/next-rest-framework/dist/index.d.ts", "generate-openapi": "cd dist && rm -f public && ln -sf ../public public && NODE_ENV='dev' NODE_PATH=`pwd` npx next-rest-framework generate", @@ -73,6 +73,7 @@ "awaiting": "^3.0.0", "base-64": "^1.0.0", "basic-auth": "^2.0.1", + "bufferutil": "^4.0.9", "csv-stringify": "^6.3.0", "dayjs": "^1.11.11", "express": "^4.21.2", diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index d30546eec0..5ec4f4995c 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -214,10 +214,10 @@ importers: version: 4.17.21 socket.io: specifier: ^4.8.1 - version: 4.8.1(utf-8-validate@6.0.5) + version: 4.8.1(bufferutil@4.0.9)(utf-8-validate@6.0.5) socket.io-client: specifier: ^4.8.1 - version: 4.8.1(utf-8-validate@6.0.5) + version: 4.8.1(bufferutil@4.0.9)(utf-8-validate@6.0.5) devDependencies: '@types/better-sqlite3': specifier: ^7.6.13 @@ -373,13 +373,13 @@ importers: version: 1.4.1 '@jupyter-widgets/base': specifier: ^4.1.1 - version: 4.1.7(react@19.1.1)(utf-8-validate@6.0.5) + version: 4.1.7(bufferutil@4.0.9)(react@19.1.1)(utf-8-validate@6.0.5) '@jupyter-widgets/controls': specifier: 5.0.0-rc.2 - version: 5.0.0-rc.2(crypto@1.0.1)(react@19.1.1)(utf-8-validate@6.0.5) + version: 5.0.0-rc.2(bufferutil@4.0.9)(crypto@1.0.1)(react@19.1.1)(utf-8-validate@6.0.5) '@jupyter-widgets/output': specifier: ^4.1.0 - version: 4.1.7(react@19.1.1)(utf-8-validate@6.0.5) + version: 4.1.7(bufferutil@4.0.9)(react@19.1.1)(utf-8-validate@6.0.5) '@microlink/react-json-view': specifier: ^1.23.3 version: 1.27.0(@types/react@19.1.10)(react-dom@19.1.1(react@19.1.1))(react@19.1.1) @@ -511,7 +511,7 @@ importers: version: 0.2.0 jest-environment-jsdom: specifier: ^30.0.2 - version: 30.0.5(utf-8-validate@6.0.5) + version: 30.0.5(bufferutil@4.0.9)(utf-8-validate@6.0.5) jquery: specifier: ^3.6.0 version: 3.7.1 @@ -956,6 +956,9 @@ importers: basic-auth: specifier: ^2.0.1 version: 2.0.1 + bufferutil: + specifier: ^4.0.9 + version: 4.0.9 csv-stringify: specifier: ^6.3.0 version: 6.6.0 @@ -1139,13 +1142,13 @@ importers: version: 8.3.2 websocket-sftp: specifier: ^0.8.4 - version: 0.8.4(utf-8-validate@6.0.5) + version: 0.8.4(bufferutil@4.0.9)(utf-8-validate@6.0.5) which: specifier: ^2.0.2 version: 2.0.2 ws: specifier: ^8.18.0 - version: 8.18.3(utf-8-validate@6.0.5) + version: 8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5) devDependencies: '@types/body-parser': specifier: ^1.19.5 @@ -1212,22 +1215,22 @@ importers: version: 1.4.1 '@langchain/anthropic': specifier: ^0.3.26 - version: 0.3.26(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76))) + version: 0.3.26(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5))(zod@3.25.76))) '@langchain/core': specifier: ^0.3.68 - version: 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)) + version: 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5))(zod@3.25.76)) '@langchain/google-genai': specifier: ^0.2.16 - version: 0.2.16(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76))) + version: 0.2.16(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5))(zod@3.25.76))) '@langchain/mistralai': specifier: ^0.2.1 - version: 0.2.1(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)))(zod@3.25.76) + version: 0.2.1(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5))(zod@3.25.76)))(zod@3.25.76) '@langchain/ollama': specifier: ^0.2.3 - version: 0.2.3(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76))) + version: 0.2.3(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5))(zod@3.25.76))) '@langchain/openai': specifier: ^0.6.6 - version: 0.6.7(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)))(ws@8.18.3(utf-8-validate@6.0.5)) + version: 0.6.7(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5))(zod@3.25.76)))(ws@8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5)) '@node-saml/passport-saml': specifier: ^5.1.0 version: 5.1.0 @@ -1335,7 +1338,7 @@ importers: version: 6.10.1 openai: specifier: ^5.12.1 - version: 5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76) + version: 5.12.2(ws@8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5))(zod@3.25.76) parse-domain: specifier: ^5.0.0 version: 5.0.0(encoding@0.1.13) @@ -1457,7 +1460,7 @@ importers: devDependencies: '@rspack/cli': specifier: ^1.3.15 - version: 1.3.15(@rspack/core@1.3.15(@swc/helpers@0.5.15))(@types/express@4.17.23)(webpack@5.100.1) + version: 1.3.15(@rspack/core@1.3.15(@swc/helpers@0.5.15))(@types/express@4.17.23)(bufferutil@4.0.9)(webpack@5.100.1) '@rspack/core': specifier: ^1.3.15 version: 1.3.15(@swc/helpers@0.5.15) @@ -1716,7 +1719,7 @@ importers: version: 8.0.9 ws: specifier: ^8.18.0 - version: 8.18.3(utf-8-validate@6.0.5) + version: 8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5) devDependencies: '@types/cookie': specifier: ^0.6.0 @@ -1800,7 +1803,7 @@ importers: version: 18.19.122 jest-environment-jsdom: specifier: ^30.0.2 - version: 30.0.4(utf-8-validate@6.0.5) + version: 30.0.4(bufferutil@4.0.9)(utf-8-validate@6.0.5) util: dependencies: @@ -5271,6 +5274,10 @@ packages: buffer@6.0.3: resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} + bufferutil@4.0.9: + resolution: {integrity: sha512-WDtdLmJvAuNNPzByAYpRo2rF1Mmradw6gvWsQKf63476DDXmomT9zUiGypLcG4ibIM67vhAj8jJRdbmEws2Aqw==} + engines: {node: '>=6.14.2'} + bundle-name@4.1.0: resolution: {integrity: sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q==} engines: {node: '>=18'} @@ -12931,7 +12938,7 @@ snapshots: - supports-color - ts-node - '@jest/environment-jsdom-abstract@30.0.4(jsdom@26.1.0(utf-8-validate@6.0.5))': + '@jest/environment-jsdom-abstract@30.0.4(jsdom@26.1.0(bufferutil@4.0.9)(utf-8-validate@6.0.5))': dependencies: '@jest/environment': 30.0.4 '@jest/fake-timers': 30.0.4 @@ -12940,9 +12947,9 @@ snapshots: '@types/node': 18.19.122 jest-mock: 30.0.2 jest-util: 30.0.2 - jsdom: 26.1.0(utf-8-validate@6.0.5) + jsdom: 26.1.0(bufferutil@4.0.9)(utf-8-validate@6.0.5) - '@jest/environment-jsdom-abstract@30.0.5(jsdom@26.1.0(utf-8-validate@6.0.5))': + '@jest/environment-jsdom-abstract@30.0.5(jsdom@26.1.0(bufferutil@4.0.9)(utf-8-validate@6.0.5))': dependencies: '@jest/environment': 30.0.5 '@jest/fake-timers': 30.0.5 @@ -12951,7 +12958,7 @@ snapshots: '@types/node': 18.19.122 jest-mock: 30.0.5 jest-util: 30.0.5 - jsdom: 26.1.0(utf-8-validate@6.0.5) + jsdom: 26.1.0(bufferutil@4.0.9)(utf-8-validate@6.0.5) '@jest/environment@29.7.0': dependencies: @@ -13190,9 +13197,9 @@ snapshots: '@juggle/resize-observer@3.4.0': {} - '@jupyter-widgets/base@4.1.7(react@19.1.1)(utf-8-validate@6.0.5)': + '@jupyter-widgets/base@4.1.7(bufferutil@4.0.9)(react@19.1.1)(utf-8-validate@6.0.5)': dependencies: - '@jupyterlab/services': 7.4.4(react@19.1.1)(utf-8-validate@6.0.5) + '@jupyterlab/services': 7.4.4(bufferutil@4.0.9)(react@19.1.1)(utf-8-validate@6.0.5) '@lumino/coreutils': 2.2.1 '@lumino/messaging': 2.0.3 '@lumino/widgets': 2.7.1 @@ -13207,9 +13214,9 @@ snapshots: - react - utf-8-validate - '@jupyter-widgets/base@6.0.11(crypto@1.0.1)(react@19.1.1)(utf-8-validate@6.0.5)': + '@jupyter-widgets/base@6.0.11(bufferutil@4.0.9)(crypto@1.0.1)(react@19.1.1)(utf-8-validate@6.0.5)': dependencies: - '@jupyterlab/services': 7.4.4(react@19.1.1)(utf-8-validate@6.0.5) + '@jupyterlab/services': 7.4.4(bufferutil@4.0.9)(react@19.1.1)(utf-8-validate@6.0.5) '@lumino/coreutils': 2.2.1 '@lumino/messaging': 1.10.3 '@lumino/widgets': 1.37.2(crypto@1.0.1) @@ -13224,9 +13231,9 @@ snapshots: - react - utf-8-validate - '@jupyter-widgets/controls@5.0.0-rc.2(crypto@1.0.1)(react@19.1.1)(utf-8-validate@6.0.5)': + '@jupyter-widgets/controls@5.0.0-rc.2(bufferutil@4.0.9)(crypto@1.0.1)(react@19.1.1)(utf-8-validate@6.0.5)': dependencies: - '@jupyter-widgets/base': 6.0.11(crypto@1.0.1)(react@19.1.1)(utf-8-validate@6.0.5) + '@jupyter-widgets/base': 6.0.11(bufferutil@4.0.9)(crypto@1.0.1)(react@19.1.1)(utf-8-validate@6.0.5) '@lumino/algorithm': 1.9.2 '@lumino/domutils': 1.8.2 '@lumino/messaging': 1.10.3 @@ -13242,9 +13249,9 @@ snapshots: - react - utf-8-validate - '@jupyter-widgets/output@4.1.7(react@19.1.1)(utf-8-validate@6.0.5)': + '@jupyter-widgets/output@4.1.7(bufferutil@4.0.9)(react@19.1.1)(utf-8-validate@6.0.5)': dependencies: - '@jupyter-widgets/base': 4.1.7(react@19.1.1)(utf-8-validate@6.0.5) + '@jupyter-widgets/base': 4.1.7(bufferutil@4.0.9)(react@19.1.1)(utf-8-validate@6.0.5) transitivePeerDependencies: - bufferutil - react @@ -13272,7 +13279,7 @@ snapshots: dependencies: '@lumino/coreutils': 2.2.1 - '@jupyterlab/services@7.4.4(react@19.1.1)(utf-8-validate@6.0.5)': + '@jupyterlab/services@7.4.4(bufferutil@4.0.9)(react@19.1.1)(utf-8-validate@6.0.5)': dependencies: '@jupyter/ydoc': 3.1.0 '@jupyterlab/coreutils': 6.4.4 @@ -13284,7 +13291,7 @@ snapshots: '@lumino/polling': 2.1.4 '@lumino/properties': 2.0.3 '@lumino/signaling': 2.1.4 - ws: 8.18.3(utf-8-validate@6.0.5) + ws: 8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5) transitivePeerDependencies: - bufferutil - react @@ -13311,20 +13318,20 @@ snapshots: '@lumino/properties': 2.0.3 '@lumino/signaling': 2.1.4 - '@langchain/anthropic@0.3.26(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)))': + '@langchain/anthropic@0.3.26(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5))(zod@3.25.76)))': dependencies: '@anthropic-ai/sdk': 0.56.0 - '@langchain/core': 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)) + '@langchain/core': 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5))(zod@3.25.76)) fast-xml-parser: 4.5.3 - '@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76))': + '@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5))(zod@3.25.76))': dependencies: '@cfworker/json-schema': 4.1.1 ansi-styles: 5.2.0 camelcase: 6.3.0 decamelize: 1.2.0 js-tiktoken: 1.0.20 - langsmith: 0.3.46(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)) + langsmith: 0.3.46(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5))(zod@3.25.76)) mustache: 4.2.0 p-queue: 6.6.2 p-retry: 4.6.2 @@ -13337,31 +13344,31 @@ snapshots: - '@opentelemetry/sdk-trace-base' - openai - '@langchain/google-genai@0.2.16(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)))': + '@langchain/google-genai@0.2.16(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5))(zod@3.25.76)))': dependencies: '@google/generative-ai': 0.24.1 - '@langchain/core': 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)) + '@langchain/core': 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5))(zod@3.25.76)) uuid: 11.1.0 - '@langchain/mistralai@0.2.1(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)))(zod@3.25.76)': + '@langchain/mistralai@0.2.1(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5))(zod@3.25.76)))(zod@3.25.76)': dependencies: - '@langchain/core': 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)) + '@langchain/core': 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5))(zod@3.25.76)) '@mistralai/mistralai': 1.7.4(zod@3.25.76) uuid: 10.0.0 transitivePeerDependencies: - zod - '@langchain/ollama@0.2.3(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)))': + '@langchain/ollama@0.2.3(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5))(zod@3.25.76)))': dependencies: - '@langchain/core': 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)) + '@langchain/core': 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5))(zod@3.25.76)) ollama: 0.5.16 uuid: 10.0.0 - '@langchain/openai@0.6.7(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)))(ws@8.18.3(utf-8-validate@6.0.5))': + '@langchain/openai@0.6.7(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5))(zod@3.25.76)))(ws@8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5))': dependencies: - '@langchain/core': 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)) + '@langchain/core': 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5))(zod@3.25.76)) js-tiktoken: 1.0.20 - openai: 5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76) + openai: 5.12.2(ws@8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5))(zod@3.25.76) zod: 3.25.76 transitivePeerDependencies: - ws @@ -14330,16 +14337,16 @@ snapshots: '@rspack/binding-win32-ia32-msvc': 1.4.5 '@rspack/binding-win32-x64-msvc': 1.4.5 - '@rspack/cli@1.3.15(@rspack/core@1.3.15(@swc/helpers@0.5.15))(@types/express@4.17.23)(webpack@5.100.1)': + '@rspack/cli@1.3.15(@rspack/core@1.3.15(@swc/helpers@0.5.15))(@types/express@4.17.23)(bufferutil@4.0.9)(webpack@5.100.1)': dependencies: '@discoveryjs/json-ext': 0.5.7 '@rspack/core': 1.3.15(@swc/helpers@0.5.15) - '@rspack/dev-server': 1.1.4(@rspack/core@1.3.15(@swc/helpers@0.5.15))(@types/express@4.17.23)(webpack@5.100.1) + '@rspack/dev-server': 1.1.4(@rspack/core@1.3.15(@swc/helpers@0.5.15))(@types/express@4.17.23)(bufferutil@4.0.9)(webpack@5.100.1) colorette: 2.0.20 exit-hook: 4.0.0 interpret: 3.1.1 rechoir: 0.8.0 - webpack-bundle-analyzer: 4.10.2 + webpack-bundle-analyzer: 4.10.2(bufferutil@4.0.9) yargs: 17.7.2 transitivePeerDependencies: - '@types/express' @@ -14375,14 +14382,14 @@ snapshots: optionalDependencies: '@swc/helpers': 0.5.15 - '@rspack/dev-server@1.1.4(@rspack/core@1.3.15(@swc/helpers@0.5.15))(@types/express@4.17.23)(webpack@5.100.1)': + '@rspack/dev-server@1.1.4(@rspack/core@1.3.15(@swc/helpers@0.5.15))(@types/express@4.17.23)(bufferutil@4.0.9)(webpack@5.100.1)': dependencies: '@rspack/core': 1.3.15(@swc/helpers@0.5.15) chokidar: 3.6.0 http-proxy-middleware: 2.0.9(@types/express@4.17.23) p-retry: 6.2.1 - webpack-dev-server: 5.2.2(webpack@5.100.1) - ws: 8.18.3(utf-8-validate@6.0.5) + webpack-dev-server: 5.2.2(bufferutil@4.0.9)(webpack@5.100.1) + ws: 8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5) transitivePeerDependencies: - '@types/express' - bufferutil @@ -15841,6 +15848,10 @@ snapshots: base64-js: 1.5.1 ieee754: 1.2.1 + bufferutil@4.0.9: + dependencies: + node-gyp-build: 4.8.4 + bundle-name@4.1.0: dependencies: run-applescript: 7.0.0 @@ -17161,12 +17172,12 @@ snapshots: dependencies: once: 1.4.0 - engine.io-client@6.6.3(utf-8-validate@6.0.5): + engine.io-client@6.6.3(bufferutil@4.0.9)(utf-8-validate@6.0.5): dependencies: '@socket.io/component-emitter': 3.1.2 debug: 4.3.7 engine.io-parser: 5.2.3 - ws: 8.17.1(utf-8-validate@6.0.5) + ws: 8.17.1(bufferutil@4.0.9)(utf-8-validate@6.0.5) xmlhttprequest-ssl: 2.1.2 transitivePeerDependencies: - bufferutil @@ -17175,7 +17186,7 @@ snapshots: engine.io-parser@5.2.3: {} - engine.io@6.6.4(utf-8-validate@6.0.5): + engine.io@6.6.4(bufferutil@4.0.9)(utf-8-validate@6.0.5): dependencies: '@types/cors': 2.8.19 '@types/node': 18.19.122 @@ -17185,7 +17196,7 @@ snapshots: cors: 2.8.5 debug: 4.3.7 engine.io-parser: 5.2.3 - ws: 8.17.1(utf-8-validate@6.0.5) + ws: 8.17.1(bufferutil@4.0.9)(utf-8-validate@6.0.5) transitivePeerDependencies: - bufferutil - supports-color @@ -19069,25 +19080,25 @@ snapshots: jest-util: 29.7.0 pretty-format: 29.7.0 - jest-environment-jsdom@30.0.4(utf-8-validate@6.0.5): + jest-environment-jsdom@30.0.4(bufferutil@4.0.9)(utf-8-validate@6.0.5): dependencies: '@jest/environment': 30.0.4 - '@jest/environment-jsdom-abstract': 30.0.4(jsdom@26.1.0(utf-8-validate@6.0.5)) + '@jest/environment-jsdom-abstract': 30.0.4(jsdom@26.1.0(bufferutil@4.0.9)(utf-8-validate@6.0.5)) '@types/jsdom': 21.1.7 '@types/node': 18.19.122 - jsdom: 26.1.0(utf-8-validate@6.0.5) + jsdom: 26.1.0(bufferutil@4.0.9)(utf-8-validate@6.0.5) transitivePeerDependencies: - bufferutil - supports-color - utf-8-validate - jest-environment-jsdom@30.0.5(utf-8-validate@6.0.5): + jest-environment-jsdom@30.0.5(bufferutil@4.0.9)(utf-8-validate@6.0.5): dependencies: '@jest/environment': 30.0.5 - '@jest/environment-jsdom-abstract': 30.0.5(jsdom@26.1.0(utf-8-validate@6.0.5)) + '@jest/environment-jsdom-abstract': 30.0.5(jsdom@26.1.0(bufferutil@4.0.9)(utf-8-validate@6.0.5)) '@types/jsdom': 21.1.7 '@types/node': 24.2.1 - jsdom: 26.1.0(utf-8-validate@6.0.5) + jsdom: 26.1.0(bufferutil@4.0.9)(utf-8-validate@6.0.5) transitivePeerDependencies: - bufferutil - supports-color @@ -19456,7 +19467,7 @@ snapshots: jsbn@1.1.0: {} - jsdom@26.1.0(utf-8-validate@6.0.5): + jsdom@26.1.0(bufferutil@4.0.9)(utf-8-validate@6.0.5): dependencies: cssstyle: 4.6.0 data-urls: 5.0.0 @@ -19476,7 +19487,7 @@ snapshots: whatwg-encoding: 3.1.1 whatwg-mimetype: 4.0.0 whatwg-url: 14.2.0 - ws: 8.18.3(utf-8-validate@6.0.5) + ws: 8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5) xml-name-validator: 5.0.0 transitivePeerDependencies: - bufferutil @@ -19651,7 +19662,7 @@ snapshots: langs@2.0.0: {} - langsmith@0.3.46(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)): + langsmith@0.3.46(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5))(zod@3.25.76)): dependencies: '@types/uuid': 10.0.0 chalk: 4.1.2 @@ -19662,7 +19673,7 @@ snapshots: uuid: 10.0.0 optionalDependencies: '@opentelemetry/api': 1.9.0 - openai: 5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76) + openai: 5.12.2(ws@8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5))(zod@3.25.76) launch-editor@2.11.1: dependencies: @@ -20306,8 +20317,7 @@ snapshots: node-forge@1.3.1: {} - node-gyp-build@4.8.4: - optional: true + node-gyp-build@4.8.4: {} node-int64@0.4.0: {} @@ -20471,9 +20481,9 @@ snapshots: is-inside-container: 1.0.0 wsl-utils: 0.1.0 - openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76): + openai@5.12.2(ws@8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5))(zod@3.25.76): optionalDependencies: - ws: 8.18.3(utf-8-validate@6.0.5) + ws: 8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5) zod: 3.25.76 opener@1.5.2: {} @@ -22330,20 +22340,20 @@ snapshots: smart-buffer@4.2.0: {} - socket.io-adapter@2.5.5(utf-8-validate@6.0.5): + socket.io-adapter@2.5.5(bufferutil@4.0.9)(utf-8-validate@6.0.5): dependencies: debug: 4.3.7 - ws: 8.17.1(utf-8-validate@6.0.5) + ws: 8.17.1(bufferutil@4.0.9)(utf-8-validate@6.0.5) transitivePeerDependencies: - bufferutil - supports-color - utf-8-validate - socket.io-client@4.8.1(utf-8-validate@6.0.5): + socket.io-client@4.8.1(bufferutil@4.0.9)(utf-8-validate@6.0.5): dependencies: '@socket.io/component-emitter': 3.1.2 debug: 4.3.7 - engine.io-client: 6.6.3(utf-8-validate@6.0.5) + engine.io-client: 6.6.3(bufferutil@4.0.9)(utf-8-validate@6.0.5) socket.io-parser: 4.2.4 transitivePeerDependencies: - bufferutil @@ -22357,14 +22367,14 @@ snapshots: transitivePeerDependencies: - supports-color - socket.io@4.8.1(utf-8-validate@6.0.5): + socket.io@4.8.1(bufferutil@4.0.9)(utf-8-validate@6.0.5): dependencies: accepts: 1.3.8 base64id: 2.0.0 cors: 2.8.5 debug: 4.3.7 - engine.io: 6.6.4(utf-8-validate@6.0.5) - socket.io-adapter: 2.5.5(utf-8-validate@6.0.5) + engine.io: 6.6.4(bufferutil@4.0.9)(utf-8-validate@6.0.5) + socket.io-adapter: 2.5.5(bufferutil@4.0.9)(utf-8-validate@6.0.5) socket.io-parser: 4.2.4 transitivePeerDependencies: - bufferutil @@ -23354,7 +23364,7 @@ snapshots: webidl-conversions@7.0.0: {} - webpack-bundle-analyzer@4.10.2: + webpack-bundle-analyzer@4.10.2(bufferutil@4.0.9): dependencies: '@discoveryjs/json-ext': 0.5.7 acorn: 8.15.0 @@ -23367,7 +23377,7 @@ snapshots: opener: 1.5.2 picocolors: 1.1.1 sirv: 2.0.4 - ws: 7.5.10 + ws: 7.5.10(bufferutil@4.0.9) transitivePeerDependencies: - bufferutil - utf-8-validate @@ -23394,7 +23404,7 @@ snapshots: optionalDependencies: webpack: 5.100.1 - webpack-dev-server@5.2.2(webpack@5.100.1): + webpack-dev-server@5.2.2(bufferutil@4.0.9)(webpack@5.100.1): dependencies: '@types/bonjour': 3.5.13 '@types/connect-history-api-fallback': 1.5.4 @@ -23423,7 +23433,7 @@ snapshots: sockjs: 0.3.24 spdy: 4.0.2 webpack-dev-middleware: 7.4.2(webpack@5.100.1) - ws: 8.18.3(utf-8-validate@6.0.5) + ws: 8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5) optionalDependencies: webpack: 5.100.1 transitivePeerDependencies: @@ -23516,12 +23526,12 @@ snapshots: websocket-extensions@0.1.4: {} - websocket-sftp@0.8.4(utf-8-validate@6.0.5): + websocket-sftp@0.8.4(bufferutil@4.0.9)(utf-8-validate@6.0.5): dependencies: awaiting: 3.0.0 debug: 4.4.1 port-get: 1.0.4 - ws: 8.18.3(utf-8-validate@6.0.5) + ws: 8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5) transitivePeerDependencies: - bufferutil - supports-color @@ -23627,14 +23637,18 @@ snapshots: imurmurhash: 0.1.4 signal-exit: 3.0.7 - ws@7.5.10: {} + ws@7.5.10(bufferutil@4.0.9): + optionalDependencies: + bufferutil: 4.0.9 - ws@8.17.1(utf-8-validate@6.0.5): + ws@8.17.1(bufferutil@4.0.9)(utf-8-validate@6.0.5): optionalDependencies: + bufferutil: 4.0.9 utf-8-validate: 6.0.5 - ws@8.18.3(utf-8-validate@6.0.5): + ws@8.18.3(bufferutil@4.0.9)(utf-8-validate@6.0.5): optionalDependencies: + bufferutil: 4.0.9 utf-8-validate: 6.0.5 wsl-utils@0.1.0: From 54ab3246f4dfd972e04c8c517e664478e96373ea Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 15 Aug 2025 17:04:59 +0000 Subject: [PATCH 300/798] delete all backend new account project setup stuff -- that is more robustly done on the frontend --- .../accounts/account-creation-actions.ts | 44 ----- .../server/accounts/create-account.ts | 6 - src/packages/server/accounts/first-project.ts | 150 ------------------ .../server/messages/support-account.ts | 1 - src/packages/server/purchases/test-data.ts | 3 - 5 files changed, 204 deletions(-) delete mode 100644 src/packages/server/accounts/first-project.ts diff --git a/src/packages/server/accounts/account-creation-actions.ts b/src/packages/server/accounts/account-creation-actions.ts index 753eedbe79..cf453db186 100644 --- a/src/packages/server/accounts/account-creation-actions.ts +++ b/src/packages/server/accounts/account-creation-actions.ts @@ -3,11 +3,7 @@ import getPool from "@cocalc/database/pool"; import addUserToProject from "@cocalc/server/projects/add-user-to-project"; -import firstProject from "./first-project"; -import getOneProject from "@cocalc/server/projects/get-one"; -import { getProject } from "@cocalc/server/projects/control"; import { getLogger } from "@cocalc/backend/logger"; -import getProjects from "@cocalc/server/projects/get"; const log = getLogger("server:accounts:creation-actions"); @@ -15,13 +11,10 @@ export default async function accountCreationActions({ email_address, account_id, tags, - noFirstProject, }: { email_address?: string; account_id: string; tags?: string[]; - // if set, don't do any initial project actions (i.e., creating or starting projects) - noFirstProject?: boolean; }): Promise { log.debug({ account_id, email_address, tags }); @@ -43,43 +36,6 @@ export default async function accountCreationActions({ } } log.debug("added user to", numProjects, "projects"); - if (!noFirstProject) { - if (numProjects == 0) { - // didn't get added to any projects - // You may be a new user with no known "reason" - // to use CoCalc, except that you found the page and signed up. You are - // VERY likely to create a project next, or you wouldn't be here. - // So we create a project for you now to increase your chance of success. - // NOTE -- wrapped in closure, since do NOT block on this: - (async () => { - try { - const projects = await getProjects({ account_id, limit: 1 }); - if (projects.length == 0) { - // you really have no projects at all. - await firstProject({ account_id, tags }); - } - } catch (err) { - // non-fatal; they can make their own project - log.error("problem configuring first project", account_id, err); - } - })(); - } else if (numProjects > 0) { - // Make sure project is running so they have a good first experience. - (async () => { - try { - const { project_id } = await getOneProject(account_id); - const project = getProject(project_id); - await project.start(); - } catch (err) { - log.error( - "failed to start newest project invited to", - err, - account_id, - ); - } - })(); - } - } } export async function creationActionsDone(account_id: string): Promise { diff --git a/src/packages/server/accounts/create-account.ts b/src/packages/server/accounts/create-account.ts index f2bf486908..df64bc4b3c 100644 --- a/src/packages/server/accounts/create-account.ts +++ b/src/packages/server/accounts/create-account.ts @@ -23,10 +23,6 @@ interface Params { tags?: string[]; signupReason?: string; owner_id?: string; - // if set, do not do any of the various heuristics to create or start user's first project. - // I added this to avoid leaks with unit testing, but it may be useful in other contexts, e.g., - // avoiding confusion with self-hosted installs. - noFirstProject?: boolean; } export default async function createAccount({ @@ -38,7 +34,6 @@ export default async function createAccount({ tags, signupReason, owner_id, - noFirstProject, }: Params): Promise { try { log.debug( @@ -77,7 +72,6 @@ export default async function createAccount({ email_address: email, account_id, tags, - noFirstProject, }); await creationActionsDone(account_id); } catch (error) { diff --git a/src/packages/server/accounts/first-project.ts b/src/packages/server/accounts/first-project.ts deleted file mode 100644 index 7b96fa93d2..0000000000 --- a/src/packages/server/accounts/first-project.ts +++ /dev/null @@ -1,150 +0,0 @@ -/* Create a first project for this user and add some content to it - Inspired by the tags. */ - -import createProject from "@cocalc/server/projects/create"; -import { TAGS_MAP } from "@cocalc/util/db-schema/accounts"; -import { getLogger } from "@cocalc/backend/logger"; -import { getProject } from "@cocalc/server/projects/control"; -import callProject from "@cocalc/server/projects/call"; -import getKernels from "@cocalc/server/jupyter/kernels"; -import { isValidUUID } from "@cocalc/util/misc"; - -// If true, create welcome files based on tags. -// disabled because based on data with users, it doesn't help. -// Or maybe the files just aren't good enough (?). In practice, -// people get confused, then finally figure out what to really do, -// and do it in the welcome/ directory, which just means -// a cluttered experience. -const WELCOME_FILES = false; - -const log = getLogger("server:accounts:first-project"); - -export default async function firstProject({ - account_id, - tags, -}: { - account_id: string; - tags?: string[]; -}): Promise { - log.debug(account_id, tags); - if (!isValidUUID(account_id)) { - throw Error("account_id must be a valid uuid"); - } - const project_id = await createProject({ - account_id, - title: "My First Project", - }); - log.debug("created new project", project_id); - const project = getProject(project_id); - await project.start(); - if (!WELCOME_FILES || tags == null || tags.length == 0) { - return project_id; - } - for (const tag of tags) { - if (tag == "ipynb") { - // make Jupyter notebooks for languages of interest - // these are the actual kernels supported by this project: - const kernels = await getKernels({ project_id, account_id }); - for (const tag2 of tags) { - const { - language, - welcome = "", - jupyterExtra = "", - } = TAGS_MAP[tag2] ?? {}; - if (language) { - await createJupyterNotebookIfAvailable( - kernels, - account_id, - project_id, - language, - welcome + jupyterExtra - ); - } - } - } else { - const welcome = TAGS_MAP[tag]?.welcome; - if (welcome != null) { - // make welcome file - await createWelcome(account_id, project_id, tag, welcome); - } - } - } - - return project_id; -} - -async function createJupyterNotebookIfAvailable( - kernels, - account_id: string, - project_id: string, - language: string, - welcome: string -): Promise { - // find the highest priority kernel with the given language - let kernelspec: any = null; - let priority: number = -9999999; - for (const kernel of kernels) { - const kernelPriority = kernel.metadata?.cocalc?.priority ?? 0; - if (kernel.language == language && kernelPriority > priority) { - kernelspec = kernel; - priority = kernelPriority; - } - } - if (kernelspec == null) return ""; - - const content = { - cells: [ - { - cell_type: "code", - execution_count: 0, - metadata: { - collapsed: false, - }, - outputs: [], - source: welcome?.split("\n").map((x) => x + "\n") ?? [], - }, - ], - metadata: { - kernelspec, - }, - nbformat: 4, - nbformat_minor: 4, - }; - const path = `welcome/${language}.ipynb`; - await callProject({ - account_id, - project_id, - mesg: { - event: "write_text_file_to_project", - path, - content: JSON.stringify(content, undefined, 2), - }, - }); - // TODO: Put an appropriate prestarted kernel in the pool. - // This is an optimization and it's not easy. - return path; -} - -async function createWelcome( - account_id: string, - project_id: string, - ext: string, - welcome: string -): Promise { - const path = `welcome/welcome.${ext}`; - const { torun } = TAGS_MAP[ext] ?? {}; - let content = welcome; - if (torun) { - content = `${torun}\n\n${content}`; - } - await callProject({ - account_id, - project_id, - mesg: { - event: "write_text_file_to_project", - path, - content, - }, - }); - return path; -} diff --git a/src/packages/server/messages/support-account.ts b/src/packages/server/messages/support-account.ts index ec39e0e728..5f70f24c6b 100644 --- a/src/packages/server/messages/support-account.ts +++ b/src/packages/server/messages/support-account.ts @@ -37,7 +37,6 @@ async function createSupportAccount() { firstName: "CoCalc", lastName: "Support", account_id, - noFirstProject: true, }); await callback2(db().set_server_setting, { name: "support_account_id", diff --git a/src/packages/server/purchases/test-data.ts b/src/packages/server/purchases/test-data.ts index 5fc995d233..9ecd7e0106 100644 --- a/src/packages/server/purchases/test-data.ts +++ b/src/packages/server/purchases/test-data.ts @@ -39,9 +39,6 @@ export async function createTestAccount(account_id: string) { firstName: "Test", lastName: "User", account_id, - // important -- otherwise every time we call createTestAccount, a project gets created and is running, - // and these do NOT get cleaned up ever. - noFirstProject: true, }); } From 146d424e7bd4b8aabd42773206149d87837fec78 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 15 Aug 2025 17:47:10 +0000 Subject: [PATCH 301/798] rewrite exec api v2 to use conat --- src/packages/api-client/src/project.ts | 90 ------------------- src/packages/frontend/client/client.ts | 3 + src/packages/next/pages/api/v2/exec.ts | 51 ++++------- .../pages/api/v2/projects/write-text-file.ts | 31 ------- src/packages/server/compute/exec.ts | 26 +++--- .../compute/maintenance/cloud/health-check.ts | 14 +-- src/packages/server/projects/exec.ts | 47 ++++++++++ 7 files changed, 86 insertions(+), 176 deletions(-) delete mode 100644 src/packages/next/pages/api/v2/projects/write-text-file.ts create mode 100644 src/packages/server/projects/exec.ts diff --git a/src/packages/api-client/src/project.ts b/src/packages/api-client/src/project.ts index e05f0abf97..29ae86efbe 100644 --- a/src/packages/api-client/src/project.ts +++ b/src/packages/api-client/src/project.ts @@ -4,8 +4,6 @@ API key should be enough to allow them. */ import { apiCall } from "./call"; -import { siteUrl } from "./urls"; -import { type JupyterApiOptions } from "@cocalc/util/jupyter/api-types"; // Starts a project running. export async function start(opts: { project_id: string }) { @@ -22,91 +20,3 @@ export async function stop(opts: { project_id: string }) { export async function touch(opts: { project_id: string }) { return await apiCall("v2/projects/touch", opts); } - -// There's a TCP connection from some hub to the project, which has -// a limited api, which callProject exposes. This includes: -// mesg={event:'ping'}, etc. -// See src/packages/server/projects/connection/call.ts -// We use this to implement some functions below. -async function callProject(opts: { project_id: string; mesg: object }) { - return await apiCall("v2/projects/call", opts); -} - -export async function ping(opts: { project_id: string }) { - return await callProject({ ...opts, mesg: { event: "ping" } }); -} - -export async function exec(opts: { - project_id: string; - path?: string; - command?: string; - args?: string[]; - timeout?: number; // in seconds; default 10 - aggregate?: any; - max_output?: number; - bash?: boolean; - err_on_exit?: boolean; // default true -}) { - return await callProject({ - project_id: opts.project_id, - mesg: { event: "project_exec", ...opts }, - }); -} - -// Returns URL of the file or directory, which you can -// download (from the postgres blob store). It gets autodeleted. -// There is a limit of about 10MB for this. -// For text use readTextFileToProject -export async function readFile(opts: { - project_id: string; - path: string; // file or directory - archive?: "tar" | "tar.bz2" | "tar.gz" | "zip" | "7z"; - ttlSeconds?: number; -}): Promise { - const { archive, data_uuid } = await callProject({ - project_id: opts.project_id, - mesg: { event: "read_file_from_project", ...opts }, - }); - return siteUrl( - `blobs/${opts.path}${archive ? `.${archive}` : ""}?uuid=${data_uuid}`, - ); -} - -// export async function writeFileToProject(opts: { -// project_id: string; -// path: string; // file or directory -// archive?: "tar" | "tar.bz2" | "tar.gz" | "zip" | "7z"; -// ttlSeconds?: number; -// }): Promise { - -// } - -export async function writeTextFile(opts: { - project_id: string; - path: string; - content: string; -}): Promise { - await callProject({ - project_id: opts.project_id, - mesg: { event: "write_text_file_to_project", ...opts }, - }); -} - -export async function readTextFile(opts: { - project_id: string; - path: string; -}): Promise { - return await callProject({ - project_id: opts.project_id, - mesg: { event: "read_text_file_from_project", ...opts }, - }); -} - -export async function jupyterExec(opts: JupyterApiOptions): Promise { - return ( - await callProject({ - project_id: opts.project_id, - mesg: { event: "jupyter_execute", ...opts }, - }) - ).output; -} diff --git a/src/packages/frontend/client/client.ts b/src/packages/frontend/client/client.ts index 786f57ef0e..187aa5becb 100644 --- a/src/packages/frontend/client/client.ts +++ b/src/packages/frontend/client/client.ts @@ -35,6 +35,7 @@ import type { CreateConatServiceFunction, } from "@cocalc/conat/service"; import { randomId } from "@cocalc/conat/names"; +import api from "./api"; // This DEBUG variable comes from webpack: declare const DEBUG; @@ -114,6 +115,7 @@ export interface WebappClient extends EventEmitter { set_connected?: Function; version: Function; alert_message: Function; + nextjsApi?: typeof api; } export const WebappClient = null; // webpack + TS es2020 modules need this @@ -194,6 +196,7 @@ class Client extends EventEmitter implements WebappClient { synctable_database: Function; async_query: Function; alert_message: Function; + nextjsApi = api; constructor() { super(); diff --git a/src/packages/next/pages/api/v2/exec.ts b/src/packages/next/pages/api/v2/exec.ts index 5f9cd57d90..95b8716790 100644 --- a/src/packages/next/pages/api/v2/exec.ts +++ b/src/packages/next/pages/api/v2/exec.ts @@ -7,12 +7,11 @@ Run code in a project. */ -import callProject from "@cocalc/server/projects/call"; -import isCollaborator from "@cocalc/server/projects/is-collaborator"; import getAccountId from "lib/account/get-account"; import getParams from "lib/api/get-params"; import { apiRoute, apiRouteOperation } from "lib/api"; import { ExecInputSchema, ExecOutputSchema } from "lib/api/schema/exec"; +import exec from "@cocalc/server/projects/exec"; async function handle(req, res) { try { @@ -48,37 +47,25 @@ async function get(req) { async_await, } = getParams(req); - if (!(await isCollaborator({ account_id, project_id }))) { - throw Error("user must be a collaborator on the project"); - } + const execOpts = { + filesystem, + path, + command, + args, + timeout, + max_output, + bash, + aggregate, + err_on_exit, + env, + async_call, + async_get, + async_stats, + async_await, + }; - const resp = await callProject({ - account_id, - project_id, - mesg: { - event: "project_exec", - project_id, - compute_server_id, - filesystem, - path, - command, - args, - timeout, - max_output, - bash, - aggregate, - err_on_exit, - env, - async_call, - async_get, - async_stats, - async_await, - }, - }); - // event and id don't make sense for http post api - delete resp.event; - delete resp.id; - return resp; + // this *does* do permissions check + return await exec({ account_id, project_id, compute_server_id, execOpts }); } export default apiRoute({ diff --git a/src/packages/next/pages/api/v2/projects/write-text-file.ts b/src/packages/next/pages/api/v2/projects/write-text-file.ts deleted file mode 100644 index 24886624bb..0000000000 --- a/src/packages/next/pages/api/v2/projects/write-text-file.ts +++ /dev/null @@ -1,31 +0,0 @@ -/* Write a text file to a project. */ - -import getAccountId from "lib/account/get-account"; -import getParams from "lib/api/get-params"; -import { isValidUUID } from "@cocalc/util/misc"; -import callProject from "@cocalc/server/projects/call"; -import { OkStatus } from "lib/api/status"; - -export default async function handle(req, res) { - const account_id = await getAccountId(req); - try { - if (account_id == null) throw Error("must be authenticated"); - const { project_id, path, content } = getParams(req); - if (!isValidUUID(project_id)) - throw Error("must set project_id to a valid uuid"); - if (!path) throw Error("must specify a 'path'"); - if (content == null) throw Error("must include content of file"); - await callProject({ - account_id, - project_id, - mesg: { - event: "write_text_file_to_project", - path, - content, - }, - }); - res.json(OkStatus); - } catch (err) { - res.json({ error: err.message }); - } -} diff --git a/src/packages/server/compute/exec.ts b/src/packages/server/compute/exec.ts index 8bf6799097..8bf627dcba 100644 --- a/src/packages/server/compute/exec.ts +++ b/src/packages/server/compute/exec.ts @@ -2,9 +2,13 @@ Execute code on a compute server. */ -import callProject from "@cocalc/server/projects/call"; +import { projectApiClient } from "@cocalc/conat/project/api"; +import { conat } from "@cocalc/backend/conat"; import { getServer } from "./get-servers"; -import type { ExecOpts } from "@cocalc/util/db-schema/projects"; +import type { + ExecuteCodeOutput, + ExecuteCodeOptions, +} from "@cocalc/util/types/execute-code"; // Run exec export default async function exec({ @@ -14,18 +18,14 @@ export default async function exec({ }: { account_id: string; id: number; - execOpts: Partial; -}) { + execOpts: ExecuteCodeOptions; +}): Promise { const server = await getServer({ account_id, id }); - - return await callProject({ - account_id, + const api = projectApiClient({ + client: conat(), + compute_server_id: id, project_id: server.project_id, - mesg: { - ...execOpts, - event: "project_exec", - compute_server_id: id, - project_id: server.project_id, - }, + timeout: execOpts.timeout ? execOpts.timeout * 1000 + 2000 : undefined, }); + return await api.system.exec(execOpts); } diff --git a/src/packages/server/compute/maintenance/cloud/health-check.ts b/src/packages/server/compute/maintenance/cloud/health-check.ts index 839584ab53..7e82938bec 100644 --- a/src/packages/server/compute/maintenance/cloud/health-check.ts +++ b/src/packages/server/compute/maintenance/cloud/health-check.ts @@ -10,7 +10,6 @@ so gets reset when this service gets restarted. import getPool from "@cocalc/database/pool"; import getLogger from "@cocalc/backend/logger"; -import callProject from "@cocalc/server/projects/call"; import { deprovision, suspend, @@ -25,6 +24,7 @@ import { HEALTH_CHECK_DEFAULTS, } from "@cocalc/util/db-schema/compute-servers"; import { map } from "awaiting"; +import exec from "@cocalc/server/compute/exec"; const PARALLEL_LIMIT = 10; @@ -93,22 +93,16 @@ async function updateComputeServer({ let success; try { logger.debug("run check on ", { compute_server_id: id, project_id }); - const resp = await callProject({ + await exec({ account_id, - project_id, - mesg: { - event: "project_exec", - project_id, - compute_server_id: id, + id, + execOpts: { command, timeout: timeoutSeconds, bash: true, err_on_exit: true, }, }); - if (resp.event == "error" || !!resp.exit_code) { - throw Error("fail"); - } logger.debug("health check worked", { id }); success = true; } catch (err) { diff --git a/src/packages/server/projects/exec.ts b/src/packages/server/projects/exec.ts new file mode 100644 index 0000000000..0ce0c1478b --- /dev/null +++ b/src/packages/server/projects/exec.ts @@ -0,0 +1,47 @@ +/* +Run arbitrarily shell command on compute server or project. +DOES check auth +*/ + +import { projectApiClient } from "@cocalc/conat/project/api"; +import { conat } from "@cocalc/backend/conat"; +import type { + ExecuteCodeOutput, + ExecuteCodeOptions, +} from "@cocalc/util/types/execute-code"; +import execOnComputeServer from "@cocalc/server/compute/exec"; +import isCollaborator from "@cocalc/server/projects/is-collaborator"; + +// checks auth and runs code +export default async function exec({ + account_id, + project_id, + compute_server_id = 0, + execOpts, +}: { + account_id: string; + project_id: string; + compute_server_id?: number; + execOpts: ExecuteCodeOptions; +}): Promise { + if (compute_server_id) { + // do separately because we may have to deny if allow collab control isn't enabled. + return await execOnComputeServer({ + account_id, + id: compute_server_id, + execOpts, + }); + } + + if (!(await isCollaborator({ account_id, project_id }))) { + throw Error("user must be collaborator on project"); + } + + const api = projectApiClient({ + client: conat(), + compute_server_id, + project_id, + timeout: execOpts.timeout ? execOpts.timeout * 1000 + 2000 : undefined, + }); + return await api.system.exec(execOpts); +} From 9f6db99d983a7876d20ea84535b15f94822da898 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 15 Aug 2025 17:50:15 +0000 Subject: [PATCH 302/798] delete the latex api v2 endpoint -- I'm not aware of anybody using it ever... and to survive we must focus --- src/packages/next/lib/api/schema/latex.ts | 103 -------------- src/packages/next/pages/api/v2/latex.ts | 162 ---------------------- 2 files changed, 265 deletions(-) delete mode 100644 src/packages/next/lib/api/schema/latex.ts delete mode 100644 src/packages/next/pages/api/v2/latex.ts diff --git a/src/packages/next/lib/api/schema/latex.ts b/src/packages/next/lib/api/schema/latex.ts deleted file mode 100644 index 87d5a5d8ad..0000000000 --- a/src/packages/next/lib/api/schema/latex.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { z } from "../framework"; - -import { DEFAULT_LATEX_COMMAND } from "../latex"; - -import { FailedAPIOperationSchema } from "./common"; - -// OpenAPI spec -// -export const LatexInputSchema = z - .object({ - path: z - .string() - .describe( - `Path to a .tex file. If the file doesn't exist, it is created with the - given content. Also, if the directory containing path doesn't exist, it - is created. If the path starts with \`/tmp\` (e.g., - \`/tmp/foo/bar.tex\`), then we do always do \`rm /tmp/foo/bar.*\` to - clean up temporary files. We do _not_ do this unless the path starts - with \`/tmp\`.`, - ), - content: z - .string() - .optional() - .describe( - `Textual content of the .tex file on which you want to run LaTeX. If - not given, path must refer to an actual file already in the project. - Then the path \`.tex\` file is created and this content written to it.`, - ), - project_id: z - .string() - .uuid() - .optional() - .describe( - `The v4 uuid of a project you have access to. If not given, your most - recent project is used, or if you have no projects, one is created. The - project is started if it isn't already running. **WARNING:** if the - project isn't running you may get an error while it is starting; if you - retry in a few seconds then it works.`, - ), - command: z - .string() - .optional() - .describe( - `LaTeX build command. This will be run from the directory containing - path and should produce the output pdf file. If not given, we use - \`${DEFAULT_LATEX_COMMAND} filename.tex\`.`, - ), - timeout: z - .number() - .gte(5) - .default(30) - .describe( - `If given, this is a timeout in seconds for how long the LaTeX build - command can run before it is killed. You should increase this from the - default if you're building large documents. See also the - \`only_read_pdf\` option.`, - ), - ttl: z - .number() - .gte(60) - .describe("Time in seconds for which generated PDF url is valid.") - .default(3600), - only_read_pdf: z - .boolean() - .optional() - .describe( `Instead of running LaTeX, we **only** try to grab the output pdf if it - exists. Currently, you must also specify the \`project_id\` if you use - this option, since we haven't implemented a way to know in which project - the latex command was run. When true, \`only_read_pdf\` is the same as - when it is false, except only the step involving reading the pdf - happens. Use this if compiling times out for some reason due to network - timeout requirements. **NOTE:** \`only_read_pdf\` doesn't currently - get the compilation output log.`, - ), - }) - .describe( - `Turn LaTeX .tex file contents into a pdf. This run in a CoCalc project - with a configurable timeout and command, so can involve arbitrarily - sophisticated processing.`, - ) - -export const LatexOutputSchema = z.union([ - FailedAPIOperationSchema, - z.object({ - compile: z.object({ - stdout: z.string(), - stderr: z.string(), - exit_code: z.number(), - }), - url: z - .string() - .describe("URL where you can view the generated PDF file"), - pdf: z - .string() - .describe( - `Information about reading the PDF from disk, e.g., an error if the PDF - does not exist.`, - ), - }), -]); - -export type LatexInput = z.infer; -export type LatexOutput = z.infer; diff --git a/src/packages/next/pages/api/v2/latex.ts b/src/packages/next/pages/api/v2/latex.ts deleted file mode 100644 index 8711fb434d..0000000000 --- a/src/packages/next/pages/api/v2/latex.ts +++ /dev/null @@ -1,162 +0,0 @@ -/* -Turn LaTeX .tex file contents into a pdf. This run in a CoCalc -project with a configurable timeout and command, so can involve -arbitrarily sophisticated processing. - -Then the path .tex file is created, if content is specified. Next the command is run which should hopefully produce a pdf file. -Finally, the pdf file is read into our database (as a blob). -*/ - -import getAccountId from "lib/account/get-account"; -import getOneProject from "@cocalc/server/projects/get-one"; -import { getProject } from "@cocalc/server/projects/control"; -import callProject from "@cocalc/server/projects/call"; -import getParams from "lib/api/get-params"; -import { path_split } from "@cocalc/util/misc"; -import getCustomize from "@cocalc/database/settings/customize"; -import isCollaborator from "@cocalc/server/projects/is-collaborator"; -import { DEFAULT_LATEX_COMMAND } from "lib/api/latex"; - -import { apiRoute, apiRouteOperation } from "lib/api"; -import { - LatexInputSchema, - LatexOutputSchema, -} from "lib/api/schema/latex"; - - -async function handle(req, res) { - const account_id = await getAccountId(req); - const params = getParams(req); - try { - if (!account_id) { - throw Error("must be authenticated"); - } - if (!params.path || !params.path.endsWith(".tex")) { - throw Error("path must be specified and end in .tex"); - } - const { head: dir, tail: filename } = path_split(params.path); - if (params.only_read_pdf) { - if (params.project_id == null) { - throw Error("if only_read_pdf is set then project_id must also be set"); - } - if (params.path.startsWith("/tmp")) { - throw Error( - "if only_read_pdf is set then path must not start with /tmp (otherwise the pdf would be removed)", - ); - } - } - - let project_id; - if (params.project_id != null) { - project_id = params.project_id; - if (!(await isCollaborator({ project_id, account_id }))) { - throw Error("must be signed in as a collaborator on the project"); - } - } else { - // don't need to check collaborator in this case: - project_id = (await getOneProject(account_id)).project_id; - } - - let result: any = undefined; - let compile: any = undefined; - let pdf: any = undefined; - let url: string | undefined = undefined; - try { - // ensure the project is running. - const project = getProject(project_id); - await project.start(); - - if (!params.only_read_pdf) { - if (params.content != null) { - // write content to the project as the file path - await callProject({ - account_id, - project_id, - mesg: { - event: "write_text_file_to_project", - path: params.path, - content: params.content, - }, - }); - } - compile = await callProject({ - account_id, - project_id, - mesg: { - event: "project_exec", - timeout: params.timeout ?? 30, - path: dir, - command: params.command ?? `${DEFAULT_LATEX_COMMAND} ${filename}`, - }, - }); - } - // TODO: should we check for errors in compile before trying to read pdf? - const ttlSeconds = params.ttl ?? 3600; - try { - pdf = await callProject({ - account_id, - project_id, - mesg: { - event: "read_file_from_project", - path: pdfFile(params.path), - ttlSeconds, - }, - }); - const { siteURL } = await getCustomize(); - if (pdf != null) { - url = pdf.data_uuid - ? siteURL + `/blobs/${pdfFile(params.path)}?uuid=${pdf.data_uuid}` - : undefined; - } - result = { compile, url, pdf }; - } catch (err) { - result = { compile, error: err.message }; - } - } finally { - if (params.path.startsWith("/tmp")) { - await callProject({ - account_id, - project_id, - mesg: { - event: "project_exec", - path: "/tmp", - bash: true, - command: `rm ${rmGlob(params.path)}`, - }, - }); - } - } - res.json(result); - } catch (err) { - res.json({ error: err.message }); - } -} - -function pdfFile(path: string): string { - return path.slice(0, path.length - 4) + ".pdf"; -} - -function rmGlob(path: string): string { - return path.slice(0, path.length - 4) + ".*"; -} - -export default apiRoute({ - latex: apiRouteOperation({ - method: "POST", - openApiOperation: { - tags: ["Utils"] - }, - }) - .input({ - contentType: "application/json", - body: LatexInputSchema, - }) - .outputs([ - { - status: 200, - contentType: "application/json", - body: LatexOutputSchema, - }, - ]) - .handler(handle), -}); From 84639846ea4bbb1876ecfd50ea0f61a7dbd67cc4 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 15 Aug 2025 17:52:05 +0000 Subject: [PATCH 303/798] remove projects/call endpoint -- also not used --- .../next/pages/api/v2/projects/call.ts | 21 ------------------- 1 file changed, 21 deletions(-) delete mode 100644 src/packages/next/pages/api/v2/projects/call.ts diff --git a/src/packages/next/pages/api/v2/projects/call.ts b/src/packages/next/pages/api/v2/projects/call.ts deleted file mode 100644 index 3791c482fd..0000000000 --- a/src/packages/next/pages/api/v2/projects/call.ts +++ /dev/null @@ -1,21 +0,0 @@ -/* -API endpoint that makes it possible to send a message to a project -that the user is a collaborator on and get back a response. - -See cocalc/src/packages/server/projects/connection/call.ts -for a list of messages. -*/ - -import getParams from "lib/api/get-params"; -import getAccountId from "lib/account/get-account"; -import callProject from "@cocalc/server/projects/call"; - -export default async function handle(req, res) { - const account_id = await getAccountId(req); - try { - const { project_id, mesg } = getParams(req); - res.json(await callProject({ account_id, project_id, mesg })); - } catch (err) { - res.json({ error: err.message }); - } -} From ce8786cc8c1645dc1c353e8b5f67f4856ab561a1 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 15 Aug 2025 18:18:42 +0000 Subject: [PATCH 304/798] eliminate entirely the tcp connection from hub to projects and all use of it; conat, etc., does all the same things better --- .../database/postgres-user-queries.coffee | 7 - src/packages/database/postgres/types.ts | 2 - src/packages/hub/local_hub_connection.coffee | 701 ------------------ src/packages/hub/projects.coffee | 101 --- .../next/pages/api/v2/projects/copy-url.ts | 15 +- src/packages/project/read_write_files.ts | 190 ----- .../project/servers/hub/handle-message.ts | 141 ---- .../hub/read-text-file-from-project.ts | 21 - .../project/servers/hub/tcp-server.ts | 74 -- src/packages/project/servers/hub/types.ts | 8 - .../servers/hub/write-text-file-to-project.ts | 20 - src/packages/project/servers/init.ts | 7 +- src/packages/server/conat/api/db.ts | 2 - src/packages/server/projects/call.ts | 25 - .../server/projects/connection/README.md | 18 - .../server/projects/connection/call.ts | 37 - .../server/projects/connection/connect.ts | 132 ---- .../server/projects/connection/handle-blob.ts | 71 -- .../projects/connection/handle-message.ts | 105 --- .../projects/connection/handle-query.ts | 117 --- .../projects/connection/handle-version.ts | 34 - .../server/projects/connection/heartbeat.ts | 17 - .../server/projects/connection/index.ts | 2 - .../server/projects/connection/initialize.ts | 51 -- src/packages/server/projects/control/index.ts | 12 - src/packages/util/message.d.ts | 3 - src/packages/util/message.js | 160 ---- 27 files changed, 7 insertions(+), 2066 deletions(-) delete mode 100644 src/packages/hub/local_hub_connection.coffee delete mode 100644 src/packages/hub/projects.coffee delete mode 100644 src/packages/project/read_write_files.ts delete mode 100644 src/packages/project/servers/hub/handle-message.ts delete mode 100644 src/packages/project/servers/hub/read-text-file-from-project.ts delete mode 100644 src/packages/project/servers/hub/tcp-server.ts delete mode 100644 src/packages/project/servers/hub/types.ts delete mode 100644 src/packages/project/servers/hub/write-text-file-to-project.ts delete mode 100644 src/packages/server/projects/call.ts delete mode 100644 src/packages/server/projects/connection/README.md delete mode 100644 src/packages/server/projects/connection/call.ts delete mode 100644 src/packages/server/projects/connection/connect.ts delete mode 100644 src/packages/server/projects/connection/handle-blob.ts delete mode 100644 src/packages/server/projects/connection/handle-message.ts delete mode 100644 src/packages/server/projects/connection/handle-query.ts delete mode 100644 src/packages/server/projects/connection/handle-version.ts delete mode 100644 src/packages/server/projects/connection/heartbeat.ts delete mode 100644 src/packages/server/projects/connection/index.ts delete mode 100644 src/packages/server/projects/connection/initialize.ts diff --git a/src/packages/database/postgres-user-queries.coffee b/src/packages/database/postgres-user-queries.coffee index 2cb8f7754d..34304cbacf 100644 --- a/src/packages/database/postgres-user-queries.coffee +++ b/src/packages/database/postgres-user-queries.coffee @@ -1762,13 +1762,6 @@ awaken_project = (db, project_id, cb) -> cb() catch err cb("error starting project = #{err}") - (cb) -> - if not db.ensure_connection_to_project? - cb() - return - dbg("also make sure there is a connection from hub to project") - # This is so the project can find out that the user wants to save a file (etc.) - db.ensure_connection_to_project(project_id, cb) ], (err) -> if err dbg("awaken project error -- #{err}") diff --git a/src/packages/database/postgres/types.ts b/src/packages/database/postgres/types.ts index 939a4b81cf..de4208b5ef 100644 --- a/src/packages/database/postgres/types.ts +++ b/src/packages/database/postgres/types.ts @@ -355,8 +355,6 @@ export interface PostgreSQL extends EventEmitter { projectControl?: (project_id: string) => Project; - ensure_connection_to_project?: (project_id: string, cb?: CB) => Promise; - get_blob(opts: { uuid: string; save_in_db?: boolean; diff --git a/src/packages/hub/local_hub_connection.coffee b/src/packages/hub/local_hub_connection.coffee deleted file mode 100644 index cb48d3fef7..0000000000 --- a/src/packages/hub/local_hub_connection.coffee +++ /dev/null @@ -1,701 +0,0 @@ -######################################################################### -# This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. -# License: MS-RSL – see LICENSE.md for details -######################################################################### - -### -NOTE/ATTENTION! - -A "local hub" is exactly the same thing as a "project". I just used to call -them "local hubs" a very long time ago. - -### - - -{PROJECT_HUB_HEARTBEAT_INTERVAL_S} = require('@cocalc/util/heartbeat') - -# Connection to a Project (="local hub", for historical reasons only.) - -async = require('async') -{callback2} = require('@cocalc/util/async-utils') - -uuid = require('uuid') -winston = require('./logger').getLogger('local-hub-connection') -underscore = require('underscore') - -message = require('@cocalc/util/message') -misc_node = require('@cocalc/backend/misc_node') -{connectToLockedSocket} = require("@cocalc/backend/tcp/locked-socket") -misc = require('@cocalc/util/misc') -{defaults, required} = misc - -blobs = require('./blobs') - -# Blobs (e.g., files dynamically appearing as output in worksheets) are kept for this -# many seconds before being discarded. If the worksheet is saved (e.g., by a user's autosave), -# then the BLOB is saved indefinitely. -BLOB_TTL_S = 60*60*24 # 1 day - -if not process.env.SMC_TEST - DEBUG = true - -connect_to_a_local_hub = (opts) -> # opts.cb(err, socket) - opts = defaults opts, - port : required - host : required - secret_token : required - timeout : 10 - cb : required - - try - socket = await connectToLockedSocket({port:opts.port, host:opts.host, token:opts.secret_token, timeout:opts.timeout}) - misc_node.enable_mesg(socket, 'connection_to_a_local_hub') - opts.cb(undefined, socket) - catch err - opts.cb(err) - -_local_hub_cache = {} -exports.new_local_hub = (project_id, database, projectControl) -> - if not project_id? - throw "project_id must be specified (it is undefined)" - H = _local_hub_cache[project_id] - if H? - winston.debug("new_local_hub('#{project_id}') -- using cached version") - else - winston.debug("new_local_hub('#{project_id}') -- creating new one") - H = new LocalHub(project_id, database, projectControl) - _local_hub_cache[project_id] = H - return H - -exports.connect_to_project = (project_id, database, projectControl, cb) -> - hub = exports.new_local_hub(project_id, database, projectControl) - hub.local_hub_socket (err) -> - if err - winston.debug("connect_to_project: error ensuring connection to #{project_id} -- #{err}") - else - winston.debug("connect_to_project: successfully ensured connection to #{project_id}") - cb?(err) - -exports.disconnect_from_project = (project_id) -> - H = _local_hub_cache[project_id] - delete _local_hub_cache[project_id] - H?.free_resources() - return - -exports.all_local_hubs = () -> - v = [] - for k, h of _local_hub_cache - if h? - v.push(h) - return v - -server_settings = undefined -init_server_settings = () -> - server_settings = await require('./servers/server-settings').default() - update = () -> - winston.debug("local_hub_connection (version might have changed) -- checking on clients") - for x in exports.all_local_hubs() - x.restart_if_version_too_old() - update() - server_settings.table.on('change', update) - -class LocalHub # use the function "new_local_hub" above; do not construct this directly! - constructor: (@project_id, @database, @projectControl) -> - if not server_settings? # module being used -- make sure server_settings is initialized - init_server_settings() - @_local_hub_socket_connecting = false - @_sockets = {} # key = session_uuid:client_id - @_sockets_by_client_id = {} #key = client_id, value = list of sockets for that client - @call_callbacks = {} - @path = '.' # should deprecate - *is* used by some random code elsewhere in this file - @dbg("getting deployed running project") - - init_heartbeat: => - @dbg("init_heartbeat") - if @_heartbeat_interval? # already running - @dbg("init_heartbeat -- already running") - return - send_heartbeat = => - @dbg("init_heartbeat -- send") - @_socket?.write_mesg('json', message.heartbeat()) - @_heartbeat_interval = setInterval(send_heartbeat, PROJECT_HUB_HEARTBEAT_INTERVAL_S*1000) - - delete_heartbeat: => - if @_heartbeat_interval? - @dbg("delete_heartbeat") - clearInterval(@_heartbeat_interval) - delete @_heartbeat_interval - - project: (cb) => - try - cb(undefined, await @projectControl(@project_id)) - catch err - cb(err) - - dbg: (m) => - ## only enable when debugging - if DEBUG - winston.debug("local_hub('#{@project_id}'): #{misc.to_json(m)}") - - restart: (cb) => - @dbg("restart") - @free_resources() - try - await (await @projectControl(@project_id)).restart() - cb() - catch err - cb(err) - - status: (cb) => - @dbg("status: get status of a project") - try - cb(undefined, await (await @projectControl(@project_id)).status()) - catch err - cb(err) - - state: (cb) => - @dbg("state: get state of a project") - try - cb(undefined, await (await @projectControl(@project_id)).state()) - catch err - cb(err) - - free_resources: () => - @dbg("free_resources") - @query_cancel_all_changefeeds() - @delete_heartbeat() - delete @_ephemeral - if @_ephemeral_timeout - clearTimeout(@_ephemeral_timeout) - delete @_ephemeral_timeout - delete @address # so we don't continue trying to use old address - delete @_status - delete @smc_version # so when client next connects we ignore version checks until they tell us their version - try - @_socket?.end() - winston.debug("free_resources: closed main local_hub socket") - catch e - winston.debug("free_resources: exception closing main _socket: #{e}") - delete @_socket - for k, s of @_sockets - try - s.end() - winston.debug("free_resources: closed #{k}") - catch e - winston.debug("free_resources: exception closing a socket: #{e}") - @_sockets = {} - @_sockets_by_client_id = {} - - free_resources_for_client_id: (client_id) => - v = @_sockets_by_client_id[client_id] - if v? - @dbg("free_resources_for_client_id(#{client_id}) -- #{v.length} sockets") - for socket in v - try - socket.end() - socket.destroy() - catch e - # do nothing - delete @_sockets_by_client_id[client_id] - - # async - init_ephemeral: () => - settings = await callback2(@database.get_project_settings, {project_id:@project_id}) - @_ephemeral = misc.copy_with(settings, ['ephemeral_disk', 'ephemeral_state']) - @dbg("init_ephemeral -- #{JSON.stringify(@_ephemeral)}") - # cache for 60s - @_ephemeral_timeout = setTimeout((() => delete @_ephemeral), 60000) - - ephemeral_disk: () => - if not @_ephemeral? - await @init_ephemeral() - return @_ephemeral.ephemeral_disk - - ephemeral_state: () => - if not @_ephemeral? - await @init_ephemeral() - return @_ephemeral.ephemeral_state - - # - # Project query support code - # - mesg_query: (mesg, write_mesg) => - dbg = (m) => winston.debug("mesg_query(project_id='#{@project_id}'): #{misc.trunc(m,200)}") - dbg(misc.to_json(mesg)) - query = mesg.query - if not query? - write_mesg(message.error(error:"query must be defined")) - return - if await @ephemeral_state() - @dbg("project has ephemeral state") - write_mesg(message.error(error:"FATAL -- project has ephemeral state so no database queries are allowed")) - return - @dbg("project does NOT have ephemeral state") - first = true - if mesg.changes - @_query_changefeeds ?= {} - @_query_changefeeds[mesg.id] = true - mesg_id = mesg.id - @database.user_query - project_id : @project_id - query : query - options : mesg.options - changes : if mesg.changes then mesg_id - cb : (err, result) => - if result?.action == 'close' - err = 'close' - if err - dbg("project_query error: #{misc.to_json(err)}") - if @_query_changefeeds?[mesg_id] - delete @_query_changefeeds[mesg_id] - write_mesg(message.error(error:err)) - if mesg.changes and not first - # also, assume changefeed got messed up, so cancel it. - @database.user_query_cancel_changefeed(id : mesg_id) - else - #if Math.random() <= .3 # for testing -- force forgetting about changefeed with probability 10%. - # delete @_query_changefeeds[mesg_id] - if mesg.changes and not first - resp = result - resp.id = mesg_id - resp.multi_response = true - else - first = false - resp = mesg - resp.query = result - write_mesg(resp) - - mesg_query_cancel: (mesg, write_mesg) => - if not @_query_changefeeds? - # no changefeeds - write_mesg(mesg) - else - @database.user_query_cancel_changefeed - id : mesg.id - cb : (err, resp) => - if err - write_mesg(message.error(error:err)) - else - mesg.resp = resp - write_mesg(mesg) - delete @_query_changefeeds?[mesg.id] - - query_cancel_all_changefeeds: (cb) => - if not @_query_changefeeds? or @_query_changefeeds.length == 0 - cb?(); return - dbg = (m) => winston.debug("query_cancel_all_changefeeds(project_id='#{@project_id}'): #{m}") - v = @_query_changefeeds - dbg("canceling #{v.length} changefeeds") - delete @_query_changefeeds - f = (id, cb) => - dbg("canceling id=#{id}") - @database.user_query_cancel_changefeed - id : id - cb : (err) => - if err - dbg("FEED: warning #{id} -- error canceling a changefeed #{misc.to_json(err)}") - else - dbg("FEED: canceled changefeed -- #{id}") - cb() - async.map(misc.keys(v), f, (err) => cb?(err)) - - # async -- throws error if project doesn't have access to string with this id. - check_syncdoc_access: (string_id) => - if not typeof string_id == 'string' and string_id.length == 40 - throw Error('string_id must be specified and valid') - return - opts = - query : "SELECT project_id FROM syncstrings" - where : {"string_id = $::CHAR(40)" : string_id} - results = await callback2(@database._query, opts) - if results.rows.length != 1 - throw Error("no such syncdoc") - if results.rows[0].project_id != @project_id - throw Error("project does NOT have access to this syncdoc") - return # everything is fine. - - # - # end project query support code - # - - # local hub just told us its version. Record it. Restart project if hub version too old. - local_hub_version: (version) => - winston.debug("local_hub_version: version=#{version}") - @smc_version = version - @restart_if_version_too_old() - - # If our known version of the project is too old compared to the - # current version_min_project in smcu-util/smc-version, then - # we restart the project, which updates the code to the latest - # version. Only restarts the project if we have an open control - # socket to it. - # Please make damn sure to update the project code on the compute - # server before updating the version, or the project will be - # forced to restart and it won't help! - restart_if_version_too_old: () => - if not @_socket? - # not connected at all -- just return - return - if not @smc_version? - # client hasn't told us their version yet - return - if server_settings.version.version_min_project <= @smc_version - # the project is up to date - return - if @_restart_goal_version == server_settings.version.version_min_project - # We already restarted the project in an attempt to update it to this version - # and it didn't get updated. Don't try again until @_restart_version is cleared, since - # we don't want to lock a user out of their project due to somebody forgetting - # to update code on the compute server! It could also be that the project just - # didn't finish restarting. - return - - winston.debug("restart_if_version_too_old(#{@project_id}): #{@smc_version}, #{server_settings.version.version_min_project}") - # record some stuff so that we don't keep trying to restart the project constantly - ver = @_restart_goal_version = server_settings.version.version_min_project # version which we tried to get to - f = () => - if @_restart_goal_version == ver - delete @_restart_goal_version - setTimeout(f, 15*60*1000) # don't try again for at least 15 minutes. - - @dbg("restart_if_version_too_old -- restarting since #{server_settings.version.version_min_project} > #{@smc_version}") - @restart (err) => - @dbg("restart_if_version_too_old -- done #{err}") - - # handle incoming JSON messages from the local_hub - handle_mesg: (mesg, socket) => - @dbg("local_hub --> hub: received mesg: #{misc.trunc(misc.to_json(mesg), 250)}") - if mesg.client_id? - # Should we worry about ensuring that message from this local hub are allowed to - # send messages to this client? NO. For them to send a message, they would have to - # know the client's id, which is a random uuid, assigned each time the user connects. - # It obviously is known to the local hub -- but if the user has connected to the local - # hub then they should be allowed to receive messages. - # *DEPRECATED* - return - if mesg.event == 'version' - @local_hub_version(mesg.version) - return - if mesg.id? - f = @call_callbacks[mesg.id] - if f? - f(mesg) - else - winston.debug("handling call from local_hub") - write_mesg = (resp) => - resp.id = mesg.id - @local_hub_socket (err, sock) => - if not err - sock.write_mesg('json', resp) - switch mesg.event - when 'ping' - write_mesg(message.pong()) - when 'query' - @mesg_query(mesg, write_mesg) - when 'query_cancel' - @mesg_query_cancel(mesg, write_mesg) - when 'file_written_to_project' - # ignore -- don't care; this is going away - return - when 'file_read_from_project' - # handle elsewhere by the code that requests the file - return - when 'error' - # ignore -- don't care since handler already gone. - return - else - write_mesg(message.error(error:"unknown event '#{mesg.event}'")) - return - - handle_blob: (opts) => - opts = defaults opts, - uuid : required - blob : required - - @dbg("local_hub --> global_hub: received a blob with uuid #{opts.uuid}") - # Store blob in DB. - blobs.save_blob - uuid : opts.uuid - blob : opts.blob - project_id : @project_id - ttl : BLOB_TTL_S - check : true # if malicious user tries to overwrite a blob with given sha1 hash, they get an error. - database : @database - cb : (err, ttl) => - if err - resp = message.save_blob(sha1:opts.uuid, error:err) - @dbg("handle_blob: error! -- #{err}") - else - resp = message.save_blob(sha1:opts.uuid, ttl:ttl) - - @local_hub_socket (err, socket) => - if not err - socket.write_mesg('json', resp) - - # Connection to the remote local_hub daemon that we use for control. - local_hub_socket: (cb) => - if @_socket? - #@dbg("local_hub_socket: re-using existing socket") - cb(undefined, @_socket) - return - - if @_local_hub_socket_connecting - @_local_hub_socket_queue.push(cb) - @dbg("local_hub_socket: added socket request to existing queue, which now has length #{@_local_hub_socket_queue.length}") - return - @_local_hub_socket_connecting = true - @_local_hub_socket_queue = [cb] - connecting_timer = undefined - - cancel_connecting = () => - @_local_hub_socket_connecting = false - if @_local_hub_socket_queue? - @dbg("local_hub_socket: canceled due to timeout") - for c in @_local_hub_socket_queue - c?('timeout') - delete @_local_hub_socket_queue - clearTimeout(connecting_timer) - - # If below fails for 20s for some reason, cancel everything to allow for future attempt. - connecting_timer = setTimeout(cancel_connecting, 20000) - - @dbg("local_hub_socket: getting new socket") - @new_socket (err, socket) => - if not @_local_hub_socket_queue? - # already gave up. - return - @_local_hub_socket_connecting = false - @dbg("local_hub_socket: new_socket returned #{err}") - if err - for c in @_local_hub_socket_queue - c?(err) - delete @_local_hub_socket_queue - else - socket.on 'mesg', (type, mesg) => - switch type - when 'blob' - @handle_blob(mesg) - when 'json' - @handle_mesg(mesg, socket) - - socket.on('end', @free_resources) - socket.on('close', @free_resources) - socket.on('error', @free_resources) - - # Send a hello message to the local hub, so it knows this is the control connection, - # and not something else (e.g., a console). - socket.write_mesg('json', {event:'hello'}) - - for c in @_local_hub_socket_queue - c?(undefined, socket) - delete @_local_hub_socket_queue - - @_socket = socket - @init_heartbeat() # start sending heartbeat over this socket - - # Finally, we wait a bit to see if the version gets sent from - # the client. If not, we set it to 0, which will cause a restart, - # which will upgrade to a new version that sends versions. - # TODO: This code can be deleted after all projects get restarted. - check_version_received = () => - if @_socket? and not @smc_version? - @smc_version = 0 - @restart_if_version_too_old() - setTimeout(check_version_received, 60*1000) - - cancel_connecting() - - # Get a new connection to the local_hub, - # authenticated via the secret_token, and enhanced - # to be able to send/receive json and blob messages. - new_socket: (cb) => # cb(err, socket) - @dbg("new_socket") - f = (cb) => - if not @address? - cb("no address") - return - if not @address.port? - cb("no port") - return - if not @address.host? - cb("no host") - return - if not @address.secret_token? - cb("no secret_token") - return - connect_to_a_local_hub - port : @address.port - host : @address.ip ? @address.host # prefer @address.ip if it exists (e.g., for cocalc-kubernetes); otherwise use host (which is where compute server is). - secret_token : @address.secret_token - cb : cb - socket = undefined - async.series([ - (cb) => - if not @address? - @dbg("get address of a working local hub") - try - @address = await (await @projectControl(@project_id)).address() - cb() - catch err - cb(err) - else - cb() - (cb) => - @dbg("try to connect to local hub socket using last known address") - f (err, _socket) => - if not err - socket = _socket - cb() - else - @dbg("failed to get address of a working local hub -- #{err}") - try - @address = await (await @projectControl(@project_id)).address() - cb() - catch err - cb(err) - (cb) => - if not socket? - @dbg("still don't have our connection -- try again") - f (err, _socket) => - socket = _socket; cb(err) - else - cb() - ], (err) => - cb(err, socket) - ) - - remove_multi_response_listener: (id) => - delete @call_callbacks[id] - - call: (opts) => - opts = defaults opts, - mesg : required - timeout : undefined # NOTE: a nonzero timeout MUST be specified, or we will not even listen for a response from the local hub! (Ensures leaking listeners won't happen.) - multi_response : false # if true, timeout ignored; call @remove_multi_response_listener(mesg.id) to remove - cb : undefined - @dbg("call") - if not opts.mesg.id? - if opts.timeout or opts.multi_response # opts.timeout being undefined or 0 both mean "don't do it" - opts.mesg.id = uuid.v4() - - @local_hub_socket (err, socket) => - if err - @dbg("call: failed to get socket -- #{err}") - opts.cb?(err) - return - @dbg("call: get socket -- now writing message to the socket -- #{misc.trunc(misc.to_json(opts.mesg),200)}") - socket.write_mesg 'json', opts.mesg, (err) => - if err - @free_resources() # at least next time it will get a new socket - opts.cb?(err) - return - if opts.multi_response - @call_callbacks[opts.mesg.id] = opts.cb - else if opts.timeout - # Listen to exactly one response, them remove the listener: - @call_callbacks[opts.mesg.id] = (resp) => - delete @call_callbacks[opts.mesg.id] - if resp.event == 'error' - opts.cb(resp.error) - else - opts.cb(undefined, resp) - # As mentioned above -- there's no else -- if not timeout then - # we do not listen for a response. - - # Read a file from a project into memory on the hub. - # I think this is used only by the API, but not by browser clients anymore. - read_file: (opts) => # cb(err, content_of_file) - {path, project_id, archive, cb} = defaults opts, - path : required - project_id : required - archive : 'tar.bz2' # for directories; if directory, then the output object "data" has data.archive=actual extension used. - cb : required - @dbg("read_file '#{path}'") - socket = undefined - id = uuid.v4() - data = undefined - data_uuid = undefined - result_archive = undefined - - async.series([ - # Get a socket connection to the local_hub. - (cb) => - @local_hub_socket (err, _socket) => - if err - cb(err) - else - socket = _socket - cb() - (cb) => - socket.write_mesg('json', message.read_file_from_project(id:id, project_id:project_id, path:path, archive:archive)) - socket.recv_mesg - type : 'json' - id : id - timeout : 60 - cb : (mesg) => - switch mesg.event - when 'error' - cb(mesg.error) - when 'file_read_from_project' - data_uuid = mesg.data_uuid - result_archive = mesg.archive - cb() - else - cb("Unknown mesg event '#{mesg.event}'") - (cb) => - socket.recv_mesg - type : 'blob' - id : data_uuid - timeout : 60 - cb : (_data) => - # recv_mesg returns either a Buffer blob - # *or* a {event:'error', error:'the error'} object. - # Fortunately `new Buffer().event` is valid (and undefined). - if _data.event == 'error' - cb(_data.error) - else - data = _data - data.archive = result_archive - cb() - ], (err) => - if err - cb(err) - else - cb(undefined, data) - ) - - # Write a file to a project - # I think this is used only by the API, but not by browser clients anymore. - write_file: (opts) => # cb(err) - {path, project_id, cb, data} = defaults opts, - path : required - project_id : required - data : required # what to write - cb : required - @dbg("write_file '#{path}'") - id = uuid.v4() - data_uuid = uuid.v4() - - @local_hub_socket (err, socket) => - if err - opts.cb(err) - return - mesg = message.write_file_to_project - id : id - project_id : project_id - path : path - data_uuid : data_uuid - socket.write_mesg('json', mesg) - socket.write_mesg('blob', {uuid:data_uuid, blob:data}) - socket.recv_mesg - type : 'json' - id : id - timeout : 10 - cb : (mesg) => - switch mesg.event - when 'file_written_to_project' - opts.cb() - when 'error' - opts.cb(mesg.error) - else - opts.cb("unexpected message type '#{mesg.event}'") diff --git a/src/packages/hub/projects.coffee b/src/packages/hub/projects.coffee deleted file mode 100644 index 7a586abbaa..0000000000 --- a/src/packages/hub/projects.coffee +++ /dev/null @@ -1,101 +0,0 @@ -######################################################################### -# This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. -# License: MS-RSL – see LICENSE.md for details -######################################################################### - -### -Projects -### - -winston = require('./logger').getLogger('projects') - -postgres = require('@cocalc/database') -local_hub_connection = require('./local_hub_connection') -message = require('@cocalc/util/message') -{callback2} = require('@cocalc/util/async-utils') -misc = require('@cocalc/util/misc') -misc_node = require('@cocalc/backend/misc_node') -{defaults, required} = misc - -# Create a project object that is connected to a local hub (using -# appropriate port and secret token), login, and enhance socket -# with our message protocol. - -_project_cache = {} -exports.new_project = (project_id, database, projectControl) -> - P = _project_cache[project_id] - if not P? - P = new Project(project_id, database, projectControl) - _project_cache[project_id] = P - return P - -class Project - constructor: (@project_id, @database, @projectControl) -> - @dbg("instantiating Project class") - @local_hub = local_hub_connection.new_local_hub(@project_id, @database, @projectControl) - # we always look this up and cache it - @get_info() - - dbg: (m) => - winston.debug("project(#{@project_id}): #{m}") - - _fixpath: (obj) => - if obj? and @local_hub? - if obj.path? - if obj.path[0] != '/' - obj.path = @local_hub.path+ '/' + obj.path - else - obj.path = @local_hub.path - - owner: (cb) => - if not @database? - cb('need database in order to determine owner') - return - @database.get_project - project_id : @project_id - columns : ['account_id'] - cb : (err, result) => - if err - cb(err) - else - cb(err, result[0]) - - # get latest info about project from database - get_info: (cb) => - if not @database? - cb('need database in order to determine owner') - return - @database.get_project - project_id : @project_id - columns : postgres.PROJECT_COLUMNS - cb : (err, result) => - if err - cb?(err) - else - @cached_info = result - cb?(undefined, result) - - call: (opts) => - opts = defaults opts, - mesg : required - multi_response : false - timeout : 15 - cb : undefined - #@dbg("call") - @_fixpath(opts.mesg) - opts.mesg.project_id = @project_id - @local_hub.call(opts) - - - read_file: (opts) => - @dbg("read_file") - @_fixpath(opts) - opts.project_id = @project_id - @local_hub.read_file(opts) - - write_file: (opts) => - @dbg("write_file") - @_fixpath(opts) - opts.project_id = @project_id - @local_hub.write_file(opts) - diff --git a/src/packages/next/pages/api/v2/projects/copy-url.ts b/src/packages/next/pages/api/v2/projects/copy-url.ts index c38cc73ac8..d71e4d206c 100644 --- a/src/packages/next/pages/api/v2/projects/copy-url.ts +++ b/src/packages/next/pages/api/v2/projects/copy-url.ts @@ -3,16 +3,15 @@ API endpoint to copy from a URL on the internet to a project. This requires the user to be signed in with appropriate access to the project. -If project doesn't have network access, we stop the project, start it with -network access, get the content, then restart the project without network access. +This grabs one file, reads it into memory, then writes it to disk in the project. */ import getAccountId from "lib/account/get-account"; import { isValidUUID } from "@cocalc/util/misc"; import isCollaborator from "@cocalc/server/projects/is-collaborator"; import getParams from "lib/api/get-params"; -import call from "@cocalc/server/projects/connection/call"; import getProxiedPublicPathInfo from "lib/share/proxy/get-proxied-public-path-info"; +import { conat } from "@cocalc/backend/conat"; export default async function handle(req, res) { const params = getParams(req); @@ -43,13 +42,9 @@ export default async function handle(req, res) { } const i = url.lastIndexOf("/"); const filename = url.slice(i + 1); - const mesg = { - event: "write_text_file_to_project", - path: path ? path : filename, - content: info.contents.content, - }; - const response = await call({ project_id, mesg }); - res.json({ response }); + const fs = conat().fs({ project_id }); + await fs.writeFile(path ? path : filename, info.contents.content); + res.json({}); } catch (err) { res.json({ error: `${err.message}` }); } diff --git a/src/packages/project/read_write_files.ts b/src/packages/project/read_write_files.ts deleted file mode 100644 index 43ec040bec..0000000000 --- a/src/packages/project/read_write_files.ts +++ /dev/null @@ -1,190 +0,0 @@ -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/main/docs/suggestions.md - */ -//######################################################################## -// This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. -// License: MS-RSL – see LICENSE.md for details -//######################################################################## - -import { CoCalcSocket } from "@cocalc/backend/tcp/enable-messaging-protocol"; -import { execFile } from "node:child_process"; -import { constants, Stats } from "node:fs"; -import { - access, - readFile as readFileAsync, - stat as statAsync, - unlink, - writeFile, -} from "node:fs/promises"; -import * as temp from "temp"; - -import ensureContainingDirectoryExists from "@cocalc/backend/misc/ensure-containing-directory-exists"; -import { abspath, uuidsha1 } from "@cocalc/backend/misc_node"; -import * as message from "@cocalc/util/message"; -import { path_split } from "@cocalc/util/misc"; -import { check_file_size } from "./common"; - -import { getLogger } from "@cocalc/backend/logger"; -const winston = getLogger("read-write-files"); - -//############################################## -// Read and write individual files -//############################################## - -// Read a file located in the given project. This will result in an -// error if the readFile function fails, e.g., if the file doesn't -// exist or the project is not open. We then send the resulting file -// over the socket as a blob message. -// -// Directories get sent as a ".tar.bz2" file. -// TODO: should support -- 'tar', 'tar.bz2', 'tar.gz', 'zip', '7z'. and mesg.archive option!!! -// -export async function read_file_from_project(socket: CoCalcSocket, mesg) { - const dbg = (...m) => - winston.debug(`read_file_from_project(path='${mesg.path}'): `, ...m); - dbg("called"); - let data: Buffer | undefined = undefined; - let path = abspath(mesg.path); - let is_dir: boolean | undefined = undefined; - let id: string | undefined = undefined; - let target: string | undefined = undefined; - let archive = undefined; - let stats: Stats | undefined = undefined; - - try { - //dbg("Determine whether the path '#{path}' is a directory or file.") - stats = await statAsync(path); - is_dir = stats.isDirectory(); - - // make sure the file isn't too large - const size_check = check_file_size(stats.size); - if (size_check) { - throw new Error(size_check); - } - - // tar jcf a directory - if (is_dir) { - if (mesg.archive !== "tar.bz2") { - throw new Error( - "The only supported directory archive format is tar.bz2" - ); - } - target = temp.path({ suffix: "." + mesg.archive }); - //dbg("'#{path}' is a directory, so archive it to '#{target}', change path, and read that file") - archive = mesg.archive; - if (path[path.length - 1] === "/") { - // common nuisance with paths to directories - path = path.slice(0, path.length - 1); - } - const split = path_split(path); - // TODO same patterns also in project.ts - const args = [ - "--exclude=.sagemathcloud*", - "--exclude=.forever", - "--exclude=.node*", - "--exclude=.npm", - "--exclude=.sage", - "-jcf", - target as string, - split.tail, - ]; - //dbg("tar #{args.join(' ')}") - await new Promise((resolve, reject) => { - execFile( - "tar", - args, - { cwd: split.head }, - function (err, stdout, stderr) { - if (err) { - winston.debug( - `Issue creating tarball: ${err}, ${stdout}, ${stderr}` - ); - return reject(err); - } else { - return resolve(); - } - } - ); - }); - } else { - //Nothing to do, it is a file. - target = path; - } - if (!target) { - throw Error("bug -- target must be set"); - } - - //dbg("Read the file into memory.") - data = await readFileAsync(target); - - // get SHA1 of contents - if (data == null) { - throw new Error("data is null"); - } - id = uuidsha1(data); - //dbg("sha1 hash = '#{id}'") - - //dbg("send the file as a blob back to the hub.") - socket.write_mesg( - "json", - message.file_read_from_project({ - id: mesg.id, - data_uuid: id, - archive, - }) - ); - - socket.write_mesg("blob", { - uuid: id, - blob: data, - ttlSeconds: mesg.ttlSeconds, // TODO does ttlSeconds work? - }); - } catch (err) { - if (err && err !== "file already known") { - socket.write_mesg( - "json", - message.error({ id: mesg.id, error: `${err}` }) - ); - } - } - - // in any case, clean up the temporary archive - if (is_dir && target) { - try { - await access(target, constants.F_OK); - //dbg("It was a directory, so remove the temporary archive '#{path}'.") - await unlink(target); - } catch (err) { - winston.debug(`Error removing temporary archive '${target}': ${err}`); - } - } -} - -export function write_file_to_project(socket: CoCalcSocket, mesg) { - const dbg = (...m) => - winston.debug(`write_file_to_project(path='${mesg.path}'): `, ...m); - dbg("called"); - - const { data_uuid } = mesg; - const path = abspath(mesg.path); - - // Listen for the blob containing the actual content that we will write. - const write_file = async function (type, value) { - if (type === "blob" && value.uuid === data_uuid) { - socket.removeListener("mesg", write_file); - try { - await ensureContainingDirectoryExists(path); - await writeFile(path, value.blob); - socket.write_mesg( - "json", - message.file_written_to_project({ id: mesg.id }) - ); - } catch (err) { - socket.write_mesg("json", message.error({ id: mesg.id, error: err })); - } - } - }; - socket.on("mesg", write_file); -} diff --git a/src/packages/project/servers/hub/handle-message.ts b/src/packages/project/servers/hub/handle-message.ts deleted file mode 100644 index 35a4f94aae..0000000000 --- a/src/packages/project/servers/hub/handle-message.ts +++ /dev/null @@ -1,141 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -Handle a general message from the hub. These are the generic message, -as opposed to the messages specific to "client" functionality such as -database queries. -*/ - -import processKill from "@cocalc/backend/misc/process-kill"; -import { CoCalcSocket } from "@cocalc/backend/tcp/enable-messaging-protocol"; -import { handle_save_blob_message } from "@cocalc/project/blobs"; -import { getClient } from "@cocalc/project/client"; -import { project_id } from "@cocalc/project/data"; -import { exec_shell_code } from "@cocalc/project/exec_shell_code"; -import { getLogger } from "@cocalc/project/logger"; -import { print_to_pdf } from "@cocalc/project/print_to_pdf"; -import { - read_file_from_project, - write_file_to_project, -} from "@cocalc/project/read_write_files"; -import * as message from "@cocalc/util/message"; -import { version } from "@cocalc/util/smc-version"; -import { Message } from "./types"; -import writeTextFileToProject from "./write-text-file-to-project"; -import readTextFileFromProject from "./read-text-file-from-project"; - -const logger = getLogger("handle-message-from-hub"); - -export default async function handleMessage( - socket: CoCalcSocket, - mesg: Message, -) { - logger.debug("received a message", { - event: mesg.event, - id: mesg.id, - "...": "...", - }); - - // We can't just log this in general, since it can be big. - // So only uncomment this for low level debugging, unfortunately. - // logger.debug("received ", mesg); - - if (getClient().handle_mesg(mesg, socket)) { - return; - } - - switch (mesg.event) { - case "heartbeat": - logger.debug(`received heartbeat on socket '${socket.id}'`); - // Update the last hearbeat timestamp, so we know socket is working. - socket.heartbeat = new Date(); - return; - - case "ping": - // ping message is used only for debugging purposes. - socket.write_mesg("json", message.pong({ id: mesg.id })); - return; - - case "project_exec": - // this is no longer used by web browser clients; however it *is* used by the HTTP api served - // by the hub to api key users, so do NOT remove it! E.g., the latex endpoint, the compute - // server, etc., use it. The web browser clients use the websocket api. - exec_shell_code(socket, mesg); - return; - - // Reading and writing files to/from project and sending over socket - case "read_file_from_project": - read_file_from_project(socket, mesg); - return; - - case "write_file_to_project": - write_file_to_project(socket, mesg); - return; - - case "write_text_file_to_project": - writeTextFileToProject(socket, mesg); - return; - - case "read_text_file_from_project": - readTextFileFromProject(socket, mesg); - return; - - case "print_to_pdf": - print_to_pdf(socket, mesg); - return; - - case "send_signal": - if ( - mesg.pid && - (mesg.signal == 2 || mesg.signal == 3 || mesg.signal == 9) - ) { - processKill(mesg.pid, mesg.signal); - } else { - if (mesg.id) { - socket.write_mesg( - "json", - message.error({ - id: mesg.id, - error: "invalid pid or signal (must be 2,3,9)", - }), - ); - } - return; - } - if (mesg.id != null) { - // send back confirmation that a signal was sent - socket.write_mesg("json", message.signal_sent({ id: mesg.id })); - } - return; - - case "save_blob": - handle_save_blob_message(mesg); - return; - - case "error": - logger.error(`ERROR from hub: ${mesg.error}`); - return; - - case "hello": - // No action -- this is used by the hub to send an initial control message that has no effect, so that - // we know this socket will be used for control messages. - logger.info(`hello from hub -- sending back our version = ${version}`); - socket.write_mesg("json", message.version({ version })); - return; - - default: - if (mesg.id != null) { - // only respond with error if there is an id -- otherwise response has no meaning to hub. - const err = message.error({ - id: mesg.id, - error: `Project ${project_id} does not implement handling mesg with event='${mesg.event}'`, - }); - socket.write_mesg("json", err); - } else { - logger.debug(`Dropping unknown message with event='${mesg.event}'`); - } - } -} diff --git a/src/packages/project/servers/hub/read-text-file-from-project.ts b/src/packages/project/servers/hub/read-text-file-from-project.ts deleted file mode 100644 index 7df319ccd7..0000000000 --- a/src/packages/project/servers/hub/read-text-file-from-project.ts +++ /dev/null @@ -1,21 +0,0 @@ -import * as message from "@cocalc/util/message"; -import { readFile } from "fs/promises"; - -export default async function readTextFileFromProject( - socket, - mesg -): Promise { - const { path } = mesg; - try { - const content = (await readFile(path)).toString(); - socket.write_mesg( - "json", - message.text_file_read_from_project({ id: mesg.id, content }) - ); - } catch (err) { - socket.write_mesg( - "json", - message.error({ id: mesg.id, error: err.message }) - ); - } -} diff --git a/src/packages/project/servers/hub/tcp-server.ts b/src/packages/project/servers/hub/tcp-server.ts deleted file mode 100644 index 2385f5bcf3..0000000000 --- a/src/packages/project/servers/hub/tcp-server.ts +++ /dev/null @@ -1,74 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2023 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* Create the TCP server that communicates with hubs */ - -import { writeFile } from "node:fs/promises"; -import { createServer } from "node:net"; -import * as uuid from "uuid"; -import enableMessagingProtocol, { - CoCalcSocket, -} from "@cocalc/backend/tcp/enable-messaging-protocol"; -import { unlockSocket } from "@cocalc/backend/tcp/locked-socket"; -import { hubPortFile, secretToken } from "@cocalc/project/data"; -import { getOptions } from "@cocalc/project/init-program"; -import { once } from "@cocalc/util/async-utils"; -import handleMessage from "./handle-message"; -import { getClient } from "@cocalc/project/client"; - -import { getLogger } from "@cocalc/project/logger"; -const winston = getLogger("hub-tcp-server"); - -export default async function init(): Promise { - winston.info("starting tcp server: project <--> hub..."); - const server = createServer(handleConnection); - const options = getOptions(); - server.listen(options.hubPort, options.hostname); - await once(server, "listening"); - const address = server.address(); - if (address == null || typeof address == "string") { - // null = failed; string doesn't happen since that's for unix domain - // sockets, which we aren't using. - // This is probably impossible, but it makes typescript happier. - throw Error("failed to assign a port"); - } - const { port } = address; - winston.info(`hub tcp_server listening ${options.hostname}:${port}`); - await writeFile(hubPortFile, `${port}`); -} - -async function handleConnection(socket: CoCalcSocket) { - winston.info(`*new* connection from ${socket.remoteAddress}`); - socket.on("error", (err) => { - winston.error(`socket '${socket.remoteAddress}' error - ${err}`); - }); - socket.on("close", () => { - winston.info(`*closed* connection from ${socket.remoteAddress}`); - }); - - try { - await unlockSocket(socket, secretToken); - } catch (err) { - winston.error( - "failed to unlock socket -- ignoring any future messages and closing connection", - ); - socket.destroy(new Error("invalid secret token")); - return; - } - - socket.id = uuid.v4(); - socket.heartbeat = new Date(); // obviously working now - enableMessagingProtocol(socket); - - socket.on("mesg", (type, mesg) => { - getClient().active_socket(socket); // record that this socket is active now. - if (type === "json") { - // non-JSON types are handled elsewhere, e.g., for sending binary data. - // I'm not sure that any other message types are actually used though. - // winston.debug("received json mesg", mesg); - handleMessage(socket, mesg); - } - }); -} diff --git a/src/packages/project/servers/hub/types.ts b/src/packages/project/servers/hub/types.ts deleted file mode 100644 index 9d0954a6f9..0000000000 --- a/src/packages/project/servers/hub/types.ts +++ /dev/null @@ -1,8 +0,0 @@ -export interface Message { - event: string; - id?: string; - pid?: number; - signal?: string | number; - error?: string; - name?: string; -} diff --git a/src/packages/project/servers/hub/write-text-file-to-project.ts b/src/packages/project/servers/hub/write-text-file-to-project.ts deleted file mode 100644 index 6c9a2c58fc..0000000000 --- a/src/packages/project/servers/hub/write-text-file-to-project.ts +++ /dev/null @@ -1,20 +0,0 @@ -import ensureContainingDirectoryExists from "@cocalc/backend/misc/ensure-containing-directory-exists"; -import * as message from "@cocalc/util/message"; -import { writeFile } from "fs/promises"; - -export default async function writeTextFileToProject( - socket, - mesg -): Promise { - const { content, path } = mesg; - try { - await ensureContainingDirectoryExists(path); - await writeFile(path, content); - socket.write_mesg("json", message.file_written_to_project({ id: mesg.id })); - } catch (err) { - socket.write_mesg( - "json", - message.error({ id: mesg.id, error: err.message }) - ); - } -} diff --git a/src/packages/project/servers/init.ts b/src/packages/project/servers/init.ts index 0293c57ffc..5075de8e51 100644 --- a/src/packages/project/servers/init.ts +++ b/src/packages/project/servers/init.ts @@ -8,15 +8,12 @@ import initPidFile from "./pid-file"; import initAPIServer from "@cocalc/project/http-api/server"; import initBrowserServer from "./browser/http-server"; -import initHubServer from "./hub/tcp-server"; - import { getLogger } from "@cocalc/project/logger"; -const winston = getLogger("init-project-server"); +const logger = getLogger("init-project-server"); export default async function init() { - winston.info("Write pid file to disk."); + logger.info("Write pid file to disk."); await initPidFile(); await initAPIServer(); await initBrowserServer(); - await initHubServer(); } diff --git a/src/packages/server/conat/api/db.ts b/src/packages/server/conat/api/db.ts index 81b9258bd4..263689ef43 100644 --- a/src/packages/server/conat/api/db.ts +++ b/src/packages/server/conat/api/db.ts @@ -30,8 +30,6 @@ export async function touch({ if (!(await isCollaborator({ account_id, project_id }))) { throw Error("user must be collaborator on project"); } - // TODO: we also connect still (this will of course go away very soon!!) - D.ensure_connection_to_project?.(project_id); await callback2(D.touch, { account_id, project_id, path, action }); } diff --git a/src/packages/server/projects/call.ts b/src/packages/server/projects/call.ts deleted file mode 100644 index 5f1c8d5fdb..0000000000 --- a/src/packages/server/projects/call.ts +++ /dev/null @@ -1,25 +0,0 @@ -import call from "@cocalc/server/projects/connection/call"; -import { isValidUUID } from "@cocalc/util/misc"; -import isCollaborator from "@cocalc/server/projects/is-collaborator"; - -export default async function callProject({ - account_id, - project_id, - mesg, -}): Promise { - if (!isValidUUID(account_id)) { - throw Error( - "callProject -- user must be authenticated (no account_id specified)", - ); - } - if (!isValidUUID(project_id)) { - throw Error("callProject -- must specify project_id"); - } - - if (!(await isCollaborator({ account_id, project_id }))) { - throw Error( - "callProject -- authenticated user must be a collaborator on the project", - ); - } - return await call({ project_id, mesg }); -} diff --git a/src/packages/server/projects/connection/README.md b/src/packages/server/projects/connection/README.md deleted file mode 100644 index 7ebec24cc5..0000000000 --- a/src/packages/server/projects/connection/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# Project Connection - -The Node.js "hub" servers are able to establish a TCP connection to any running project. The code in this directory creates and manages that connection. - -## What is this used for? - -This connection is used by hubs to implement a couple of things for projects (e.g., reading and writing a text file). - -This connection is used much more by the projects to send and share state. For example, projects can persist data from collaborative editing sessions and directory listings to the central PostgreSQL database via this connection. - -## How do is it work? - -There is a long random token associated to each project, which is stored in the database (or the file system). When a hub connects to a project via TCP, it must first send this token before any further communication is allowed. - -For security reasons, the TCP connection is _**always**_ initiated from a hub to the project, and there can be several distinct hubs connected to the same project at once. - -The project often wants to send information to the hub. If no hubs are connected to it, then that project must sit and wait for a connection. This is for security reasons, since in some contexts we do not allow the project to create any outgoing network connections at all. Or, even if we do, the outgoing network connections are only to the external Internet, and not to anything within our cluster (except the internal ssh gateway "ssh" and http server "cocalc", which have clear security constraints). This is just a basic firewall security requirement for "defense in depth". - diff --git a/src/packages/server/projects/connection/call.ts b/src/packages/server/projects/connection/call.ts deleted file mode 100644 index b9182ccc39..0000000000 --- a/src/packages/server/projects/connection/call.ts +++ /dev/null @@ -1,37 +0,0 @@ -/* -Use the project hub socket API to communicate with the project. - -The supported messages are implemented here in the project: - - cocalc/src/packages/project/servers/hub/handle-message.ts - -and messages must be defined as in - - cocalc/src/packages/util/message.js - -and they include: - -- ping: for testing; returns a pong -- heartbeat: used for maintaining the connection -- project_exec: run shell command -- read_file_from_project: reads file and stores it as a blob in the database. blob expires in 24 hours. -- write_file_to_project: write abitrary file to disk in project (goes via a blob) -- write_text_file_to_project: write a text file, whose contents is in the message, to the project. -- print_to_pdf: tells sage worksheet to print -- send_signal: send a signal to a process -- seems maybe broken?; use project_exec instead! -- jupyter_execute: { input, history, kernel } ==> {output:object[]} or {error?:string} -*/ - -import { callProjectMessage } from "./handle-message"; -import getConnection from "./connect"; - -export default async function call({ - project_id, - mesg, -}: { - project_id: string; - mesg; -}): Promise { - const socket = await getConnection(project_id); - return await callProjectMessage({ mesg, socket }); -} diff --git a/src/packages/server/projects/connection/connect.ts b/src/packages/server/projects/connection/connect.ts deleted file mode 100644 index 3d99eb82a9..0000000000 --- a/src/packages/server/projects/connection/connect.ts +++ /dev/null @@ -1,132 +0,0 @@ -/* -Create or return the TCP connection from this server to a given project. - -The connection is cached and calling this is async debounced, so call it -all you want. - -This will also try to start the project up to about a minute. -*/ - -import getLogger from "@cocalc/backend/logger"; -import { getProject } from "@cocalc/server/projects/control"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { delay } from "awaiting"; -import { cancelAll } from "./handle-query"; -import initialize from "./initialize"; -import { callProjectMessage } from "./handle-message"; -import { CoCalcSocket } from "@cocalc/backend/tcp/enable-messaging-protocol"; -import { connectToLockedSocket } from "@cocalc/backend/tcp/locked-socket"; - -const logger = getLogger("project-connection:connect"); -type Connection = any; - -const CACHE: { [project_id: string]: Connection } = {}; - -// Whenever grabbing a socket, if we haven't tested that it is working for this long, -// we'll test it and remove it from the cache if it doesn't work. -const TEST_INTERVAL_MS = 30000; -const TEST: { [project_id: string]: number } = {}; - -async function testConnection(project_id: string) { - const socket = CACHE[project_id]; - if (socket == null) { - return; - } - const lastTest = TEST[project_id] ?? 0; - const now = Date.now(); - if (now - lastTest <= TEST_INTERVAL_MS) { - return; - } - logger.debug("testing connecting to ", project_id); - try { - const resp = await callProjectMessage({ - socket, - mesg: { event: "ping" }, - timeoutSeconds: 15, - }); - if (resp?.event != "pong") { - throw Error("sent ping but got back", resp); - } - logger.debug( - "testing connection to ", - project_id, - "working -- ping time ", - Date.now() - now, - " ms", - ); - } catch (err) { - logger.debug("testing connection to ", project_id, "failed -- ", err); - delete CACHE[project_id]; - } -} - -const END_EVENTS = ["end", "close", "error"]; - -async function connect(project_id: string): Promise { - logger.info("connect to ", project_id); - const dbg = (...args) => logger.debug(project_id, ...args); - if (CACHE[project_id]) { - const socket = CACHE[project_id]; - // This is not 100% reliable, so we also periodically ping - // project and remove socket from cache if it fails to pong back. - // This DOES happen sometime, e.g., when frequently restarting a project. - if (socket.destroyed || socket.readyState != "open") { - delete CACHE[project_id]; - socket.unref(); - } else { - dbg("got connection to ", project_id, " from cache"); - testConnection(project_id); - return CACHE[project_id]; - } - } - - const project = getProject(project_id); - - // Calling address starts the project running, then returns - // information about where it is running and how to connect. - // We retry a few times, in case project isn't running yet. - dbg("getting address of ", project_id); - let socket: CoCalcSocket; - let i = 0; - while (true) { - try { - const { host, port, secret_token: token } = await project.address(); - dbg("got ", host, port); - socket = await connectToLockedSocket({ host, port, token }); - break; - } catch (err) { - dbg(err); - if (i >= 10) { - // give up! - throw err; - } - await project.start(); - await delay(1000 * i); - i += 1; - } - } - initialize(project_id, socket); - - const free = () => { - delete CACHE[project_id]; - logger.info("disconnect from ", project_id); - // don't want free to be triggered more than once. - for (const evt of END_EVENTS) { - socket.removeListener(evt, free); - } - try { - socket.end(); - } catch (_) {} - cancelAll(project_id); - }; - for (const evt of END_EVENTS) { - socket.on(evt, free); - } - - CACHE[project_id] = socket; - return socket; -} - -const getConnection: (project_id: string) => Promise = - reuseInFlight(connect); -export default getConnection; diff --git a/src/packages/server/projects/connection/handle-blob.ts b/src/packages/server/projects/connection/handle-blob.ts deleted file mode 100644 index 27fa7a44d6..0000000000 --- a/src/packages/server/projects/connection/handle-blob.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { isValidUUID } from "@cocalc/util/misc"; -import { uuidsha1 } from "@cocalc/backend/sha1"; -import { db } from "@cocalc/database"; -import { callback2 } from "@cocalc/util/async-utils"; -import { save_blob } from "@cocalc/util/message"; -import getLogger from "@cocalc/backend/logger"; - -const logger = getLogger("project-connection:handle-blob"); - -// Blobs (e.g., files and images dynamically appearing as output in worksheets) are kept for this -// many seconds before being discarded. If the worksheet is saved (e.g., by a user's autosave), -// then the BLOB is saved indefinitely. -const TTL = 60 * 60 * 24; // 1 day -const MAX_BLOB_SIZE = 15000000; -const MAX_BLOB_SIZE_HUMAN = "15MB"; - -interface Options { - socket; - project_id: string; - uuid: string; - blob: Buffer; - ttlSeconds?: number; -} - -export default async function handleBlob({ - socket, - project_id, - uuid, - blob, - ttlSeconds, -}: Options): Promise { - let resp; - try { - await saveBlob({ project_id, uuid, blob }); - resp = save_blob({ sha1: uuid, ttl: ttlSeconds ?? TTL }); - } catch (err) { - resp = save_blob({ sha1: uuid, error: `${err}` }); - } - socket.write_mesg("json", resp); -} - -async function saveBlob({ - project_id, - uuid, - blob, -}: Pick): Promise { - logger.debug("saving blob in ", project_id, " with uuid ", uuid); - // return ttl in seconds. - if (!isValidUUID(project_id)) throw Error("project_id is invalid"); - if (!isValidUUID(uuid)) throw Error("uuid is invalid"); - if (!blob) throw Error("blob is required"); - if (uuid != uuidsha1(blob)) { - throw Error( - `uuid must be the sha1-uuid of blob but got ${uuid} != ${uuidsha1(blob)}`, - ); - } - if (blob.length > MAX_BLOB_SIZE) { - throw Error( - `saveBlob: blobs are limited to ${MAX_BLOB_SIZE_HUMAN} and you just tried to save one of size ${ - blob.length / 1000000 - }MB`, - ); - } - const database = db(); - return await callback2(database.save_blob, { - uuid, - blob, - ttl: TTL, - project_id, - }); -} diff --git a/src/packages/server/projects/connection/handle-message.ts b/src/packages/server/projects/connection/handle-message.ts deleted file mode 100644 index 91136b1987..0000000000 --- a/src/packages/server/projects/connection/handle-message.ts +++ /dev/null @@ -1,105 +0,0 @@ -/* -Handle incoming JSON messages from a project. -*/ - -import { promisify } from "node:util"; -import { v4 } from "uuid"; - -import getLogger from "@cocalc/backend/logger"; -import { TIMEOUT_CALLING_PROJECT } from "@cocalc/util/consts/project"; -import { error, pong } from "@cocalc/util/message"; -import handleQuery from "./handle-query"; -import handleVersion from "./handle-version"; - -const logger = getLogger("project-connection:handle-message"); - -interface Options { - socket; - project_id: string; - mesg; -} - -const callCallbacks: { [id: string]: Function } = {}; - -export default async function handleMessage({ - socket, - project_id, - mesg, -}: Options): Promise { - logger.debug("received message ", project_id); - - if (mesg.event == "version") { - handleVersion(project_id, mesg.version); - return; - } - // globally unique random uuid - const { id } = mesg; - if (id == null) { - // all messages except "version" must have an id - logger.warn("WARNING: all messages except 'version' must have an id", mesg); - return; - } - - const f = callCallbacks[id]; - if (f != null) { - f(mesg); - return; - } - - logger.debug("handling call from project"); - function sendResponse(resp) { - resp.id = id; - socket.write_mesg("json", resp); - } - - try { - switch (mesg.event) { - case "ping": - sendResponse(pong()); - return; - case "query": - case "query_cancel": - await handleQuery({ project_id, mesg, sendResponse }); - return; - case "file_written_to_project": - case "file_read_from_project": - case "error": - // ignore/deprecated/don't care...? - return; - default: - throw Error(`unknown event '${mesg.event}'`); - } - } catch (err) { - sendResponse(error({ error: `${err}` })); - } -} - -export async function callProjectMessage({ - socket, - mesg, - timeoutSeconds = 60, // DEV: change this to 3 to simulate quick timeouts -}): Promise { - logger.debug("callProjectMessage", mesg.event, mesg.id); - while (mesg.id == null || callCallbacks[mesg.id] != null) { - mesg.id = v4(); - } - - const getResponse = promisify((cb: (err: any, resp?: any) => void) => { - callCallbacks[mesg.id] = (resp) => { - logger.debug("callProjectMessage -- got response", resp.id); - cb(undefined, resp); - }; - setTimeout(() => { - cb(TIMEOUT_CALLING_PROJECT); - callCallbacks[mesg.id] = () => { - logger.debug( - mesg.id, - `callProjectMessage -- ignoring response due to timeout ${timeoutSeconds}s`, - ); - }; - }, timeoutSeconds * 1000); - }); - - socket.write_mesg("json", mesg); - return await getResponse(); -} diff --git a/src/packages/server/projects/connection/handle-query.ts b/src/packages/server/projects/connection/handle-query.ts deleted file mode 100644 index 9b7a49d5ee..0000000000 --- a/src/packages/server/projects/connection/handle-query.ts +++ /dev/null @@ -1,117 +0,0 @@ -/* -Handle a project query (or query cancel) message from a project. -*/ - -import { db } from "@cocalc/database"; -import { callback2 } from "@cocalc/util/async-utils"; -import { error } from "@cocalc/util/message"; -import getLogger from "@cocalc/backend/logger"; -const logger = getLogger("project-connection:handle-query"); - -interface Options { - project_id: string; - mesg; - sendResponse: (any) => void; -} - -export default function handleQuery(opts: Options) { - switch (opts.mesg.event) { - case "query": - query(opts); - return; - case "query_cancel": - cancel(opts); - return; - default: - throw Error(`unknown event ${opts.mesg.event}`); - } -} - -const changefeeds: { [project_id: string]: Set } = {}; - -function query({ project_id, mesg, sendResponse }: Options) { - logger.debug("query", project_id); - const { id, changes, options, query } = mesg; - if (!query) { - throw Error("query must be defined"); - } - let first = true; // relevant if changes is true - if (changes) { - if (changefeeds[project_id] === undefined) { - changefeeds[project_id] = new Set([id]); - } else { - changefeeds[project_id].add(id); - } - } - const database = db(); - database.user_query({ - // use callback rather than async/await here, due to changefeed - project_id, - query, - options, - changes: changes ? id : undefined, - cb: (err, result) => { - if (result?.action == "close") { - err = "close"; - } - if (err) { - if (err != "close") { - logger.debug("query: err=", err); - } - if (changefeeds[project_id]?.has(id)) { - changefeeds[project_id]?.delete(id); - } - sendResponse(error({ error: `${err}` })); - if (changes && !first) { - database.user_query_cancel_changefeed({ id }); - } - } else { - let resp; - if (changes && !first) { - resp = result; - resp.id = id; - resp.multi_response = true; - } else { - first = false; - resp = { ...mesg }; - resp.query = result; - } - sendResponse(resp); - } - }, - }); -} - -async function cancel({ - project_id, - mesg, - sendResponse, -}: Options): Promise { - const c = changefeeds[project_id]; - if (!c?.has(mesg.id)) { - // no such changefeed -- nothing to do - sendResponse(mesg); - return; - } - const database = db(); - const resp = await callback2(database.user_query_cancel_changefeed, { - id: mesg.id, - }); - mesg.resp = resp; - sendResponse(mesg); - c.delete(mesg.id); -} - -export async function cancelAll(project_id: string): Promise { - const database = db(); - const c = changefeeds[project_id]; - if (!c) return; - for (const id of c) { - try { - await callback2(database.user_query_cancel_changefeed, { id }); - c.delete(id); - } catch (err) { - logger.debug("WARNING: error canceling changefeed", id, err); - } - } -} diff --git a/src/packages/server/projects/connection/handle-version.ts b/src/packages/server/projects/connection/handle-version.ts deleted file mode 100644 index f012eee67b..0000000000 --- a/src/packages/server/projects/connection/handle-version.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { getServerSettings } from "@cocalc/database/settings"; -import { getProject } from "@cocalc/server/projects/control"; -import LRU from "lru-cache"; - -import getLogger from "@cocalc/backend/logger"; -const logger = getLogger("project-connection:handle-version"); - -const restarted = new LRU({ - ttl: 15 * 1000 * 60, // never try to restart more than once every 15 minutes - max: 10000, -}); - -export default async function handleVersion( - project_id: string, - version: number -): Promise { - if (restarted.has(project_id)) return; - - // Restart project if version of project code is too old. - const { version_min_project } = await getServerSettings(); - if (!version_min_project || version_min_project <= version) return; - - restarted.set(project_id, true); - const project = getProject(project_id); - try { - await project.restart(); - } catch (err) { - logger.debug( - "WARNING -- error restarting project due to version too old", - project_id, - err - ); - } -} diff --git a/src/packages/server/projects/connection/heartbeat.ts b/src/packages/server/projects/connection/heartbeat.ts deleted file mode 100644 index 6438ac4201..0000000000 --- a/src/packages/server/projects/connection/heartbeat.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { heartbeat } from "@cocalc/util/message"; -import { PROJECT_HUB_HEARTBEAT_INTERVAL_S } from "@cocalc/util/heartbeat"; - -export default function initHeartbeat(socket) { - let alive: boolean = true; - const stop = () => (alive = false); - socket.on("end", stop); - socket.on("close", stop); - socket.on("error", stop); - const sendHeartbeat = () => { - if (!alive) return; - socket.write_mesg("json", heartbeat()); - setTimeout(sendHeartbeat, PROJECT_HUB_HEARTBEAT_INTERVAL_S * 1000); - }; - // start the heart beating! - sendHeartbeat(); -} diff --git a/src/packages/server/projects/connection/index.ts b/src/packages/server/projects/connection/index.ts deleted file mode 100644 index 0c5ba9f955..0000000000 --- a/src/packages/server/projects/connection/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -import getConnection from "./connect"; -export default getConnection; diff --git a/src/packages/server/projects/connection/initialize.ts b/src/packages/server/projects/connection/initialize.ts deleted file mode 100644 index bd906c1aeb..0000000000 --- a/src/packages/server/projects/connection/initialize.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* -Initialize a TCP socket connection to a project. - -This mainly involves setting up the socket to so we we can send and receive -message over it. -*/ - -import getLogger from "@cocalc/backend/logger"; -import handleBlob from "./handle-blob"; -import handleMessage from "./handle-message"; -import initHeartbeat from "./heartbeat"; - -import enableMessagingProtocol, { - CoCalcSocket, -} from "@cocalc/backend/tcp/enable-messaging-protocol"; - -const logger = getLogger("project-connection:initialize"); - -export default function initialize( - project_id: string, - socket: CoCalcSocket -): void { - logger.info("initializing socket"); - enableMessagingProtocol(socket, "connection_to_a_local_hub"); - - socket.on("mesg", (type, mesg) => { - switch (type) { - case "blob": - handleBlob({ - socket, - project_id, - uuid: mesg.uuid, - blob: mesg.blob, - ttlSeconds: mesg.ttlSeconds, - }); - return; - case "json": - handleMessage({ socket, project_id, mesg }); - return; - default: - logger.warn("WARNING: unknown message type", type); - } - }); - - // Send a hello message to the project. I'm not sure if this is used for anything at all, - // but it is nice to see in the logs. - socket.write_mesg("json", { event: "hello" }); - - // start sending heartbeats over this socket, so project knows it is working. - initHeartbeat(socket); -} diff --git a/src/packages/server/projects/control/index.ts b/src/packages/server/projects/control/index.ts index 88ec83fcb7..6635b5471a 100644 --- a/src/packages/server/projects/control/index.ts +++ b/src/packages/server/projects/control/index.ts @@ -10,17 +10,5 @@ export default function init(): ProjectControlFunction { logger.debug("init"); const database = db(); database.projectControl = getProject; - - // This is used by the database when handling certain writes to make sure - // that the there is a connection to the corresponding project, so that - // the project can respond. - database.ensure_connection_to_project = async ( - _project_id: string, - cb?: Function, - ): Promise => { - console.log("database.ensure_connection_to_project -- DEPRECATED"); - cb?.(); - }; - return getProject; } diff --git a/src/packages/util/message.d.ts b/src/packages/util/message.d.ts index c3baf892e8..b94647807f 100644 --- a/src/packages/util/message.d.ts +++ b/src/packages/util/message.d.ts @@ -24,11 +24,8 @@ export const project_exec: any; export const project_exec_output: any; export const read_file_from_project: any; export const file_read_from_project: any; -export const read_text_file_from_project: any; export const text_file_read_from_project: any; export const write_file_to_project: any; -export const write_text_file_to_project: any; -export const file_written_to_project: any; export const project_users: any; export const version: any; export const save_blob: any; diff --git a/src/packages/util/message.js b/src/packages/util/message.js index eb9e861b5e..00b94fca7f 100644 --- a/src/packages/util/message.js +++ b/src/packages/util/message.js @@ -659,166 +659,6 @@ message({ stats: undefined, }); - -//############################################################################ - -// The read_file_from_project message is sent by the hub to request -// that the project_server read a file from a project and send it back -// to the hub as a blob. Also sent by client to hub to request a file -// or directory. If path is a directory, the optional archive field -// specifies how to create a single file archive, with supported -// options including: 'tar', 'tar.bz2', 'tar.gz', 'zip', '7z'. -// -// client --> hub --> project_server -message({ - event: "read_file_from_project", - id: undefined, - project_id: required, - path: required, - archive: "tar.bz2", - ttlSeconds: undefined, // if given, time to live in seconds for blob; default is "1 day". -}); - -// The file_read_from_project message is sent by the project_server -// when it finishes reading the file from disk. -// project_server --> hub -message({ - event: "file_read_from_project", - id: required, - data_uuid: required, // The project_server will send the raw data of the file as a blob with this uuid. - archive: undefined, // if defined, means that file (or directory) was archived (tarred up) and this string was added to end of filename. -}); - -// The client sends this message to the hub in order to read -// a plain text file (binary files not allowed, since sending -// them via JSON makes no sense). -// client --> hub -API( - message2({ - event: "read_text_file_from_project", - fields: { - id: { - init: undefined, - desc: "A unique UUID for the query", - }, - project_id: { - init: required, - desc: "id of project containing file to be read (or array of project_id's)", - }, - path: { - init: required, - desc: "path to file to be read in target project (or array of paths)", - }, - }, - desc: `\ -Read a text file in the project whose \`project_id\` is supplied. - -Argument \`'path'\` is relative to home directory in target project. - -You can also read multiple \`project_id\`/\`path\`'s at once by -making \`project_id\` and \`path\` arrays (of the same length). -In that case, the result will be an array -of \`{project_id, path, content}\` objects, in some random order. -If there is an error reading a particular file, -instead \`{project_id, path, error}\` is included. - -**Note:** You need to have read access to the project, -the Linux user \`user\` in the target project must have permissions to read the file -and containing directories. - -Example: - -Read a text file. -\`\`\` - curl -u sk_abcdefQWERTY090900000000: \\ - -d project_id=e49e86aa-192f-410b-8269-4b89fd934fba \\ - -d path=Assignments/A1/h1.txt \\ - https://cocalc.com/api/v1/read_text_file_from_project - ==> {"event":"text_file_read_from_project", - "id":"481d6055-5609-450f-a229-480e518b2f84", - "content":"hello"} -\`\`\`\ -`, - }), -); - -// hub --> client -message({ - event: "text_file_read_from_project", - id: required, - content: required, -}); - -// The write_file_to_project message is sent from the hub to the -// project_server to tell the project_server to write a file to a -// project. If the path includes directories that don't exists, -// they are automatically created (this is in fact the only way -// to make a new directory except of course project_exec). -// hub --> project_server -message({ - event: "write_file_to_project", - id: required, - project_id: required, - path: required, - data_uuid: required, -}); // hub sends raw data as a blob with this uuid immediately. - -// The client sends this message to the hub in order to write (or -// create) a plain text file (binary files not allowed, since sending -// them via JSON makes no sense). -// client --> hub -API( - message2({ - event: "write_text_file_to_project", - fields: { - id: { - init: undefined, - desc: "A unique UUID for the query", - }, - project_id: { - init: required, - desc: "id of project where file is created", - }, - path: { - init: required, - desc: "path to file, relative to home directory in destination project", - }, - content: { - init: required, - desc: "contents of the text file to be written", - }, - }, - desc: `\ -Create a text file in the target project with the given \`project_id\`. -Directories containing the file are created if they do not exist already. -If a file already exists at the destination path, it is overwritten. - -**Note:** You need to have read access to the project. -The Linux user \`user\` in the target project must have permissions to create files -and containing directories if they do not already exist. - -Example: - -Create a text file. -\`\`\` - curl -u sk_abcdefQWERTY090900000000: \\ - -d project_id=e49e86aa-192f-410b-8269-4b89fd934fba \\ - -d "content=hello$'\\n'world" \\ - -d path=Assignments/A1/h1.txt \\ - https://cocalc.com/api/v1/write_text_file_to_project -\`\`\`\ -`, - }), -); - -// The file_written_to_project message is sent by a project_server to -// confirm successful write of the file to the project. -// project_server --> hub -message({ - event: "file_written_to_project", - id: required, -}); - //########################################### // Managing multiple projects //########################################### From 93c45978f61e7b84665ebfcbad616987c2c76e26 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 15 Aug 2025 18:26:25 +0000 Subject: [PATCH 305/798] invalid import --- src/packages/util/db-schema/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/packages/util/db-schema/index.ts b/src/packages/util/db-schema/index.ts index 8fd69895ec..60c42d505a 100644 --- a/src/packages/util/db-schema/index.ts +++ b/src/packages/util/db-schema/index.ts @@ -36,7 +36,6 @@ import "./file-use"; import "./groups"; import "./hub-servers"; import "./instances"; // probably deprecated -import "./jupyter"; import "./listings"; import "./llm"; import "./lti"; From 2a3a42da965dce35b95172289970a2a8075eb613 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 15 Aug 2025 19:31:03 +0000 Subject: [PATCH 306/798] fix depcheck issue -- api-client no longer depends on @cocalc/util --- src/packages/api-client/package.json | 15 +++++++++++---- src/packages/pnpm-lock.yaml | 3 --- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/packages/api-client/package.json b/src/packages/api-client/package.json index b5d16040ec..d2592e4da8 100644 --- a/src/packages/api-client/package.json +++ b/src/packages/api-client/package.json @@ -9,14 +9,21 @@ "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput", "depcheck": "pnpx depcheck --ignores @cocalc/api-client " }, - "files": ["dist/**", "bin/**", "README.md", "package.json"], + "files": [ + "dist/**", + "bin/**", + "README.md", + "package.json" + ], "author": "SageMath, Inc.", - "keywords": ["cocalc", "jupyter"], + "keywords": [ + "cocalc", + "jupyter" + ], "license": "SEE LICENSE.md", "dependencies": { "@cocalc/api-client": "workspace:*", - "@cocalc/backend": "workspace:*", - "@cocalc/util": "workspace:*" + "@cocalc/backend": "workspace:*" }, "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/compute", "repository": { diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 5ec4f4995c..275a6e786f 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -55,9 +55,6 @@ importers: '@cocalc/backend': specifier: workspace:* version: link:../backend - '@cocalc/util': - specifier: workspace:* - version: link:../util devDependencies: '@types/node': specifier: ^18.16.14 From 6c7fd686d20c3b83175922128805a55051d20357 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 15 Aug 2025 19:33:09 +0000 Subject: [PATCH 307/798] no longer using uuid module (and actually our little code in util/misc is faster) --- src/packages/pnpm-lock.yaml | 6 ------ src/packages/project/package.json | 4 +--- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 275a6e786f..3c81481253 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -1134,9 +1134,6 @@ importers: tmp: specifier: 0.2.4 version: 0.2.4 - uuid: - specifier: ^8.3.2 - version: 8.3.2 websocket-sftp: specifier: ^0.8.4 version: 0.8.4(bufferutil@4.0.9)(utf-8-validate@6.0.5) @@ -1162,9 +1159,6 @@ importers: '@types/primus': specifier: ^7.3.9 version: 7.3.9 - '@types/uuid': - specifier: ^8.3.1 - version: 8.3.4 server: dependencies: diff --git a/src/packages/project/package.json b/src/packages/project/package.json index a687db3224..a820e4a1cd 100644 --- a/src/packages/project/package.json +++ b/src/packages/project/package.json @@ -49,7 +49,6 @@ "rimraf": "^5.0.5", "temp": "^0.9.4", "tmp": "0.2.4", - "uuid": "^8.3.2", "websocket-sftp": "^0.8.4", "which": "^2.0.2", "ws": "^8.18.0" @@ -59,8 +58,7 @@ "@types/express": "^4.17.21", "@types/lodash": "^4.14.202", "@types/node": "^18.16.14", - "@types/primus": "^7.3.9", - "@types/uuid": "^8.3.1" + "@types/primus": "^7.3.9" }, "scripts": { "preinstall": "npx only-allow pnpm", From 0cc27c9f2b69d1fd778e77c4fefa30543edef9ef Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 15 Aug 2025 19:45:35 +0000 Subject: [PATCH 308/798] support global max-workers testing option --- src/packages/backend/sandbox/find.test.ts | 2 +- src/packages/database/package.json | 2 +- src/packages/sync-fs/package.json | 2 +- src/workspaces.py | 15 ++++++++++++--- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/packages/backend/sandbox/find.test.ts b/src/packages/backend/sandbox/find.test.ts index fa265b5b49..d504fb6d07 100644 --- a/src/packages/backend/sandbox/find.test.ts +++ b/src/packages/backend/sandbox/find.test.ts @@ -69,7 +69,7 @@ describe("find files", () => { const t = Date.now(); const { stdout, truncated } = await find(tempDir, { options: ["-printf", "%f\n"], - timeout: 0.1, + timeout: 0.002, }); expect(truncated).toBe(true); diff --git a/src/packages/database/package.json b/src/packages/database/package.json index 15b29d94fe..d14229573e 100644 --- a/src/packages/database/package.json +++ b/src/packages/database/package.json @@ -45,7 +45,7 @@ "build": "../node_modules/.bin/tsc --build && coffee -c -o dist/ ./", "clean": "rm -rf dist", "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput", - "test": "pnpm exec jest --forceExit --runInBand", + "test": "pnpm exec jest --forceExit --maxWorkers=1", "depcheck": "pnpx depcheck | grep -Ev '\\.coffee|coffee$'", "prepublishOnly": "pnpm test" }, diff --git a/src/packages/sync-fs/package.json b/src/packages/sync-fs/package.json index 08b34f4b7c..5b05b812e4 100644 --- a/src/packages/sync-fs/package.json +++ b/src/packages/sync-fs/package.json @@ -19,7 +19,7 @@ "build": "../node_modules/.bin/tsc --build", "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput", "clean": "rm -rf node_modules dist", - "test": "pnpm exec jest --forceExit --runInBand", + "test": "pnpm exec jest --forceExit --maxWorkers=1", "depcheck": "pnpx depcheck" }, "author": "SageMath, Inc.", diff --git a/src/workspaces.py b/src/workspaces.py index 53011ac7f6..f8944f09d3 100755 --- a/src/workspaces.py +++ b/src/workspaces.py @@ -285,7 +285,7 @@ def test(args) -> None: success = [] def status(): - print("Status: ", {"flaky": flaky, "fails": fails, "success": success}) + print("Status: ", {"fails": fails, "flaky": flaky, "success": success}) v = packages(args) v.sort() @@ -295,6 +295,7 @@ def status(): package_path = os.path.join(CUR, path) if package_path.endswith('packages/'): continue + package_json = open(os.path.join(package_path, 'package.json')).read() def f(): print("\n" * 3) @@ -304,13 +305,14 @@ def f(): print(f"TESTING {n}/{len(v)}: {path}") print("*") print("*" * 40) - if args.test_github_ci and 'test-github-ci' in open( - os.path.join(package_path, 'package.json')).read(): + if args.test_github_ci and 'test-github-ci' in package_json: test_cmd = "pnpm run test-github-ci" else: test_cmd = "pnpm run --if-present test" if args.report: test_cmd += " --reporters=default --reporters=jest-junit" + if args.max_workers: + test_cmd += f' --maxWorkers={args.max_workers} ' cmd(test_cmd, package_path) success.append(path) @@ -595,6 +597,13 @@ def packages_arg(parser): action="store_const", const=True, help='if given, generate test reports') + subparser.add_argument( + '--max-workers', + type=str, + default='', + help= + 'optional maxWorkers argument to be passed to all all calls to pnpm test. This can be helpful to prevent overly optimistic hyperthreading.' + ) packages_arg(subparser) subparser.set_defaults(func=test) From ccdafb4398acd8087de421cc07987e57016a595f Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 15 Aug 2025 20:30:51 +0000 Subject: [PATCH 309/798] remove a bunch of not-needed chown's when starting project, since they are beaking CI on github --- src/packages/server/conat/project/run.ts | 6 +-- src/packages/server/projects/control/util.ts | 57 +------------------- 2 files changed, 5 insertions(+), 58 deletions(-) diff --git a/src/packages/server/conat/project/run.ts b/src/packages/server/conat/project/run.ts index d12e2e3ded..d5e637a7ec 100644 --- a/src/packages/server/conat/project/run.ts +++ b/src/packages/server/conat/project/run.ts @@ -141,15 +141,15 @@ async function start({ const home = await homePath(project_id); await mkdir(home, { recursive: true }); - await ensureConfFilesExists(home, uid); + await ensureConfFilesExists(home); const env = await getEnvironment( project_id, // for admin HOME stays with original source, to avoid complexity of bind mounting /home/user // via the unshare system call... config?.admin ? { HOME: home } : undefined, ); - await setupDataPath(home, uid); - await writeSecretToken(home, await getProjectSecretToken(project_id), uid); + await setupDataPath(home); + await writeSecretToken(home, await getProjectSecretToken(project_id)); if (config?.disk) { // TODO: maybe this should be done in parallel with other things diff --git a/src/packages/server/projects/control/util.ts b/src/packages/server/projects/control/util.ts index 67cefd4ab1..a384ad99a4 100644 --- a/src/packages/server/projects/control/util.ts +++ b/src/packages/server/projects/control/util.ts @@ -1,13 +1,9 @@ -import { promisify } from "util"; import { join } from "path"; -import { exec as exec0 } from "child_process"; -import * as fs from "fs"; -import { writeFile } from "fs/promises"; +import { copyFile, mkdir, readFile, rm, stat, writeFile } from "fs/promises"; import { root } from "@cocalc/backend/data"; import { callback2 } from "@cocalc/util/async-utils"; import getLogger from "@cocalc/backend/logger"; import { ProjectState, ProjectStatus } from "./base"; -import { getUid } from "@cocalc/backend/misc"; import base_path from "@cocalc/backend/base-path"; import { db } from "@cocalc/database"; import { getProject } from "."; @@ -21,16 +17,6 @@ import { } from "@cocalc/server/conat/file-server"; const logger = getLogger("project-control:util"); -export const mkdir = promisify(fs.mkdir); -const readFile = promisify(fs.readFile); -const stat = promisify(fs.stat); -const copyFile = promisify(fs.copyFile); -const rm = promisify(fs.rm); - -export async function chown(path: string, uid: number): Promise { - await promisify(fs.chown)(path, uid, uid); -} - export function dataPath(HOME: string): string { return join(HOME, ".smc"); } @@ -104,14 +90,11 @@ export async function getProjectPID(HOME: string): Promise { return parseInt((await readFile(path)).toString()); } -export async function setupDataPath(HOME: string, uid?: number): Promise { +export async function setupDataPath(HOME: string): Promise { const data = dataPath(HOME); logger.debug(`setup "${data}"...`); await rm(data, { recursive: true, force: true }); await mkdir(data); - if (uid != null) { - await chown(data, uid); - } } // see also packages/project/secret-token.ts @@ -123,42 +106,10 @@ export function secretTokenPath(HOME: string) { export async function writeSecretToken( HOME: string, secretToken: string, - uid?: number, ): Promise { const path = secretTokenPath(HOME); await ensureContainingDirectoryExists(path); await writeFile(path, secretToken); - if (uid) { - await chown(path, uid); - } -} - -async function exec( - command: string, - verbose?: boolean, -): Promise<{ stdout: string; stderr: string }> { - logger.debug(`exec '${command}'`); - const output = await promisify(exec0)(command); - if (verbose) { - logger.debug(`output: ${JSON.stringify(output)}`); - } - return output; -} - -export async function stopProjectProcesses(project_id: string): Promise { - const uid = `${getUid(project_id)}`; - const scmd = `pkill -9 -u ${uid} | true `; // | true since pkill exit 1 if nothing killed. - await exec(scmd); -} - -export async function deleteUser(project_id: string): Promise { - await stopProjectProcesses(project_id); - const username = getUsername(project_id); - try { - await exec(`/usr/sbin/userdel ${username}`); // this also deletes the group - } catch (_) { - // not error if not there... - } } const ENV_VARS_DELETE = [ @@ -294,7 +245,6 @@ export async function getStatus(HOME: string): Promise { export async function ensureConfFilesExists( HOME: string, - uid?: number, ): Promise { for (const path of ["bashrc", "bash_profile"]) { const target = join(HOME, `.${path}`); @@ -310,9 +260,6 @@ export async function ensureConfFilesExists( ); try { await copyFile(source, target); - if (uid != null) { - await chown(target, uid); - } } catch (err) { logger.error(`ensureConfFilesExists -- ${err}`); } From 3c396fc78ed1e36bc42690ee332fb78b4fc10901 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 15 Aug 2025 20:58:54 +0000 Subject: [PATCH 310/798] rename file; create folder -- without starting project --- .../frontend/project/ask-filename.tsx | 2 +- .../frontend/project/explorer/explorer.tsx | 2 +- .../explorer/file-listing/no-files.tsx | 4 +- .../frontend/project/explorer/rename-file.tsx | 2 +- .../frontend/project/new/new-file-page.tsx | 2 +- .../frontend/project/page/flyouts/new.tsx | 2 +- src/packages/frontend/project_actions.ts | 102 +++++++----------- 7 files changed, 44 insertions(+), 72 deletions(-) diff --git a/src/packages/frontend/project/ask-filename.tsx b/src/packages/frontend/project/ask-filename.tsx index d60f88d35b..7fe834f9d3 100644 --- a/src/packages/frontend/project/ask-filename.tsx +++ b/src/packages/frontend/project/ask-filename.tsx @@ -52,7 +52,7 @@ export default function AskNewFilename({ project_id }: Props) { const create = (name, focus) => { actions.ask_filename(undefined); if (ext_selection == "/") { - actions.create_folder({ + actions.createFolder({ name: name, current_path: current_path, switch_over: focus, diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index de33eac026..7a99a37f1b 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -262,7 +262,7 @@ export function Explorer() { }; const create_folder = (switch_over = true): void => { - actions.create_folder({ + actions.createFolder({ name: file_search ?? "", current_path: current_path, switch_over, diff --git a/src/packages/frontend/project/explorer/file-listing/no-files.tsx b/src/packages/frontend/project/explorer/file-listing/no-files.tsx index a4c42fbf2b..e9441ea927 100644 --- a/src/packages/frontend/project/explorer/file-listing/no-files.tsx +++ b/src/packages/frontend/project/explorer/file-listing/no-files.tsx @@ -110,7 +110,7 @@ export default function NoFiles({ if (!file_search?.trim()) { actions.set_active_tab("new"); } else if (file_search[file_search.length - 1] === "/") { - actions.create_folder({ + actions.createFolder({ name: join(current_path ?? "", file_search), }); } else { @@ -153,7 +153,7 @@ export default function NoFiles({ }} create_folder={() => { const filename = default_filename(undefined, project_id); - actions.create_folder({ + actions.createFolder({ name: file_search.trim() ? file_search : join(current_path ?? "", filename), diff --git a/src/packages/frontend/project/explorer/rename-file.tsx b/src/packages/frontend/project/explorer/rename-file.tsx index c99ad1e808..4559a29b5d 100644 --- a/src/packages/frontend/project/explorer/rename-file.tsx +++ b/src/packages/frontend/project/explorer/rename-file.tsx @@ -78,7 +78,7 @@ export default function RenameFile({ duplicate, clear }: Props) { only_contents: true, }); } else { - await actions.rename_file(opts); + await actions.renameFile(opts); } } catch (err) { setLoading(false); diff --git a/src/packages/frontend/project/new/new-file-page.tsx b/src/packages/frontend/project/new/new-file-page.tsx index d8f9ace744..c9c631f3c8 100644 --- a/src/packages/frontend/project/new/new-file-page.tsx +++ b/src/packages/frontend/project/new/new-file-page.tsx @@ -178,7 +178,7 @@ export default function NewFilePage(props: Props) { } function createFolder() { - getActions().create_folder({ + getActions().createFolder({ name: filename, current_path, switch_over: true, diff --git a/src/packages/frontend/project/page/flyouts/new.tsx b/src/packages/frontend/project/page/flyouts/new.tsx index 40e3baee03..3b7a37c1dc 100644 --- a/src/packages/frontend/project/page/flyouts/new.tsx +++ b/src/packages/frontend/project/page/flyouts/new.tsx @@ -169,7 +169,7 @@ export function NewFlyout({ current_path, }); } else { - await actions?.create_folder({ + await actions?.createFolder({ name: newFilename.trim(), current_path, }); diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index f196b2752c..97db6a71c4 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -2317,7 +2317,7 @@ export class ProjectActions extends Actions { }); await webapp_client.project_client.copyPathBetweenProjects(opts); - + const withSlashes = await this.appendSlashToDirectoryPaths(files, 0); this.log({ event: "file_action", @@ -2331,32 +2331,27 @@ export class ProjectActions extends Actions { this.set_activity({ id, stop: "" }); }; - public async rename_file(opts: { + renameFile = async ({ + src, + dest, + compute_server_id, + }: { src: string; dest: string; compute_server_id?: number; - }): Promise { - const id = misc.uuid(); - const status = `Renaming ${opts.src} to ${opts.dest}`; + }): Promise => { let error: any = undefined; - const intl = await getIntl(); - const what = intl.formatMessage(dialogs.project_actions_rename_file, { - src: opts.src, - }); - if (!(await ensure_project_running(this.project_id, what))) { - return; - } - + const id = misc.uuid(); + const status = `Renaming ${src} to ${dest}`; this.set_activity({ id, status }); try { - const api = await this.api(); - const compute_server_id = this.getComputeServerId(opts.compute_server_id); - await api.rename_file(opts.src, opts.dest, compute_server_id); + const fs = this.fs(compute_server_id); + await fs.rename(src, dest); this.log({ event: "file_action", action: "renamed", - src: opts.src, - dest: opts.dest + ((await this.isDir(opts.dest)) ? "/" : ""), + src, + dest: dest + ((await this.isDir(dest)) ? "/" : ""), compute_server_id, }); } catch (err) { @@ -2364,7 +2359,7 @@ export class ProjectActions extends Actions { } finally { this.set_activity({ id, stop: "", error }); } - } + }; // note: there is no need to explicitly close or await what is returned by // fs(...) since it's just a lightweight wrapper object to format appropriate RPC calls. @@ -2441,7 +2436,7 @@ export class ProjectActions extends Actions { return stats.isDirectory(); }; - public async moveFiles({ + moveFiles = async ({ src, dest, compute_server_id, @@ -2449,7 +2444,7 @@ export class ProjectActions extends Actions { src: string[]; dest: string; compute_server_id?: number; - }): Promise { + }): Promise => { const id = misc.uuid(); const status = `Moving ${src.length} ${misc.plural( src.length, @@ -2476,7 +2471,7 @@ export class ProjectActions extends Actions { } finally { this.set_activity({ id, stop: "", error }); } - } + }; private checkForSandboxError(message): boolean { const projectsStore = this.redux.getStore("projects"); @@ -2619,14 +2614,14 @@ export class ProjectActions extends Actions { } }; - print_file(opts): void { + print_file = (opts): void => { opts.print = true; this.download_file(opts); - } + }; - show_upload(show): void { + show_upload = (show): void => { this.setState({ show_upload: show }); - } + }; // Compute the absolute path to the file with given name but with the // given extension added to the file (e.g., "md") if the file doesn't have @@ -2651,54 +2646,31 @@ export class ProjectActions extends Actions { return s; }; - async create_folder(opts: { + createFolder = async ({ + name, + current_path, + switch_over = true, + compute_server_id, + }: { name: string; current_path?: string; + // Whether or not to switch to the new folder (default: true) switch_over?: boolean; compute_server_id?: number; - }): Promise { - let p; - opts = defaults(opts, { - name: required, - current_path: undefined, - switch_over: true, // Whether or not to switch to the new folder - compute_server_id: undefined, - }); - if ( - !(await ensure_project_running( - this.project_id, - `create the folder '${opts.name}'`, - )) - ) { - return; - } - let { compute_server_id, name } = opts; - const { current_path, switch_over } = opts; - compute_server_id = this.getComputeServerId(compute_server_id); - this.setState({ file_creation_error: undefined }); - if (name[name.length - 1] === "/") { - name = name.slice(0, -1); - } - try { - p = this.construct_absolute_path(name, current_path); - } catch (e) { - this.setState({ file_creation_error: e.message }); - return; - } + }): Promise => { + const path = current_path ? join(current_path, name) : name; + const fs = this.fs(compute_server_id); try { - await this.ensure_directory_exists(p, compute_server_id); + await fs.mkdir(path, { recursive: true }); } catch (err) { - this.setState({ - file_creation_error: `Error creating directory '${p}' -- ${err}`, - }); - return; + this.setState({ file_creation_error: `${err}` }); } if (switch_over) { - this.open_directory(p); + this.open_directory(path); } // Log directory creation to the event log. / at end of path says it is a directory. - this.log({ event: "file_action", action: "created", files: [p + "/"] }); - } + this.log({ event: "file_action", action: "created", files: [path + "/"] }); + }; create_file = async (opts: { name: string; @@ -2730,7 +2702,7 @@ export class ProjectActions extends Actions { } if (name[name.length - 1] === "/") { if (opts.ext == null) { - this.create_folder({ + this.createFolder({ name, current_path: opts.current_path, compute_server_id, From 35d3c9b3758c2158b7485db0bbd8b6e7e314a675 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 15 Aug 2025 21:44:36 +0000 Subject: [PATCH 311/798] make it possible to create files and folders, open files and save to disk without starting the project --- src/packages/conat/project/api/editor.ts | 8 --- src/packages/frontend/client/welcome-file.ts | 10 +-- .../frame-editors/code-editor/actions.ts | 3 - .../frontend/project/ask-filename.tsx | 2 +- .../frontend/project/explorer/explorer.tsx | 2 +- .../explorer/file-listing/no-files.tsx | 4 +- .../frontend/project/new/new-file-page.tsx | 2 +- src/packages/frontend/project/open-file.ts | 37 ++-------- .../project/page/flyouts/files-header.tsx | 2 +- .../frontend/project/page/flyouts/new.tsx | 2 +- src/packages/frontend/project_actions.ts | 72 ++++++++----------- src/packages/frontend/project_store.ts | 16 +++-- src/packages/project/conat/api/editor.ts | 1 - src/packages/project/conat/api/jupyter.ts | 1 - .../smc_pyutil/templates/linux/default.rnw | 35 --------- .../smc_pyutil/templates/linux/default.rtex | 66 ----------------- .../smc_pyutil/templates/linux/default.sagews | 0 .../smc_pyutil/templates/linux/default.tex | 33 --------- 18 files changed, 58 insertions(+), 238 deletions(-) delete mode 100755 src/smc_pyutil/smc_pyutil/templates/linux/default.rnw delete mode 100644 src/smc_pyutil/smc_pyutil/templates/linux/default.rtex delete mode 100755 src/smc_pyutil/smc_pyutil/templates/linux/default.sagews delete mode 100755 src/smc_pyutil/smc_pyutil/templates/linux/default.tex diff --git a/src/packages/conat/project/api/editor.ts b/src/packages/conat/project/api/editor.ts index 66750e2c51..cf9984b9da 100644 --- a/src/packages/conat/project/api/editor.ts +++ b/src/packages/conat/project/api/editor.ts @@ -1,8 +1,6 @@ import type { Options as FormatterOptions } from "@cocalc/util/code-formatter"; export const editor = { - newFile: true, - formatString: true, printSageWS: true, @@ -24,12 +22,6 @@ export interface CreateTerminalOptions { } export interface Editor { - // Create a new file with the given name, possibly aware of templates. - // This was cc-new-file in the old smc_pyutils python library. This - // is in editor, since it's meant to be for creating a file aware of the - // context of our editors. - newFile: (path: string) => Promise; - // returns formatted version of str. formatString: (opts: { str: string; diff --git a/src/packages/frontend/client/welcome-file.ts b/src/packages/frontend/client/welcome-file.ts index 4cf04b7685..824894546a 100644 --- a/src/packages/frontend/client/welcome-file.ts +++ b/src/packages/frontend/client/welcome-file.ts @@ -99,7 +99,7 @@ export class WelcomeFile { async open() { if (this.path == null) return; - await this.create_file(); + await this.createFile(); await this.extra_setup(); } @@ -153,14 +153,14 @@ export class WelcomeFile { }); } - // Calling the "create file" action will properly initialize certain files, + // Calling the "createFile" action will properly initialize certain files, // in particular .tex - private async create_file(): Promise { + private async createFile(): Promise { if (this.path == null) - throw new Error("WelcomeFile::create_file – path is not defined"); + throw new Error("WelcomeFile::createFile – path is not defined"); const project_actions = redux.getProjectActions(this.project_id); const { name, ext } = separate_file_extension(this.path); - await project_actions.create_file({ + await project_actions.createFile({ name, ext, current_path: "", diff --git a/src/packages/frontend/frame-editors/code-editor/actions.ts b/src/packages/frontend/frame-editors/code-editor/actions.ts index e0adf71f0e..b6e6bd479f 100644 --- a/src/packages/frontend/frame-editors/code-editor/actions.ts +++ b/src/packages/frontend/frame-editors/code-editor/actions.ts @@ -1248,9 +1248,6 @@ export class Actions< // several other formatting actions. // Doing this automatically is fraught with error, since cursors aren't precise... if (explicit) { - if (!(await this.ensureProjectIsRunning(`save ${this.path} to disk`))) { - return; - } const account: any = this.redux.getStore("account"); if ( account && diff --git a/src/packages/frontend/project/ask-filename.tsx b/src/packages/frontend/project/ask-filename.tsx index 7fe834f9d3..023798281a 100644 --- a/src/packages/frontend/project/ask-filename.tsx +++ b/src/packages/frontend/project/ask-filename.tsx @@ -58,7 +58,7 @@ export default function AskNewFilename({ project_id }: Props) { switch_over: focus, }); } else { - actions.create_file({ + actions.createFile({ name: name, ext: ext_selection, current_path: current_path, diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index 7a99a37f1b..1a9c70c584 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -252,7 +252,7 @@ export function Explorer() { ext = default_ext(disabled_ext); } - actions.create_file({ + actions.createFile({ name: file_search ?? "", ext, current_path: current_path, diff --git a/src/packages/frontend/project/explorer/file-listing/no-files.tsx b/src/packages/frontend/project/explorer/file-listing/no-files.tsx index e9441ea927..ec07c981e9 100644 --- a/src/packages/frontend/project/explorer/file-listing/no-files.tsx +++ b/src/packages/frontend/project/explorer/file-listing/no-files.tsx @@ -114,7 +114,7 @@ export default function NoFiles({ name: join(current_path ?? "", file_search), }); } else { - actions.create_file({ + actions.createFile({ name: join(current_path ?? "", actualNewFilename), }); } @@ -147,7 +147,7 @@ export default function NoFiles({ const filename = file_search.trim() ? file_search + "." + ext : default_filename(ext, project_id); - actions.create_file({ + actions.createFile({ name: join(current_path ?? "", filename), }); }} diff --git a/src/packages/frontend/project/new/new-file-page.tsx b/src/packages/frontend/project/new/new-file-page.tsx index c9c631f3c8..4a17c9a590 100644 --- a/src/packages/frontend/project/new/new-file-page.tsx +++ b/src/packages/frontend/project/new/new-file-page.tsx @@ -121,7 +121,7 @@ export default function NewFilePage(props: Props) { : filename; try { setCreatingFile(name + (ext ? "." + ext : "")); - await getActions().create_file({ + await getActions().createFile({ name, ext, current_path, diff --git a/src/packages/frontend/project/open-file.ts b/src/packages/frontend/project/open-file.ts index 9b5ee78917..b4707ca36c 100644 --- a/src/packages/frontend/project/open-file.ts +++ b/src/packages/frontend/project/open-file.ts @@ -21,7 +21,6 @@ import { required, uuid, } from "@cocalc/util/misc"; -import { SITE_NAME } from "@cocalc/util/theme"; import { normalize } from "./utils"; import { syncdbPath as ipynbSyncdbPath } from "@cocalc/util/jupyter/names"; import { termPath } from "@cocalc/util/terminal/names"; @@ -154,15 +153,9 @@ export async function open_file( } try { - // Unfortunately (it adds a roundtrip to the server), we **have** to do this - // due to https://github.com/sagemathinc/cocalc/issues/4732 until we actually - // genuinely implement symlink support. Otherwise bad things happen. Much of - // cocalc was implemented basically assuming links don't exist; it's not easy - // to change that! - const realpath = await webapp_client.project_client.realpath({ - project_id: actions.project_id, - path: opts.path, - }); + const fs = actions.fs(opts.compute_server_id); + // cocalc assumes the path is not a symlink + const realpath = await fs.realpath(opts.path); if (!tabIsOpened()) { return; } @@ -198,29 +191,7 @@ export async function open_file( } const is_public = group === "public"; - if (!is_public) { - // Check if have capability to open this file. Important - // to only do this if not public, since again, if public we - // are not even using the project (it is all client side). - const can_open_file = await store.can_open_file_ext(ext, actions); - if (!tabIsOpened()) { - return; - } - if (!can_open_file) { - const site_name = - redux.getStore("customize").get("site_name") || SITE_NAME; - alert_message({ - type: "error", - message: `This ${site_name} project cannot open ${ext} files!`, - timeout: 20, - }); - // console.log( - // `abort project_actions::open_file due to lack of support for "${ext}" files` - // ); - return; - } - // Wait for the project to start opening (only do this if not public -- public users don't // know anything about the state of the project). try { @@ -266,6 +237,8 @@ export async function open_file( const file_info = open_files.getIn([opts.path, "component"], { is_public: false, }) as any; + + if (!alreadyOpened || file_info.is_public !== is_public) { const was_public = file_info.is_public; diff --git a/src/packages/frontend/project/page/flyouts/files-header.tsx b/src/packages/frontend/project/page/flyouts/files-header.tsx index 02f384b0e7..0ef2bde66d 100644 --- a/src/packages/frontend/project/page/flyouts/files-header.tsx +++ b/src/packages/frontend/project/page/flyouts/files-header.tsx @@ -152,7 +152,7 @@ export function FilesHeader(props: Readonly): React.JSX.Element { async function createFileOrFolder() { const fn = searchToFilename(file_search); - await actions?.create_file({ + await actions?.createFile({ name: fn, current_path, }); diff --git a/src/packages/frontend/project/page/flyouts/new.tsx b/src/packages/frontend/project/page/flyouts/new.tsx index 3b7a37c1dc..28681b8e65 100644 --- a/src/packages/frontend/project/page/flyouts/new.tsx +++ b/src/packages/frontend/project/page/flyouts/new.tsx @@ -163,7 +163,7 @@ export function NewFlyout({ try { setCreating(true); if (isFile(fn)) { - await actions?.create_file({ + await actions?.createFile({ name: newFilename.trim(), ext: ext.trim(), current_path, diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index 97db6a71c4..036542651e 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -38,8 +38,6 @@ import { exec, } from "@cocalc/frontend/frame-editors/generic/client"; import { set_url } from "@cocalc/frontend/history"; -import { dialogs } from "@cocalc/frontend/i18n"; -import { getIntl } from "@cocalc/frontend/i18n/get-intl"; import { download_file, open_new_tab, @@ -112,12 +110,13 @@ import { } from "@cocalc/frontend/project/listing/use-files"; import { search } from "@cocalc/frontend/project/search/run"; import { type CopyOptions } from "@cocalc/conat/files/fs"; +import { getFileTemplate } from "./project/templates"; const { defaults, required } = misc; const BAD_FILENAME_CHARACTERS = "\\"; const BAD_LATEX_FILENAME_CHARACTERS = '\'"()"~%$'; -const BANNED_FILE_TYPES = ["doc", "docx", "pdf", "sws"]; +const BANNED_FILE_TYPES = new Set(["doc", "docx", "pdf", "sws"]); const FROM_WEB_TIMEOUT_S = 45; @@ -591,7 +590,7 @@ export class ProjectActions extends Actions { // or a file_redux_name // Pushes to browser history // Updates the URL - public set_active_tab( + set_active_tab = ( key: string, opts: { update_file_listing?: boolean; @@ -602,7 +601,7 @@ export class ProjectActions extends Actions { update_file_listing: true, change_history: true, }, - ): void { + ): void => { const store = this.get_store(); if (store == undefined) return; // project closed const prev_active_project_tab = store.get("active_project_tab"); @@ -766,7 +765,7 @@ export class ProjectActions extends Actions { } } this.setState(change); - } + }; public toggleFlyout(name: FixedTab): void { const store = this.get_store(); @@ -2672,39 +2671,36 @@ export class ProjectActions extends Actions { this.log({ event: "file_action", action: "created", files: [path + "/"] }); }; - create_file = async (opts: { + createFile = async ({ + name, + ext, + current_path, + switch_over = true, + compute_server_id, + }: { name: string; ext?: string; current_path?: string; switch_over?: boolean; compute_server_id?: number; }) => { - let p; - opts = defaults(opts, { - name: undefined, - ext: undefined, - current_path: undefined, - switch_over: true, // Whether or not to switch to the new file - compute_server_id: undefined, - }); - const compute_server_id = this.getComputeServerId(opts.compute_server_id); this.setState({ file_creation_error: undefined }); // clear any create file display state - let { name } = opts; - if ((name === ".." || name === ".") && opts.ext == null) { + if ((name === ".." || name === ".") && ext == null) { this.setState({ file_creation_error: "Cannot create a file named . or ..", }); return; } if (misc.is_only_downloadable(name)) { - this.new_file_from_web(name, opts.current_path ?? ""); + this.new_file_from_web(name, current_path ?? ""); return; } + if (name[name.length - 1] === "/") { - if (opts.ext == null) { + if (ext == null) { this.createFolder({ name, - current_path: opts.current_path, + current_path, compute_server_id, }); return; @@ -2712,23 +2708,14 @@ export class ProjectActions extends Actions { name = name.slice(0, name.length - 1); } } - try { - p = this.construct_absolute_path(name, opts.current_path, opts.ext); - } catch (e) { - console.warn("Absolute path creation error"); - this.setState({ file_creation_error: e.message }); - return; - } - const intl = await getIntl(); - const what = intl.formatMessage(dialogs.project_actions_create_file_what, { - path: p, - }); - if (!(await ensure_project_running(this.project_id, what))) { - return; + let path = current_path ? join(current_path, name) : name; + if (ext) { + path += "." + ext; } - const ext = misc.filename_extension(p); - if (BANNED_FILE_TYPES.indexOf(ext) != -1) { + ext = misc.filename_extension(path); + + if (BANNED_FILE_TYPES.has(ext)) { this.setState({ file_creation_error: `Cannot create a file with the ${ext} extension`, }); @@ -2737,7 +2724,7 @@ export class ProjectActions extends Actions { if (ext === "tex") { const filename = misc.path_split(name).tail; for (const bad_char of BAD_LATEX_FILENAME_CHARACTERS) { - if (filename.indexOf(bad_char) !== -1) { + if (filename.includes(bad_char)) { this.setState({ file_creation_error: `Cannot use '${bad_char}' in a LaTeX filename '${filename}'`, }); @@ -2745,24 +2732,27 @@ export class ProjectActions extends Actions { } } } + const content = getFileTemplate(ext); + const fs = this.fs(compute_server_id); try { - await this.projectApi().editor.newFile(p); + await fs.writeFile(path, content); } catch (err) { this.setState({ file_creation_error: `${err}`, }); return; } - this.log({ event: "file_action", action: "created", files: [p] }); + this.log({ event: "file_action", action: "created", files: [path] }); if (ext) { redux.getActions("account")?.addTag(`create-${ext}`); } - if (opts.switch_over) { + if (switch_over) { this.open_file({ - path: p, + path, // so opens on current compute server, and because switch_over is only something // we do when user is explicitly opening the file explicit: true, + foreground: true, compute_server_id, }); } diff --git a/src/packages/frontend/project_store.ts b/src/packages/frontend/project_store.ts index f7b2a392ff..a24edc01fb 100644 --- a/src/packages/frontend/project_store.ts +++ b/src/packages/frontend/project_store.ts @@ -397,18 +397,22 @@ export class ProjectStore extends Store { }; // returns false, if this project isn't capable of opening a file with the given extension - async can_open_file_ext( + can_open_file_ext = async ( ext: string, actions: ProjectActions, - ): Promise { + ): Promise => { // to make sure we know about disabled file types const conf = await actions.init_configuration("main"); - // if we don't know anything, we're optimistic and skip this check - if (conf == null) return true; - if (!isMainConfiguration(conf)) return true; + // if we don't know anything; we're optimistic and skip this check + if (conf == null) { + return true; + } + if (!isMainConfiguration(conf)) { + return true; + } const disabled_ext = conf.disabled_ext; return !disabled_ext.includes(ext); - } + }; public has_file_been_viewed(path: string): boolean { // note that component is NOT an immutable.js object: diff --git a/src/packages/project/conat/api/editor.ts b/src/packages/project/conat/api/editor.ts index b03f0c4ea1..7ddc203753 100644 --- a/src/packages/project/conat/api/editor.ts +++ b/src/packages/project/conat/api/editor.ts @@ -1,5 +1,4 @@ export { formatString } from "../../formatters"; -export { newFile } from "@cocalc/backend/misc/new-file"; import { printSageWS as printSageWS0 } from "@cocalc/project/print_to_pdf"; export { sagewsStart, sagewsStop } from "@cocalc/project/sagews/control"; diff --git a/src/packages/project/conat/api/jupyter.ts b/src/packages/project/conat/api/jupyter.ts index 82fcefdd47..c1db3d175d 100644 --- a/src/packages/project/conat/api/jupyter.ts +++ b/src/packages/project/conat/api/jupyter.ts @@ -4,7 +4,6 @@ export { nbconvert } from "../../jupyter/convert"; export { formatString } from "../../formatters"; export { logo as kernelLogo } from "@cocalc/jupyter/kernel/logo"; export { get_kernel_data as kernels } from "@cocalc/jupyter/kernel/kernel-data"; -export { newFile } from "@cocalc/backend/misc/new-file"; import { getClient } from "@cocalc/project/client"; import { project_id } from "@cocalc/project/data"; import * as control from "@cocalc/jupyter/control"; diff --git a/src/smc_pyutil/smc_pyutil/templates/linux/default.rnw b/src/smc_pyutil/smc_pyutil/templates/linux/default.rnw deleted file mode 100755 index 257942ac30..0000000000 --- a/src/smc_pyutil/smc_pyutil/templates/linux/default.rnw +++ /dev/null @@ -1,35 +0,0 @@ -\documentclass{article} -\usepackage[utf8]{inputenc} -\usepackage[T1]{fontenc} -\usepackage{url} -\begin{document} - -% learn more about knitr: https://yihui.name/knitr/ - -<>= -library(knitr) -opts_chunk$set(cache=TRUE, autodep=TRUE) -options(formatR.arrow=TRUE, width=90) -@ - -\title{Knitr in CoCalc} - -\author{Author Name} - -\maketitle - -<>= -x <- c(2,1,7,4,4,5,4,6,4,5,4,3,4,5,1) -summary(x) -@ - -<>= -hist(x) -@ - -Sum of \Sexpr{paste(x, collapse="+")} is \Sexpr{sum(x)}. - - -\end{document} - - diff --git a/src/smc_pyutil/smc_pyutil/templates/linux/default.rtex b/src/smc_pyutil/smc_pyutil/templates/linux/default.rtex deleted file mode 100644 index 8356dd00cc..0000000000 --- a/src/smc_pyutil/smc_pyutil/templates/linux/default.rtex +++ /dev/null @@ -1,66 +0,0 @@ -\documentclass{article} -\usepackage[utf8]{inputenc} -\usepackage[T1]{fontenc} -\usepackage{url} -\usepackage{graphicx} - -% this is based on https://github.com/yihui/knitr-examples/blob/master/005-latex.Rtex - -%% for inline R code: if the inline code is not correctly parsed, you will see a message -\newcommand{\rinline}[1]{SOMETHING WRONG WITH knitr} - -%% begin.rcode setup, include=FALSE -% library(knitr) -% opts_chunk$set(fig.path='figure/latex-', cache.path='cache/latex-') -%% end.rcode - -\begin{document} - -\title{Rtex Knitr in CoCalc} - -\author{Author Name} - -\maketitle - -Boring stuff as usual: - -%% a chunk with default options -%% begin.rcode -% 1+1 -% -% x=rnorm(5); t(x) -%% end.rcode - -For the cached chunk below, you will need to wait for 3 seconds for -the first time you compile this document, but it takes no time the -next time you run it again. - -%% chunk options: cache this chunk -%% begin.rcode my-cache, cache=TRUE -% set.seed(123) -% x = runif(10) -% sd(x) # standard deviation -% -% Sys.sleep(3) # test cache -%% end.rcode - -Now we know the first element of x is \rinline{x[1]}. -And we also know the 26 letters are \rinline{LETTERS}. -An expression that returns a value of length 0 will be removed from the output, \rinline{x[1] = 2011; NULL} but it was indeed evaluated, -i.~e. now the first element of x becomes \rinline{x[1]}. - -How about figures? Let's use the Cairo PDF device (assumes R $\geq$ 2.14.0). - -%% begin.rcode cairo-scatter, dev='cairo_pdf', fig.width=5, fig.height=5, out.width='.8\\textwidth' -% plot(cars) # a scatter plot -%% end.rcode - -Warnings, messages and errors are preserved by default. - -%% begin.rcode -% sqrt(-1) # here is a warning! -% message('this is a message you should know') -% 1+'a' # impossible -%% end.rcode - -\end{document} diff --git a/src/smc_pyutil/smc_pyutil/templates/linux/default.sagews b/src/smc_pyutil/smc_pyutil/templates/linux/default.sagews deleted file mode 100755 index e69de29bb2..0000000000 diff --git a/src/smc_pyutil/smc_pyutil/templates/linux/default.tex b/src/smc_pyutil/smc_pyutil/templates/linux/default.tex deleted file mode 100755 index 99a09819de..0000000000 --- a/src/smc_pyutil/smc_pyutil/templates/linux/default.tex +++ /dev/null @@ -1,33 +0,0 @@ -\documentclass{article} - -% set font encoding for PDFLaTeX, XeLaTeX, or LuaTeX -\usepackage{ifxetex,ifluatex} -\if\ifxetex T\else\ifluatex T\else F\fi\fi T% - \usepackage{fontspec} -\else - \usepackage[T1]{fontenc} - \usepackage[utf8]{inputenc} - \usepackage{lmodern} -\fi - -\usepackage{hyperref} -\usepackage{amsmath} - -\title{Title of Document} -\author{Name of Author} - -% Enable SageTeX to run SageMath code right inside this LaTeX file. -% http://doc.sagemath.org/html/en/tutorial/sagetex.html -% \usepackage{sagetex} - -% Enable PythonTeX to run Python – https://ctan.org/pkg/pythontex -% \usepackage{pythontex} - -\begin{document} -\maketitle - - - - - -\end{document} From a82290f5c66472f8dda454c524abb1bc84ee8e72 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 15 Aug 2025 21:47:01 +0000 Subject: [PATCH 312/798] do not show directory sizes in flyout panel --- src/packages/frontend/project/page/flyouts/files.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/packages/frontend/project/page/flyouts/files.tsx b/src/packages/frontend/project/page/flyouts/files.tsx index 5834d2ec0c..b79746a899 100644 --- a/src/packages/frontend/project/page/flyouts/files.tsx +++ b/src/packages/frontend/project/page/flyouts/files.tsx @@ -475,7 +475,7 @@ export function FilesFlyout({ return capitalize(file_options(item.name).name) || ext; case "name": case "size": - return human_readable_size(item.size, true); + return item.isDir ? "" : human_readable_size(item.size, true); default: return null; } @@ -487,7 +487,7 @@ export function FilesFlyout({ switch (col) { case "time": case "type": - return human_readable_size(item.size, true); + return item.isDir ? "" : human_readable_size(item.size, true); case "size": case "name": return renderTimeAgo(item); From 9f79445d85f4dfaf2b10b7433ec9d9a32f3058b5 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 15 Aug 2025 22:38:43 +0000 Subject: [PATCH 313/798] the templates for new file creation --- src/packages/frontend/project/templates.ts | 132 +++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 src/packages/frontend/project/templates.ts diff --git a/src/packages/frontend/project/templates.ts b/src/packages/frontend/project/templates.ts new file mode 100644 index 0000000000..893c323944 --- /dev/null +++ b/src/packages/frontend/project/templates.ts @@ -0,0 +1,132 @@ +export function getFileTemplate(ext: string): string { + return TEMPLATES[ext] ?? ""; +} + +const TEMPLATES = { + rnw: String.raw`\documentclass{article} +\usepackage[utf8]{inputenc} +\usepackage[T1]{fontenc} +\usepackage{url} +\begin{document} + +% learn more about knitr: https://yihui.name/knitr/ + +<>= +library(knitr) +opts_chunk$set(cache=TRUE, autodep=TRUE) +options(formatR.arrow=TRUE, width=90) +@ + +\title{Knitr in CoCalc} + +\author{Author Name} + +\maketitle + +<>= +x <- c(2,1,7,4,4,5,4,6,4,5,4,3,4,5,1) +summary(x) +@ + +<>= +hist(x) +@ + +Sum of \Sexpr{paste(x, collapse="+")} is \Sexpr{sum(x)}. + + +\end{document}`, + tex: String.raw`\documentclass{article} + +% set font encoding for PDFLaTeX, XeLaTeX, or LuaTeX +\usepackage{ifxetex,ifluatex} +\if\ifxetex T\else\ifluatex T\else F\fi\fi T% + \usepackage{fontspec} +\else + \usepackage[T1]{fontenc} + \usepackage[utf8]{inputenc} + \usepackage{lmodern} +\fi + +\usepackage{hyperref} +\usepackage{amsmath} + +\title{Title of Document} +\author{Name of Author} + +% Enable SageTeX to run SageMath code right inside this LaTeX file. +% http://doc.sagemath.org/html/en/tutorial/sagetex.html +% \usepackage{sagetex} + +% Enable PythonTeX to run Python – https://ctan.org/pkg/pythontex +% \usepackage{pythontex} + +\begin{document} +\maketitle + + + + + +\end{document} + +`, + rtex: String.raw`\documentclass{article} +\usepackage[utf8]{inputenc} +\usepackage[T1]{fontenc} +\usepackage{url} +\usepackage{graphicx} + +% this is based on https://github.com/yihui/knitr-examples/blob/master/005-latex.Rtex + +%% for inline R code: if the inline code is not correctly parsed, you will see a message +\newcommand{\rinline}[1]{SOMETHING WRONG WITH knitr} + +\begin{document} + +\title{Rtex Knitr in CoCalc} + +\author{Author Name} + +\maketitle + +Boring stuff as usual: + +%% a chunk with default options +%% begin.rcode +% 1+1 +% +% x=rnorm(5); t(x) +%% end.rcode + +For the cached chunk below, you will need to wait for 3 seconds for +the first time you compile this document, but it takes no time the +next time you run it again. + +%% chunk options: cache this chunk +%% begin.rcode my-cache, cache=TRUE +% set.seed(123) +% x = runif(10) +% sd(x) # standard deviation +% +% Sys.sleep(3) # test cache +%% end.rcode + +Now we know the first element of x is \rinline{x[1]}. +And we also know the 26 letters are \rinline{LETTERS}. +An expression that returns a value of length 0 will be removed from the output, \rinline{x[1] = 2011; NULL} but it was indeed evaluated, +i.~e. now the first element of x becomes \rinline{x[1]}. + +How about figures? Let's use the Cairo PDF device (assumes R $\geq$ 2.14.0). + +Warnings, messages and errors are preserved by default. + +%% begin.rcode +% sqrt(-1) # here is a warning! +% message('this is a message you should know') +% 1+'a' # impossible +%% end.rcode + +\end{document} +`, +}; From bbbbfd136f6c142ba2b9cb98b6553039eb897301 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 15 Aug 2025 22:38:48 +0000 Subject: [PATCH 314/798] clear output when running cells (matches jupyterlab behavior) --- .../conat/project/jupyter/run-code.ts | 6 ++-- .../frontend/jupyter/browser-actions.ts | 29 +++++++++++++++---- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/src/packages/conat/project/jupyter/run-code.ts b/src/packages/conat/project/jupyter/run-code.ts index 5643b21dd2..58c98c2f3d 100644 --- a/src/packages/conat/project/jupyter/run-code.ts +++ b/src/packages/conat/project/jupyter/run-code.ts @@ -213,9 +213,9 @@ async function handleRequest({ } } -class JupyterClient { +export class JupyterClient { private iter?: EventIterator; - private socket; + public readonly socket; constructor( private client: ConatClient, private subject: string, @@ -299,7 +299,7 @@ export function jupyterClient(opts: { prompt: string; password?: boolean; }) => Promise; -}) { +}): JupyterClient { const subject = getSubject(opts); return new JupyterClient( opts.client, diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index e1eb10554a..404facd72c 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -60,6 +60,7 @@ import { delay } from "awaiting"; import { until } from "@cocalc/util/async-utils"; import { jupyterClient, + type JupyterClient, type InputCell, } from "@cocalc/conat/project/jupyter/run-code"; import { OutputHandler } from "@cocalc/jupyter/execute/output-handler"; @@ -1473,7 +1474,7 @@ export class JupyterActions extends JupyterActions0 { await api.start(this.syncdbPath); return true; } catch (err) { - console.log("failed to initialize ", this.path, err); + console.warn("failed to initialize ", this.path, err); return false; } }, @@ -1516,7 +1517,19 @@ export class JupyterActions extends JupyterActions0 { pendingCells = pendingCells.add(id); } this.store.setState({ pendingCells }); + + // to avoid ugly flicker, we don't clear output until + // waiting a little while first (since often the output + // already appears in a fraction of a second): + setTimeout(() => { + if (this.isClosed()) { + return; + } + const p = this.store.get("pendingCells") ?? iSet(); + this.clear_outputs(ids.filter((id) => p.has(id))); + }, 250); }; + private deletePendingCells = (ids: string[]) => { let pendingCells = this.store.get("pendingCells"); if (pendingCells == null) { @@ -1534,7 +1547,7 @@ export class JupyterActions extends JupyterActions0 { this.runQueue.length = 0; } - private jupyterClient?; + private jupyterClient?: JupyterClient; private runQueue: any[] = []; private runningNow = false; runCells = async (ids: string[], opts: { noHalt?: boolean } = {}) => { @@ -1546,6 +1559,7 @@ export class JupyterActions extends JupyterActions0 { this.addPendingCells(ids); return; } + let client: null | JupyterClient = null; try { this.runningNow = true; if ( @@ -1581,7 +1595,7 @@ export class JupyterActions extends JupyterActions0 { this.runningNow = false; }); } - const client = this.jupyterClient; + client = this.jupyterClient; if (client == null) { throw Error("bug"); } @@ -1658,9 +1672,12 @@ export class JupyterActions extends JupyterActions0 { } }, 1000); } catch (err) { - console.warn("runCells", err); - this.clearRunQueue(); - this.set_error(err); + if (client?.socket?.state == "ready") { + // very strange err that wasn't just caused by the socket closing: + console.warn("runCells", err); + this.clearRunQueue(); + this.set_error(err); + } } finally { if (this.isClosed()) return; this.runningNow = false; From c9950fd4eb1ed589f910cc1017d7b5aba80be6ef Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 15 Aug 2025 23:56:51 +0000 Subject: [PATCH 315/798] mainly improving typescript for jupyter code in prep for implementing loading ipynb from disk --- src/packages/frontend/project/open-file.ts | 1 - .../jupyter/execute/output-handler.ts | 2 +- .../jupyter/ipynb/import-from-ipynb.ts | 73 +++++++++---------- src/packages/jupyter/ipynb/parse.ts | 2 +- src/packages/jupyter/redux/actions.ts | 2 +- src/packages/sync/editor/generic/sync-doc.ts | 1 + 6 files changed, 39 insertions(+), 42 deletions(-) diff --git a/src/packages/frontend/project/open-file.ts b/src/packages/frontend/project/open-file.ts index b4707ca36c..44710955ad 100644 --- a/src/packages/frontend/project/open-file.ts +++ b/src/packages/frontend/project/open-file.ts @@ -238,7 +238,6 @@ export async function open_file( is_public: false, }) as any; - if (!alreadyOpened || file_info.is_public !== is_public) { const was_public = file_info.is_public; diff --git a/src/packages/jupyter/execute/output-handler.ts b/src/packages/jupyter/execute/output-handler.ts index 94d5ae0cd0..040cf10da8 100644 --- a/src/packages/jupyter/execute/output-handler.ts +++ b/src/packages/jupyter/execute/output-handler.ts @@ -40,7 +40,7 @@ import { type Cell } from "@cocalc/jupyter/ipynb/export-to-ipynb"; export { type Cell }; -interface Message { +export interface Message { execution_state?; execution_count?: number; exec_count?: number | null; diff --git a/src/packages/jupyter/ipynb/import-from-ipynb.ts b/src/packages/jupyter/ipynb/import-from-ipynb.ts index f1ecc68d99..2feff53f6d 100644 --- a/src/packages/jupyter/ipynb/import-from-ipynb.ts +++ b/src/packages/jupyter/ipynb/import-from-ipynb.ts @@ -12,6 +12,8 @@ import { JUPYTER_MIMETYPES, JUPYTER_MIMETYPES_SET, } from "@cocalc/jupyter/util/misc"; +import { type Message } from "@cocalc/jupyter/execute/output-handler"; +import { close } from "@cocalc/util/misc"; const DEFAULT_IPYNB = { cells: [ @@ -33,28 +35,40 @@ const DEFAULT_IPYNB = { export class IPynbImporter { private _ipynb: any; - private _new_id: any; - private _output_handler: any; - private _existing_ids: any; + private _new_id?: (is_available?: (string) => boolean) => string; + private _cellOutputHandler?: (cell) => { + message: (content: Message) => void; + done?: () => void; + }; + private _existing_ids: string[]; private _cells: any; private _kernel: any; private _metadata: any; - private _language_info: any; - import = (opts: any) => { - opts = misc.defaults(opts, { - ipynb: {}, - new_id: undefined, // function that returns an unused id given - // an is_available function; new_id(is_available) = a new id. - existing_ids: [], // re-use these on loading for efficiency purposes - output_handler: undefined, // h = output_handler(cell); h.message(...) -- hard to explain - }); // process attachments: attachment(base64, mime) --> sha1 - this._ipynb = misc.deep_copy(opts.ipynb); - this._new_id = opts.new_id; - this._output_handler = opts.output_handler; - this._existing_ids = opts.existing_ids; // option to re-use existing ids + import = ({ + ipynb, + new_id, + existing_ids = [], + cellOutputHandler, + }: { + ipynb: object; + // function that returns an unused id given + // an is_available function; new_id(is_available) = a new id. + new_id?: (is_available?: (string) => boolean) => string; + // re-use these on loading for efficiency purposes + existing_ids?: string[]; + cellOutputHandler?: (cell) => { + message: (content: Message) => void; + done?: () => void; + }; + }) => { + this._ipynb = misc.deep_copy(ipynb); + this._new_id = new_id; + this._cellOutputHandler = cellOutputHandler; + this._existing_ids = existing_ids; // option to re-use existing ids - this._handle_old_versions(); // must come before sanity checks, as old versions are "insane". -- see https://github.com/sagemathinc/cocalc/issues/1937 + // must come before sanity checks, as old versions are "insane". -- see https://github.com/sagemathinc/cocalc/issues/1937 + this._handle_old_versions(); this._sanity_improvements(); this._import_settings(); this._import_metadata(); @@ -73,14 +87,7 @@ export class IPynbImporter { }; close = () => { - delete this._cells; - delete this._kernel; - delete this._metadata; - delete this._language_info; - delete this._ipynb; - delete this._existing_ids; - delete this._new_id; - delete this._output_handler; + close(this); }; // Everything below is the internal private implementation. @@ -298,11 +305,8 @@ export class IPynbImporter { if (outputs == null || outputs.length == 0) { return null; } - let handler: any; const cell: any = { id, output: {} }; - if (this._output_handler != null) { - handler = this._output_handler(cell); - } + const handler = this._cellOutputHandler?.(cell); let k: string; // it's perfectly fine that k is a string here. for (k in outputs) { let content = outputs[k]; @@ -316,9 +320,7 @@ export class IPynbImporter { cell.output[k] = content; } } - if (handler != null && typeof handler.done === "function") { - handler.done(); - } + handler?.done?.(); return cell.output; }; @@ -338,12 +340,7 @@ export class IPynbImporter { } _import_cell(cell: any, n: any) { - const id = - (this._existing_ids != null ? this._existing_ids[n] : undefined) != null - ? this._existing_ids != null - ? this._existing_ids[n] - : undefined - : this._get_new_id(cell); + const id = this._existing_ids[n] ?? this._get_new_id(cell); const obj: any = { type: "cell", id, diff --git a/src/packages/jupyter/ipynb/parse.ts b/src/packages/jupyter/ipynb/parse.ts index c70873785f..d26f3d9b1f 100644 --- a/src/packages/jupyter/ipynb/parse.ts +++ b/src/packages/jupyter/ipynb/parse.ts @@ -27,7 +27,7 @@ export default function parse(content: string): CoCalcJupyter { const importer = new IPynbImporter(); importer.import({ ipynb, - output_handler: (cell) => { + cellOutputHandler: (cell) => { let k: number = 0; return { message: (content) => { diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index b66bddb166..97bed2e869 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -1696,7 +1696,7 @@ export class JupyterActions extends Actions { ipynb, existing_ids, new_id: this.new_id.bind(this), - output_handler: + cellOutputHandler: this.jupyter_kernel != null ? this._output_handler.bind(this) : undefined, // undefined in client; defined in project diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index d9642da370..48fb5dd977 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -2857,6 +2857,7 @@ export class SyncDoc extends EventEmitter { await this.stat(); return true; } catch (err) { + console.log("sync fileExists err", err); if (err.code == "ENOENT") { // file not there now. return false; From 7de82fc8541888c205b7c60ef6114e198657bdca Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 16 Aug 2025 00:06:22 +0000 Subject: [PATCH 316/798] refactor jupyter message processing so can use on frontend --- src/packages/jupyter/control.ts | 4 +- src/packages/jupyter/kernel/kernel.ts | 85 ------------------- src/packages/jupyter/redux/actions.ts | 82 ++++++++++++++++++ .../jupyter/types/project-interface.ts | 1 - 4 files changed, 84 insertions(+), 88 deletions(-) diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts index b59eb8d1bb..a6bf1f895a 100644 --- a/src/packages/jupyter/control.ts +++ b/src/packages/jupyter/control.ts @@ -113,8 +113,8 @@ export async function run({ path, cells, noHalt, socket }: RunOptions) { for await (const mesg0 of output.iter()) { const content = mesg0?.content; if (content != null) { - // this mutates content, removing large messages - await kernel.process_output(content); + // this mutates content, removing large base64/svg, etc. images, pdf's, etc. + await actions.processOutput(content); } const mesg = { ...mesg0, id: cell.id }; yield mesg; diff --git a/src/packages/jupyter/kernel/kernel.ts b/src/packages/jupyter/kernel/kernel.ts index 564d839013..89dec92b8b 100644 --- a/src/packages/jupyter/kernel/kernel.ts +++ b/src/packages/jupyter/kernel/kernel.ts @@ -35,7 +35,6 @@ import type { MessageType } from "@cocalc/jupyter/zmq/types"; import { jupyterSockets, type JupyterSockets } from "@cocalc/jupyter/zmq"; import { EventEmitter } from "node:events"; import { unlink } from "@cocalc/backend/misc/async-utils-node"; -import { remove_redundant_reps } from "@cocalc/jupyter/ipynb/import-from-ipynb"; import { type BlobStoreInterface, CodeExecutionEmitterInterface, @@ -45,11 +44,6 @@ import { } from "@cocalc/jupyter/types/project-interface"; import { JupyterActions } from "@cocalc/jupyter/redux/project-actions"; import { JupyterStore } from "@cocalc/jupyter/redux/store"; -import { - JUPYTER_MIMETYPES, - isJupyterBase64MimeType, -} from "@cocalc/jupyter/util/misc"; -import { isSha1 } from "@cocalc/util/misc"; import type { SyncDB } from "@cocalc/sync/editor/db/sync"; import { retry_until_success, until } from "@cocalc/util/async-utils"; import createChdirCommand from "@cocalc/util/jupyter-api/chdir-commands"; @@ -785,85 +779,6 @@ export class JupyterKernel return v; }; - private saveBlob = async (data: string, type: string) => { - if (this._actions == null) { - throw Error("blob store not available"); - } - const buf = Buffer.from( - data, - isJupyterBase64MimeType(type) ? "base64" : undefined, - ); - - const sha1: string = misc_node_sha1(buf); - await this._actions.asyncBlobStore.set(sha1, buf); - return sha1; - }; - - process_output = async (content: any) => { - if (this._state === "closed") { - return; - } - const dbg = this.dbg("process_output"); - dbg(); - if (content.data == null) { - // No data -- https://github.com/sagemathinc/cocalc/issues/6665 - // NO do not do this sort of thing. This is exactly the sort of situation where - // content could be very large, and JSON.stringify could use huge amounts of memory. - // If you need to see this for debugging, uncomment it. - // dbg(trunc(JSON.stringify(content), 300)); - // todo: FOR now -- later may remove large stdout, stderr, etc... - // dbg("no data, so nothing to do"); - return; - } - - remove_redundant_reps(content.data); - - const saveBlob = async (data, type) => { - try { - return await this.saveBlob(data, type); - } catch (err) { - dbg("WARNING: Jupyter blob store not working -- skipping use", err); - // i think it'll just send the large data on in the usual way instead - // via the output, instead of using the blob store. It's probably just - // less efficient. - } - }; - - let type: string; - for (type of JUPYTER_MIMETYPES) { - if (content.data[type] == null) { - continue; - } - if ( - type.split("/")[0] === "image" || - type === "application/pdf" || - type === "text/html" - ) { - // Store all images and PDF and text/html in a binary blob store, so we don't have - // to involve it in realtime sync. It tends to be large, etc. - if (isSha1(content.data[type])) { - // it was already processed, e.g., this happens when a browser that was - // processing output closes and we cutoff to the project processing output. - continue; - } - const sha1 = await saveBlob(content.data[type], type); - if (sha1) { - dbg("put content in blob store: ", { type, sha1 }); - // only remove if the save actually worked -- we don't want to break output - // for the user for a little optimization! - if (type == "text/html") { - // NOTE: in general, this may or may not get rendered as an iframe -- - // we use iframe for backward compatibility. - content.data["iframe"] = sha1; - delete content.data["text/html"]; - } else { - content.data[type] = sha1; - } - } - } - } - }; - call = async (msg_type: string, content?: any): Promise => { this.dbg("call")(msg_type); if (!this.has_ensured_running) { diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index 97bed2e869..4dd99ab91a 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -45,6 +45,12 @@ import latexEnvs from "@cocalc/util/latex-envs"; import { jupyterApiClient } from "@cocalc/conat/service/jupyter"; import { type AKV, akv } from "@cocalc/conat/sync/akv"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { + JUPYTER_MIMETYPES, + isJupyterBase64MimeType, +} from "@cocalc/jupyter/util/misc"; +import { remove_redundant_reps } from "@cocalc/jupyter/ipynb/import-from-ipynb"; +import { isSha1, sha1 } from "@cocalc/util/misc"; const { close, required, defaults } = misc; @@ -2228,6 +2234,82 @@ export class JupyterActions extends Actions { return this.set_trust_notebook(true); } }; + + private saveBlob = async (data: string, type: string) => { + const buf = Buffer.from( + data, + isJupyterBase64MimeType(type) ? "base64" : undefined, + ); + + const s: string = sha1(buf); + await this.asyncBlobStore.set(s, buf); + return s; + }; + + processOutput = async (content: any) => { + if (this.isClosed()) { + return; + } + const dbg = this.dbg("process_output"); + dbg(); + if (content.data == null) { + // No data -- https://github.com/sagemathinc/cocalc/issues/6665 + // NO do not do this sort of thing. This is exactly the sort of situation where + // content could be very large, and JSON.stringify could use huge amounts of memory. + // If you need to see this for debugging, uncomment it. + // dbg(trunc(JSON.stringify(content), 300)); + // todo: FOR now -- later may remove large stdout, stderr, etc... + // dbg("no data, so nothing to do"); + return; + } + + remove_redundant_reps(content.data); + + const saveBlob = async (data, type) => { + try { + return await this.saveBlob(data, type); + } catch (err) { + dbg("WARNING: Jupyter blob store not working -- skipping use", err); + // i think it'll just send the large data on in the usual way instead + // via the output, instead of using the blob store. It's probably just + // less efficient. + } + }; + + let type: string; + for (type of JUPYTER_MIMETYPES) { + if (content.data[type] == null) { + continue; + } + if ( + type.split("/")[0] === "image" || + type === "application/pdf" || + type === "text/html" + ) { + // Store all images and PDF and text/html in a binary blob store, so we don't have + // to involve it in realtime sync. It tends to be large, etc. + if (isSha1(content.data[type])) { + // it was already processed, e.g., this happens when a browser that was + // processing output closes and we cutoff to the project processing output. + continue; + } + const s = await saveBlob(content.data[type], type); + if (s) { + dbg("put content in blob store: ", { type, sha1:s }); + // only remove if the save actually worked -- we don't want to break output + // for the user for a little optimization! + if (type == "text/html") { + // NOTE: in general, this may or may not get rendered as an iframe -- + // we use iframe for backward compatibility. + content.data["iframe"] = s; + delete content.data["text/html"]; + } else { + content.data[type] = s; + } + } + } + } + }; } function extractLabel(content: string): string { diff --git a/src/packages/jupyter/types/project-interface.ts b/src/packages/jupyter/types/project-interface.ts index 986d16cd28..3681d69fb4 100644 --- a/src/packages/jupyter/types/project-interface.ts +++ b/src/packages/jupyter/types/project-interface.ts @@ -107,7 +107,6 @@ export interface JupyterKernelInterface extends EventEmitterInterface { execute_code(opts: ExecOpts): CodeExecutionEmitterInterface; cancel_execute(id: string): void; execute_code_now(opts: ExecOpts): Promise; - process_output(content: any): void; get_blob_store(): BlobStoreInterface | undefined; complete(opts: { code: any; cursor_pos: any }); introspect(opts: { From aee5781db0595b2d5edbc35d3271846157f64a20 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 16 Aug 2025 00:08:24 +0000 Subject: [PATCH 317/798] more refactoring --- src/packages/jupyter/control.ts | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts index a6bf1f895a..2b880beafe 100644 --- a/src/packages/jupyter/control.ts +++ b/src/packages/jupyter/control.ts @@ -164,14 +164,7 @@ class MulticellOutputHandler { }, ); this.handler.on("change", f); - - this.handler.on("process", async (mesg) => { - const kernel = this.actions.jupyter_kernel; - if ((kernel?.get_state() ?? "closed") == "closed") { - return; - } - await kernel.process_output(mesg); - }); + this.handler.on("process", this.actions.processOutput); } this.handler!.process(mesg); }; From 0ca79bd064b17f8ea59bebf9b7b6c3f773e7d14a Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 16 Aug 2025 00:31:56 +0000 Subject: [PATCH 318/798] jupyter: with this we can now fully import an ipynb file -- including moving images to sha1-keyed blob store data -- client side without using the project at all --- src/packages/frontend/jupyter/raw-editor.tsx | 2 +- .../jupyter/ipynb/import-from-ipynb.ts | 10 +- src/packages/jupyter/redux/actions.ts | 275 +++++++++--------- 3 files changed, 146 insertions(+), 141 deletions(-) diff --git a/src/packages/frontend/jupyter/raw-editor.tsx b/src/packages/frontend/jupyter/raw-editor.tsx index d669aa9f05..97f02e0a60 100644 --- a/src/packages/frontend/jupyter/raw-editor.tsx +++ b/src/packages/frontend/jupyter/raw-editor.tsx @@ -99,7 +99,7 @@ export const RawEditor: React.FC = ({ actions.set_to_ipynb(obj)} + on_change={(obj) => actions.setToIpynb(obj)} cm_options={cm_options} undo={() => actions.undo()} redo={() => actions.redo()} diff --git a/src/packages/jupyter/ipynb/import-from-ipynb.ts b/src/packages/jupyter/ipynb/import-from-ipynb.ts index 2feff53f6d..72d45a1707 100644 --- a/src/packages/jupyter/ipynb/import-from-ipynb.ts +++ b/src/packages/jupyter/ipynb/import-from-ipynb.ts @@ -37,7 +37,7 @@ export class IPynbImporter { private _ipynb: any; private _new_id?: (is_available?: (string) => boolean) => string; private _cellOutputHandler?: (cell) => { - message: (content: Message) => void; + message: (content: Message, k?) => void; done?: () => void; }; private _existing_ids: string[]; @@ -58,7 +58,7 @@ export class IPynbImporter { // re-use these on loading for efficiency purposes existing_ids?: string[]; cellOutputHandler?: (cell) => { - message: (content: Message) => void; + message: (content: Message, k?) => void; done?: () => void; }; }) => { @@ -315,7 +315,7 @@ export class IPynbImporter { } this._import_cell_output_content(content); if (handler != null) { - handler.message(content); + handler.message(content, k); } else { cell.output[k] = content; } @@ -339,7 +339,7 @@ export class IPynbImporter { } } - _import_cell(cell: any, n: any) { + _import_cell = (cell: any, n: any) => { const id = this._existing_ids[n] ?? this._get_new_id(cell); const obj: any = { type: "cell", @@ -401,7 +401,7 @@ export class IPynbImporter { } } return obj; - } + }; } // mutate data removing redundant reps diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index 4dd99ab91a..7d47b8f5a5 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -694,10 +694,6 @@ export class JupyterActions extends Actions { // nontrivial in the project, but not in client or here. } - protected _output_handler(_cell: any) { - throw Error("define in a derived class."); - } - /* WARNING: Changes via set that are made when the actions are not 'ready' or the syncdb is not ready are ignored. @@ -1629,114 +1625,6 @@ export class JupyterActions extends Actions { } }; - set_to_ipynb = async ( - ipynb: any, - data_only: boolean = false, - ): Promise => { - /* - * set_to_ipynb - set from ipynb object. This is - * mainly meant to be run on the backend in the project, - * but is also run on the frontend too, e.g., - * for client-side nbviewer (in which case it won't remove images, etc.). - * - * See the documentation for load_ipynb_file in project-actions.ts for - * documentation about the data_only input variable. - */ - if (typeof ipynb != "object") { - throw Error("ipynb must be an object"); - } - - this._state = "load"; - - //dbg(misc.to_json(ipynb)) - - // We try to parse out the kernel so we can use process_output below. - // (TODO: rewrite so process_output is not associated with a specific kernel) - let kernel: string | undefined; - const ipynb_metadata = ipynb.metadata; - if (ipynb_metadata != null) { - const kernelspec = ipynb_metadata.kernelspec; - if (kernelspec != null) { - kernel = kernelspec.name; - } - } - //dbg("kernel in ipynb: name='#{kernel}'") - - const existing_ids = this.store.get_cell_list().toJS(); - - let set, trust; - if (data_only) { - trust = undefined; - set = function () {}; - } else { - if (typeof this.reset_more_output === "function") { - this.reset_more_output(); - // clear the more output handler (only on backend) - } - // We delete all of the cells. - // We do NOT delete everything, namely the last_loaded and - // the settings entry in the database, because that would - // throw away important information, e.g., the current kernel - // and its state. NOTe: Some of that extra info *should* be - // moved to a different ephemeral table, but I haven't got - // around to doing so. - this.syncdb.delete({ type: "cell" }); - // preserve trust state across file updates/loads - trust = this.store.get("trust"); - set = (obj) => { - this.syncdb.set(obj); - }; - } - - // Change kernel to what is in the file if necessary: - set({ type: "settings", kernel }); - this.ensure_backend_kernel_setup(); - - const importer = new IPynbImporter(); - - // NOTE: Below we re-use any existing ids to make the patch that defines changing - // to the contents of ipynb more efficient. In case of a very slight change - // on disk, this can be massively more efficient. - - importer.import({ - ipynb, - existing_ids, - new_id: this.new_id.bind(this), - cellOutputHandler: - this.jupyter_kernel != null - ? this._output_handler.bind(this) - : undefined, // undefined in client; defined in project - }); - - if (data_only) { - importer.close(); - return; - } - - // Set all the cells - const object = importer.cells(); - for (const _ in object) { - const cell = object[_]; - set(cell); - } - - // Set the settings - set({ type: "settings", kernel: importer.kernel(), trust }); - - // Set extra user-defined metadata - const metadata = importer.metadata(); - if (metadata != null) { - set({ type: "settings", metadata }); - } - - importer.close(); - - this.syncdb.commit(); - await this.syncdb.save(); - this.ensure_backend_kernel_setup(); - this._state = "ready"; - }; - set_cell_slide = (id: string, value: any) => { if (!value) { value = null; // delete @@ -2235,18 +2123,7 @@ export class JupyterActions extends Actions { } }; - private saveBlob = async (data: string, type: string) => { - const buf = Buffer.from( - data, - isJupyterBase64MimeType(type) ? "base64" : undefined, - ); - - const s: string = sha1(buf); - await this.asyncBlobStore.set(s, buf); - return s; - }; - - processOutput = async (content: any) => { + processOutput = (content: any) => { if (this.isClosed()) { return; } @@ -2265,15 +2142,29 @@ export class JupyterActions extends Actions { remove_redundant_reps(content.data); - const saveBlob = async (data, type) => { - try { - return await this.saveBlob(data, type); - } catch (err) { - dbg("WARNING: Jupyter blob store not working -- skipping use", err); - // i think it'll just send the large data on in the usual way instead - // via the output, instead of using the blob store. It's probably just - // less efficient. - } + // Here we compute sha1 and start saving the blob to the blob store + // over the network. There is no guarantee that will work and this + // won't just be lost. It's unlikely but could happen. This is + // always output images in notebooks, so not critical data. + // By doing this, the blobs gets saved in parallel and code that + // uses this for import etc. is non-async, so simpler. + const saveBlob = (data: string, type: string) => { + const buf = Buffer.from( + data, + isJupyterBase64MimeType(type) ? "base64" : undefined, + ); + const s: string = sha1(buf); + (async () => { + try { + await this.asyncBlobStore.set(s, buf); + } catch (err) { + dbg("WARNING: Jupyter blob store not working -- skipping use", err); + // i think it'll just send the large data on in the usual way instead + // via the output, instead of using the blob store. It's probably just + // less efficient. + } + })(); + return s; }; let type: string; @@ -2293,9 +2184,9 @@ export class JupyterActions extends Actions { // processing output closes and we cutoff to the project processing output. continue; } - const s = await saveBlob(content.data[type], type); + const s = saveBlob(content.data[type], type); if (s) { - dbg("put content in blob store: ", { type, sha1:s }); + dbg("put content in blob store: ", { type, sha1: s }); // only remove if the save actually worked -- we don't want to break output // for the user for a little optimization! if (type == "text/html") { @@ -2310,6 +2201,120 @@ export class JupyterActions extends Actions { } } }; + + private cellOutputHandler = (cell) => { + return { + message: (content, k) => { + this.processOutput(content); + cell.output[k] = content; + }, + }; + }; + + setToIpynb = async ( + ipynb: any, + data_only: boolean = false, + ): Promise => { + /* + * set_to_ipynb - set from ipynb object. This is + * mainly meant to be run on the backend in the project, + * but is also run on the frontend too, e.g., + * for client-side nbviewer (in which case it won't remove images, etc.). + * + * See the documentation for load_ipynb_file in project-actions.ts for + * documentation about the data_only input variable. + */ + if (typeof ipynb != "object") { + throw Error("ipynb must be an object"); + } + + this._state = "load"; + + //dbg(misc.to_json(ipynb)) + + // We try to parse out the kernel so we can use process_output below. + // (TODO: rewrite so process_output is not associated with a specific kernel) + let kernel: string | undefined; + const ipynb_metadata = ipynb.metadata; + if (ipynb_metadata != null) { + const kernelspec = ipynb_metadata.kernelspec; + if (kernelspec != null) { + kernel = kernelspec.name; + } + } + //dbg("kernel in ipynb: name='#{kernel}'") + + const existing_ids = this.store.get_cell_list().toJS(); + + let set, trust; + if (data_only) { + trust = undefined; + set = function () {}; + } else { + if (typeof this.reset_more_output === "function") { + this.reset_more_output(); + // clear the more output handler (only on backend) + } + // We delete all of the cells. + // We do NOT delete everything, namely the last_loaded and + // the settings entry in the database, because that would + // throw away important information, e.g., the current kernel + // and its state. NOTe: Some of that extra info *should* be + // moved to a different ephemeral table, but I haven't got + // around to doing so. + this.syncdb.delete({ type: "cell" }); + // preserve trust state across file updates/loads + trust = this.store.get("trust"); + set = (obj) => { + this.syncdb.set(obj); + }; + } + + // Change kernel to what is in the file if necessary: + set({ type: "settings", kernel }); + this.ensure_backend_kernel_setup(); + + const importer = new IPynbImporter(); + + // NOTE: Below we re-use any existing ids to make the patch that defines changing + // to the contents of ipynb more efficient. In case of a very slight change + // on disk, this can be massively more efficient. + + importer.import({ + ipynb, + existing_ids, + new_id: this.new_id.bind(this), + cellOutputHandler: this.cellOutputHandler, + }); + + if (data_only) { + importer.close(); + return; + } + + // Set all the cells + const object = importer.cells(); + for (const _ in object) { + const cell = object[_]; + set(cell); + } + + // Set the settings + set({ type: "settings", kernel: importer.kernel(), trust }); + + // Set extra user-defined metadata + const metadata = importer.metadata(); + if (metadata != null) { + set({ type: "settings", metadata }); + } + + importer.close(); + + this.syncdb.commit(); + await this.syncdb.save(); + this.ensure_backend_kernel_setup(); + this._state = "ready"; + }; } function extractLabel(content: string): string { From 16b70ab81d32ae7c7557f166e21e44363f904adf Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 16 Aug 2025 00:39:06 +0000 Subject: [PATCH 319/798] jupyter: load from ipynb function that works - but now save to disk does NOT work --- src/packages/frontend/jupyter/browser-actions.ts | 1 + src/packages/jupyter/redux/actions.ts | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 404facd72c..689eb2f83e 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -1969,6 +1969,7 @@ export class JupyterActions extends JupyterActions0 { const api = await this.jupyterApi(); return await api.getConnectionFile({ path: this.path }); }; + } function getCompletionGroup(x: string): number { diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index 7d47b8f5a5..72cab00030 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -2315,6 +2315,15 @@ export class JupyterActions extends Actions { this.ensure_backend_kernel_setup(); this._state = "ready"; }; + + // load the ipynb version of this notebook from disk + loadFromDisk = async () => { + const fs = this.syncdb.fs; + const ipynb = JSON.parse( + Buffer.from(await fs.readFile(this.path)).toString(), + ); + await this.setToIpynb(ipynb); + }; } function extractLabel(content: string): string { From a31fa2acd62aa8b726aa94519f360ef5c8fc9e28 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 16 Aug 2025 01:05:06 +0000 Subject: [PATCH 320/798] jupyter: fix issue with exporting svg --- src/packages/jupyter/ipynb/export-to-ipynb.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/packages/jupyter/ipynb/export-to-ipynb.ts b/src/packages/jupyter/ipynb/export-to-ipynb.ts index 97aa2a9806..21867a679c 100644 --- a/src/packages/jupyter/ipynb/export-to-ipynb.ts +++ b/src/packages/jupyter/ipynb/export-to-ipynb.ts @@ -10,6 +10,7 @@ Exporting from our in-memory sync-friendly format to ipynb import { CellType } from "@cocalc/util/jupyter/types"; import { deep_copy, filename_extension, keys } from "@cocalc/util/misc"; import { isSha1 } from "@cocalc/util/misc"; +import { isJupyterBase64MimeType } from "@cocalc/jupyter/util/misc"; type Tags = { [key: string]: boolean }; @@ -290,12 +291,15 @@ function processOutputN( ) { if (blob_store != null) { let value; - if (k === "iframe") { - delete output_n.data[k]; - k = "text/html"; - value = blob_store.getString(v); - } else { + if (isJupyterBase64MimeType(k)) { value = blob_store.getBase64(v); + } else { + value = blob_store.getString(v); + value = value?.split("\n").map((x) => x + "\n"); + if (k === "iframe") { + delete output_n.data[k]; + k = "text/html"; + } } if (value == null) { // The image is no longer known; this could happen if the user reverts in the history From 4ecaa7add8520c3e09a75fee302d52130042da9f Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 16 Aug 2025 04:43:22 +0000 Subject: [PATCH 321/798] implement a new "never save" mode for sync-doc --- .../conat/test/sync-doc/no-disk.test.ts | 42 ++++++++++++++ .../conat/test/sync-doc/syncstring.test.ts | 55 ++++++++++++++++++- src/packages/backend/sandbox/find.test.ts | 2 +- src/packages/backend/sandbox/index.ts | 3 +- src/packages/conat/files/fs.ts | 8 +++ src/packages/conat/sync-doc/syncdb.ts | 6 +- src/packages/conat/sync-doc/syncstring.ts | 6 +- .../jupyter-editor/jupyter-actions.ts | 1 - .../server/conat/project/test/sync.test.ts | 13 ++++- src/packages/sync/editor/generic/sync-doc.ts | 44 ++++++++++++--- 10 files changed, 162 insertions(+), 18 deletions(-) create mode 100644 src/packages/backend/conat/test/sync-doc/no-disk.test.ts diff --git a/src/packages/backend/conat/test/sync-doc/no-disk.test.ts b/src/packages/backend/conat/test/sync-doc/no-disk.test.ts new file mode 100644 index 0000000000..59296d236d --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/no-disk.test.ts @@ -0,0 +1,42 @@ + + +import { before, after, uuid, connect, server, once, delay } from "./setup"; + +beforeAll(before); +afterAll(after); + +describe("loading/saving syncstring to disk and setting values", () => { + let s; + const project_id = uuid(); + let client; + + it("creates the client", () => { + client = connect(); + }); + + it("create syncstring -- we still have to give a filename to define the 'location'", async () => { + s = client.sync.string({ + project_id, + path: "new.txt", + service: server.service, + noFs: true, + }); + await once(s, "ready"); + expect(s.to_str()).toBe(""); + expect(s.versions().length).toBe(0); + s.from_str("foo"); + await s.save_to_disk(); + }); + + let fs; + it("get fs access and observe new.txt is NOT created on disk", async () => { + fs = client.fs({ project_id, service: server.service }); + expect(await fs.exists("new.txt")).toBe(false); + }); + + it("writing to new.txt has no impact on the non-fs syncstring", async () => { + await fs.writeFile("new.txt", "hello"); + await delay(200); + expect(s.to_str()).toEqual("foo"); + }); +}); diff --git a/src/packages/backend/conat/test/sync-doc/syncstring.test.ts b/src/packages/backend/conat/test/sync-doc/syncstring.test.ts index 3990baaa55..441fec0c26 100644 --- a/src/packages/backend/conat/test/sync-doc/syncstring.test.ts +++ b/src/packages/backend/conat/test/sync-doc/syncstring.test.ts @@ -1,9 +1,18 @@ -import { before, after, uuid, wait, connect, server, once } from "./setup"; +import { + before, + after, + uuid, + wait, + connect, + server, + once, + delay, +} from "./setup"; beforeAll(before); afterAll(after); -describe("loading/saving syncstring to disk and setting values", () => { +describe("create syncstring without setting fs, so saving to disk and loading from disk are a no-op", () => { let s; const project_id = uuid(); let client; @@ -124,4 +133,44 @@ describe("synchronized editing with two copies of a syncstring", () => { s2.show_history({ log: (x) => v2.push(x) }); expect(v1).toEqual(v2); }); -}); \ No newline at end of file +}); + +describe.only("opening a new syncstring for a file that does NOT exist on disk does not emit a 'deleted' event", () => { + let s; + const project_id = uuid(); + let deleted = 0; + it("a syncstring associated to a file that does not exist on disk is initialized to the empty string", async () => { + const client = connect(); + s = client.sync.string({ + project_id, + path: "this-file-is-not-on-disk.txt", + service: server.service, + // very aggressive: + deletedThreshold: 50, + deletedCheckInterval: 50, + ignoreOnSaveInterval: 50, + watchRecreateWait: 50, + }); + s.on("deleted", () => { + deleted++; + }); + await once(s, "ready"); + expect(s.to_str()).toBe(""); + expect(s.versions().length).toBe(0); + }); + + it("wait a bit and deleted is NOT emited", async () => { + await delay(500); + expect(deleted).toEqual(0); + }); + + it("change, save to disk, then delete file and get the deleted event", async () => { + s.from_str("foo"); + s.commit(); + await s.save_to_disk(); + await delay(500); + await s.fs.unlink(s.path); + await delay(500); + expect(deleted).toBe(1); + }); +}); diff --git a/src/packages/backend/sandbox/find.test.ts b/src/packages/backend/sandbox/find.test.ts index d504fb6d07..4164237f85 100644 --- a/src/packages/backend/sandbox/find.test.ts +++ b/src/packages/backend/sandbox/find.test.ts @@ -62,7 +62,7 @@ describe("find files", () => { // this is NOT a great test, unfortunately. const count = 5000; - it(`hopefully exceed the timeout by creating ${count} files`, async () => { + it.skip(`hopefully exceed the timeout by creating ${count} files`, async () => { for (let i = 0; i < count; i++) { await writeFile(join(tempDir, `${i}`), ""); } diff --git a/src/packages/backend/sandbox/index.ts b/src/packages/backend/sandbox/index.ts index 72e84b7a31..92a99351d8 100644 --- a/src/packages/backend/sandbox/index.ts +++ b/src/packages/backend/sandbox/index.ts @@ -443,7 +443,8 @@ export class SandboxedFilesystem { // NOTE: in node v24 they fixed the fs/promises watch to have a queue, but previous // versions were clearly badly implemented so we reimplement it from scratch // using the non-promise watch. - const watcher = watch(await this.safeAbsPath(filename), options as any); + const path = await this.safeAbsPath(filename); + const watcher = watch(path, options as any); const iter = new EventIterator(watcher, "change", { maxQueue: options?.maxQueue ?? 2048, overflow: options?.overflow, diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index 0a71e6bb26..14f7356236 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -536,6 +536,14 @@ export function fsClient({ const ensureWatchServerExists = call.watch.bind(call); call.watch = async (path: string, options?) => { + if (!(await call.exists(path))) { + const err = new Error( + `ENOENT: no such file or directory, watch '${path}'`, + ); + // @ts-ignore + err.code = "ENOENT"; + throw err; + } await ensureWatchServerExists(path, options); return await watchClient({ client, subject, path, options }); }; diff --git a/src/packages/conat/sync-doc/syncdb.ts b/src/packages/conat/sync-doc/syncdb.ts index 4211ad8477..6fe2e63b02 100644 --- a/src/packages/conat/sync-doc/syncdb.ts +++ b/src/packages/conat/sync-doc/syncdb.ts @@ -8,6 +8,8 @@ export type MakeOptional = Omit & export interface SyncDBOptions extends MakeOptional, "fs"> { client: ConatClient; + // no filesystem + noFs?: boolean; // name of the file service that hosts this file: service?: string; } @@ -15,7 +17,9 @@ export interface SyncDBOptions export type { SyncDB }; export function syncdb({ client, service, ...opts }: SyncDBOptions): SyncDB { - const fs = opts.fs ?? client.fs({ service, project_id: opts.project_id }); + const fs = opts.noFs + ? undefined + : (opts.fs ?? client.fs({ service, project_id: opts.project_id })); const syncClient = new SyncClient(client); return new SyncDB({ ...opts, fs, client: syncClient }); } diff --git a/src/packages/conat/sync-doc/syncstring.ts b/src/packages/conat/sync-doc/syncstring.ts index 322a2621ed..bfa11b35eb 100644 --- a/src/packages/conat/sync-doc/syncstring.ts +++ b/src/packages/conat/sync-doc/syncstring.ts @@ -9,6 +9,8 @@ import { type MakeOptional } from "./syncdb"; export interface SyncStringOptions extends MakeOptional, "fs"> { client: ConatClient; + // no filesystem + noFs?: boolean; // name of the file server that hosts this document: service?: string; } @@ -20,7 +22,9 @@ export function syncstring({ service, ...opts }: SyncStringOptions): SyncString { - const fs = opts.fs ?? client.fs({ service, project_id: opts.project_id }); + const fs = opts.noFs + ? undefined + : (opts.fs ?? client.fs({ service, project_id: opts.project_id })); const syncClient = new SyncClient(client); return new SyncString({ ...opts, fs, client: syncClient }); } diff --git a/src/packages/frontend/frame-editors/jupyter-editor/jupyter-actions.ts b/src/packages/frontend/frame-editors/jupyter-editor/jupyter-actions.ts index 61b57dda48..bd8b3bdba5 100644 --- a/src/packages/frontend/frame-editors/jupyter-editor/jupyter-actions.ts +++ b/src/packages/frontend/frame-editors/jupyter-editor/jupyter-actions.ts @@ -31,7 +31,6 @@ export function create_jupyter_actions( initial_jupyter_store_state, ); const syncdb_path = syncdbPath(path); - // Ensure meta_file isn't marked as deleted, which would block // opening the syncdb, which is clearly not the user's intention // at this point (since we're opening the ipynb file). diff --git a/src/packages/server/conat/project/test/sync.test.ts b/src/packages/server/conat/project/test/sync.test.ts index fd0ab0c731..adf26cb55c 100644 --- a/src/packages/server/conat/project/test/sync.test.ts +++ b/src/packages/server/conat/project/test/sync.test.ts @@ -81,6 +81,15 @@ describe("basic collab editing of a file *on disk* in a project -- verifying int expect(syncstring2.versions().length).toEqual(1); }); + it("the clients loaded the file at the same time but this does NOT result in two copies (via a merge conflict)", async () => { + const change = once(syncstring, "change"); + const change2 = once(syncstring2, "change"); + await Promise.all([syncstring.save(), syncstring2.save()]); + await Promise.all([change, change2]); + expect(syncstring.to_str()).toEqual("hello"); + expect(syncstring2.to_str()).toEqual("hello"); + }); + it("change the file and save to disk, then read from filesystem", async () => { syncstring.from_str("hello world"); await syncstring.save_to_disk(); @@ -93,6 +102,8 @@ describe("basic collab editing of a file *on disk* in a project -- verifying int await delay(syncstring.opts.ignoreOnSaveInterval + 50); await fs.writeFile("a.txt", "Hello World!"); await change; + console.log(syncstring.to_str()); + console.log(syncstring.show_history()); expect(syncstring.to_str()).toEqual("Hello World!"); }); @@ -134,7 +145,7 @@ describe("basic collab editing of a file *on disk* in a project -- verifying int await fs.cp("old.txt", "a.txt", { preserveTimestamps: true }); await change; expect(syncstring.to_str()).toEqual("i am old"); - // [ ] TODO: it's very disconceting that isDeleted stays true for + // [ ] TODO: it's very disconcerting that isDeleted stays true for // one of these! // await wait({ // until: () => { diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 48fb5dd977..7268849c38 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -129,8 +129,10 @@ export interface SyncOpts0 { // which data/changefeed server to use data_server?: DataServer; - // filesystem interface. - fs: Filesystem; + // filesystem interface -- if not set then never saves + // to disk or load from disk. This is NOT ephemeral -- the + // state is all tracked and sync'd via conat (and sqlite). + fs?: Filesystem; // if true, do not implicitly save on commit. This is very // useful for unit testing to easily simulate offline state. @@ -2267,17 +2269,20 @@ export class SyncDoc extends EventEmitter { this.emit("metadata-change"); }; + private lastReadFile: number = 0; readFile = reuseInFlight(async (): Promise => { + if (this.fs == null) { + return 0; + } const dbg = this.dbg("readFile"); let size: number; let contents; try { contents = await this.fs.readFile(this.path, "utf8"); + this.lastReadFile = Date.now(); // console.log(this.client.client.id, "read from disk --isDeleted = false"); - if (this.isDeleted) { - this.isDeleted = false; - } + this.isDeleted = false; this.valueOnDisk = contents; dbg("file exists"); size = contents.length; @@ -2285,7 +2290,10 @@ export class SyncDoc extends EventEmitter { } catch (err) { if (err.code == "ENOENT") { // console.log(this.client.client.id, "reset doc and set isDeleted=true"); - this.isDeleted = true; + if (!this.isDeleted) { + this.isDeleted = true; + this.emit("deleted"); + } dbg("file no longer exists -- setting to blank"); size = 0; this.from_str(""); @@ -2311,8 +2319,13 @@ export class SyncDoc extends EventEmitter { private stats?: Stats; stat = async (): Promise => { + if (this.fs == null) { + throw Error("fs disabled"); + } const prevStats = this.stats; this.stats = (await this.fs.stat(this.path)) as Stats; + this.isDeleted = false; // definitely not deleted since we just stat' it + this.lastReadFile = Date.now(); if (prevStats?.mode != this.stats.mode) { // used by clients to track read-only state. this.emit("metadata-change"); @@ -2447,6 +2460,9 @@ export class SyncDoc extends EventEmitter { }; writeFile = async () => { + if (this.fs == null) { + return; + } const dbg = this.dbg("writeFile"); if (this.client.is_deleted(this.path, this.project_id)) { dbg("not saving to disk because deleted"); @@ -2477,6 +2493,7 @@ export class SyncDoc extends EventEmitter { this.last_save_to_disk_time = new Date(); try { await this.fs.writeFile(this.path, value); + this.lastReadFile = Date.now(); } catch (err) { if (err.code == "EACCES") { try { @@ -2783,6 +2800,9 @@ export class SyncDoc extends EventEmitter { private fileWatcher?: any; private initFileWatcher = async () => { + if (this.fs == null) { + return; + } // use this.fs interface to watch path for changes -- we try once: try { this.fileWatcher = await this.fs.watch(this.path, { unique: true }); @@ -2857,7 +2877,7 @@ export class SyncDoc extends EventEmitter { await this.stat(); return true; } catch (err) { - console.log("sync fileExists err", err); + // console.log("sync fileExists err", err); if (err.code == "ENOENT") { // file not there now. return false; @@ -2868,6 +2888,10 @@ export class SyncDoc extends EventEmitter { private signalIfFileDeleted = async (): Promise => { if (this.isClosed()) return; + if (!this.lastReadFile) { + // can't be a 'deleted' because it doesn't even exist yet. + return; + } const start = Date.now(); const threshold = this.opts.deletedThreshold ?? DELETED_THRESHOLD; while (!this.isClosed()) { @@ -2890,8 +2914,10 @@ export class SyncDoc extends EventEmitter { this.from_str(""); this.commit(); // console.log("emit deleted and set isDeleted=true"); - this.isDeleted = true; - this.emit("deleted"); + if (!this.isDeleted) { + this.isDeleted = true; + this.emit("deleted"); + } return; } await delay( From 806a2e7c2012d963dd1fbe03305da8c6ab895018 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 16 Aug 2025 04:52:58 +0000 Subject: [PATCH 322/798] using noSaveToDisk explicitly is a much clearer way to set the option for not using the disk --- .../conat/test/sync-doc/no-disk.test.ts | 2 +- src/packages/conat/sync-doc/syncdb.ts | 6 +----- src/packages/conat/sync-doc/syncstring.ts | 6 +----- src/packages/jupyter/redux/sync.ts | 1 + src/packages/sync/editor/generic/sync-doc.ts | 20 ++++++++++--------- 5 files changed, 15 insertions(+), 20 deletions(-) diff --git a/src/packages/backend/conat/test/sync-doc/no-disk.test.ts b/src/packages/backend/conat/test/sync-doc/no-disk.test.ts index 59296d236d..67bdb5d380 100644 --- a/src/packages/backend/conat/test/sync-doc/no-disk.test.ts +++ b/src/packages/backend/conat/test/sync-doc/no-disk.test.ts @@ -19,7 +19,7 @@ describe("loading/saving syncstring to disk and setting values", () => { project_id, path: "new.txt", service: server.service, - noFs: true, + noSaveToDisk: true, }); await once(s, "ready"); expect(s.to_str()).toBe(""); diff --git a/src/packages/conat/sync-doc/syncdb.ts b/src/packages/conat/sync-doc/syncdb.ts index 6fe2e63b02..4211ad8477 100644 --- a/src/packages/conat/sync-doc/syncdb.ts +++ b/src/packages/conat/sync-doc/syncdb.ts @@ -8,8 +8,6 @@ export type MakeOptional = Omit & export interface SyncDBOptions extends MakeOptional, "fs"> { client: ConatClient; - // no filesystem - noFs?: boolean; // name of the file service that hosts this file: service?: string; } @@ -17,9 +15,7 @@ export interface SyncDBOptions export type { SyncDB }; export function syncdb({ client, service, ...opts }: SyncDBOptions): SyncDB { - const fs = opts.noFs - ? undefined - : (opts.fs ?? client.fs({ service, project_id: opts.project_id })); + const fs = opts.fs ?? client.fs({ service, project_id: opts.project_id }); const syncClient = new SyncClient(client); return new SyncDB({ ...opts, fs, client: syncClient }); } diff --git a/src/packages/conat/sync-doc/syncstring.ts b/src/packages/conat/sync-doc/syncstring.ts index bfa11b35eb..322a2621ed 100644 --- a/src/packages/conat/sync-doc/syncstring.ts +++ b/src/packages/conat/sync-doc/syncstring.ts @@ -9,8 +9,6 @@ import { type MakeOptional } from "./syncdb"; export interface SyncStringOptions extends MakeOptional, "fs"> { client: ConatClient; - // no filesystem - noFs?: boolean; // name of the file server that hosts this document: service?: string; } @@ -22,9 +20,7 @@ export function syncstring({ service, ...opts }: SyncStringOptions): SyncString { - const fs = opts.noFs - ? undefined - : (opts.fs ?? client.fs({ service, project_id: opts.project_id })); + const fs = opts.fs ?? client.fs({ service, project_id: opts.project_id }); const syncClient = new SyncClient(client); return new SyncString({ ...opts, fs, client: syncClient }); } diff --git a/src/packages/jupyter/redux/sync.ts b/src/packages/jupyter/redux/sync.ts index b94935c196..0638339611 100644 --- a/src/packages/jupyter/redux/sync.ts +++ b/src/packages/jupyter/redux/sync.ts @@ -5,4 +5,5 @@ export const SYNCDB_OPTIONS = { string_cols: ["input"], cursors: true, persistent: true, + noSaveToDisk: true, }; diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 7268849c38..a95d5b82c3 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -129,15 +129,17 @@ export interface SyncOpts0 { // which data/changefeed server to use data_server?: DataServer; - // filesystem interface -- if not set then never saves - // to disk or load from disk. This is NOT ephemeral -- the - // state is all tracked and sync'd via conat (and sqlite). - fs?: Filesystem; + // filesystem interface + fs: Filesystem; // if true, do not implicitly save on commit. This is very // useful for unit testing to easily simulate offline state. noAutosave?: boolean; + // if true, never saves to disk or loads from disk -- this is NOT + // ephemeral -- the history is tracked in the conat database! + noSaveToDisk?: boolean; + // optional timeout for how long to wait from when a file is // deleted until emiting a 'deleted' event. deletedThreshold?: number; @@ -2271,7 +2273,7 @@ export class SyncDoc extends EventEmitter { private lastReadFile: number = 0; readFile = reuseInFlight(async (): Promise => { - if (this.fs == null) { + if (this.opts.noSaveToDisk) { return 0; } const dbg = this.dbg("readFile"); @@ -2319,8 +2321,8 @@ export class SyncDoc extends EventEmitter { private stats?: Stats; stat = async (): Promise => { - if (this.fs == null) { - throw Error("fs disabled"); + if (this.opts.noSaveToDisk) { + throw Error("the noSaveToDisk options is set"); } const prevStats = this.stats; this.stats = (await this.fs.stat(this.path)) as Stats; @@ -2460,7 +2462,7 @@ export class SyncDoc extends EventEmitter { }; writeFile = async () => { - if (this.fs == null) { + if (this.opts.noSaveToDisk) { return; } const dbg = this.dbg("writeFile"); @@ -2800,7 +2802,7 @@ export class SyncDoc extends EventEmitter { private fileWatcher?: any; private initFileWatcher = async () => { - if (this.fs == null) { + if (this.opts.noSaveToDisk) { return; } // use this.fs interface to watch path for changes -- we try once: From 3abfc8a6e49315e626ea48e79304fadf600faf89 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 16 Aug 2025 05:20:11 +0000 Subject: [PATCH 323/798] jupyter: eliminate file from disk; store ipynb mtime in syncdb --- .../frame-editors/jupyter-editor/actions.ts | 2 +- .../frontend/jupyter/browser-actions.ts | 26 +++++++++++++++---- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/packages/frontend/frame-editors/jupyter-editor/actions.ts b/src/packages/frontend/frame-editors/jupyter-editor/actions.ts index 9591a8b9f4..d325fd2081 100644 --- a/src/packages/frontend/frame-editors/jupyter-editor/actions.ts +++ b/src/packages/frontend/frame-editors/jupyter-editor/actions.ts @@ -91,7 +91,7 @@ export class JupyterEditorActions extends BaseActions { syncdb.on("has-uncommitted-changes", (has_uncommitted_changes) => this.setState({ has_uncommitted_changes }), ); - syncdb.on("has-unsaved-changes", (has_unsaved_changes) => { + this.jupyter_actions.store.on("has-unsaved-changes", (has_unsaved_changes) => { this.setState({ has_unsaved_changes }); }); diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 689eb2f83e..16ae3d620a 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -85,6 +85,7 @@ export class JupyterActions extends JupyterActions0 { public syncdbPath: string; private lastCursorMoveTime: number = 0; public jupyterEditorActions?; + private savedVersion: number = 0; protected init2(): void { this.syncdbPath = syncdbPath(this.path); @@ -97,18 +98,21 @@ export class JupyterActions extends JupyterActions0 { const do_set = () => { if (this.syncdb == null || this._state === "closed") return; - const has_unsaved_changes = this.syncdb.has_unsaved_changes(); const has_uncommitted_changes = this.syncdb.has_uncommitted_changes(); + const has_unsaved_changes = + has_uncommitted_changes || + this.savedVersion != this.syncdb.last_changed(); this.setState({ has_unsaved_changes, has_uncommitted_changes }); + this.store.emit("has-unsaved-changes", has_unsaved_changes); if (has_uncommitted_changes) { this.syncdb.save(); // save them. } }; const f = () => { do_set(); - return setTimeout(do_set, 3000); + return setTimeout(do_set, 2000); }; - this.set_save_status = debounce(f, 1500); + this.set_save_status = debounce(f, 1000, { leading: true, trailing: true }); this.syncdb.on("metadata-change", this.set_save_status); this.syncdb.on("connected", this.set_save_status); @@ -1337,13 +1341,26 @@ export class JupyterActions extends JupyterActions0 { private saveIpynb = async () => { if (this.isClosed()) return; + const before = this.syncdb.last_changed(); const ipynb = await this.toIpynb(); const serialize = JSON.stringify(ipynb, undefined, 2); this.syncdb.fs.writeFile(this.path, serialize); + const has_unsaved_changes = this.syncdb.last_changed() != before; + this.setState({ has_unsaved_changes }); + this.store.emit("has-unsaved-changes", has_unsaved_changes); + const stats = await this.syncdb.fs.stat(this.path); + const stillNotChanged = this.syncdb.last_changed() == before; + this.syncdb.set({ type: "fs", mtime: stats.mtime.valueOf() }); + this.syncdb.commit(); + if (stillNotChanged) { + this.savedVersion = this.syncdb.last_changed(); + } else { + this.savedVersion = 0; + } }; save = async () => { - await Promise.all([this.saveIpynb(), this.syncdb.save_to_disk()]); + await this.saveIpynb(); }; private getBase64Blobs = async (cells) => { @@ -1969,7 +1986,6 @@ export class JupyterActions extends JupyterActions0 { const api = await this.jupyterApi(); return await api.getConnectionFile({ path: this.path }); }; - } function getCompletionGroup(x: string): number { From 083a79a14428a7e361d3ee5bf0830959ba3c69be Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 16 Aug 2025 13:42:16 +0000 Subject: [PATCH 324/798] change default new file name back to something clean/canonical --- src/packages/util/db-schema/defaults.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/packages/util/db-schema/defaults.ts b/src/packages/util/db-schema/defaults.ts index d4a9b061c3..fe0e77ff94 100644 --- a/src/packages/util/db-schema/defaults.ts +++ b/src/packages/util/db-schema/defaults.ts @@ -16,7 +16,7 @@ export type NewFilenameTypes = // key for new filenames algorithm in account/other_settings and associated default value export const NEW_FILENAMES = "new_filenames"; -export const DEFAULT_NEW_FILENAMES: NewFilenameTypes = "ymd_semantic"; +export const DEFAULT_NEW_FILENAMES: NewFilenameTypes = "iso"; // This is used on cocalc.com, and the storage server has images named "default", "ubuntu2004" and "ubuntu2204" // For on-prem, you have to configure the "software environment" configuration, which includes a default image name. From 6eb9900d5a275b3591cb027ef0f9662cd0d93f23 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 16 Aug 2025 13:57:22 +0000 Subject: [PATCH 325/798] implement watching the ipynb file on disk --- .../frontend/jupyter/browser-actions.ts | 180 +++++++++++++++--- src/packages/jupyter/redux/actions.ts | 8 - src/packages/sync/editor/generic/sync-doc.ts | 8 +- src/packages/util/relative-time.ts | 6 + 4 files changed, 167 insertions(+), 35 deletions(-) diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 16ae3d620a..a9381d724a 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -70,6 +70,12 @@ import { codemirror_to_jupyter_pos, js_idx_to_char_idx, } from "@cocalc/jupyter/util/misc"; +import { + IGNORE_ON_SAVE_INTERVAL, + WATCH_RECREATE_WAIT, + DELETED_THRESHOLD, + DELETED_CHECK_INTERVAL, +} from "@cocalc/sync/editor/generic/sync-doc"; const OUTPUT_FPS = 29; @@ -85,7 +91,6 @@ export class JupyterActions extends JupyterActions0 { public syncdbPath: string; private lastCursorMoveTime: number = 0; public jupyterEditorActions?; - private savedVersion: number = 0; protected init2(): void { this.syncdbPath = syncdbPath(this.path); @@ -165,6 +170,7 @@ export class JupyterActions extends JupyterActions0 { // cell notebook that has nothing to do with nbgrader). this.nbgrader_actions.update_metadata(); } + this.watchIpynb(); }); this.initOpenLog(); @@ -1261,8 +1267,8 @@ export class JupyterActions extends JupyterActions0 { // mime types like images in base64. This makes a first pass // to find the sha1-indexed blobs, gets the blobs, then does // a second pass to fill them in. There is a similar function - // in store that is sync, but doesn't fill in the blobs on the - // frontend like this does. + // in store that is sync, but doesn't fill in the blobs + // like this does. toIpynb = async () => { const store = this.store; if (store?.get("cells") == null || store?.get("cell_list") == null) { @@ -1339,26 +1345,6 @@ export class JupyterActions extends JupyterActions0 { return export_to_ipynb({ ...options, blob_store: blob_store2 }); }; - private saveIpynb = async () => { - if (this.isClosed()) return; - const before = this.syncdb.last_changed(); - const ipynb = await this.toIpynb(); - const serialize = JSON.stringify(ipynb, undefined, 2); - this.syncdb.fs.writeFile(this.path, serialize); - const has_unsaved_changes = this.syncdb.last_changed() != before; - this.setState({ has_unsaved_changes }); - this.store.emit("has-unsaved-changes", has_unsaved_changes); - const stats = await this.syncdb.fs.stat(this.path); - const stillNotChanged = this.syncdb.last_changed() == before; - this.syncdb.set({ type: "fs", mtime: stats.mtime.valueOf() }); - this.syncdb.commit(); - if (stillNotChanged) { - this.savedVersion = this.syncdb.last_changed(); - } else { - this.savedVersion = 0; - } - }; - save = async () => { await this.saveIpynb(); }; @@ -1986,6 +1972,154 @@ export class JupyterActions extends JupyterActions0 { const api = await this.jupyterApi(); return await api.getConnectionFile({ path: this.path }); }; + + // load the ipynb version of this notebook from disk + loadFromDisk = async (mtime?) => { + const fs = this.syncdb.fs; + const ipynb = JSON.parse( + Buffer.from(await fs.readFile(this.path)).toString(), + ); + mtime ??= (await this.syncdb.fs.stat(this.path)).mtime.valueOf(); + this.setIpynbMtime(mtime); + await this.setToIpynb(ipynb); + }; + + private setIpynbMtime = (mtime: number) => { + this.syncdb.set({ type: "fs", mtime }); + this.syncdb.commit(); + }; + + public savedVersion: number = 0; + saveIpynb = async () => { + if (this.isClosed()) return; + + try { + await this.fileWatcher?.ignore( + this.syncdb.opts.ignoreOnSaveInterval ?? IGNORE_ON_SAVE_INTERVAL, + ); + } catch (err) { + console.warn("issue ignoring file watcher", err); + } + + const before = this.syncdb.last_changed(); + const ipynb = await this.toIpynb(); + const serialize = JSON.stringify(ipynb, undefined, 2); + this.syncdb.fs.writeFile(this.path, serialize); + const has_unsaved_changes = this.syncdb.last_changed() != before; + this.setState({ has_unsaved_changes }); + this.store.emit("has-unsaved-changes", has_unsaved_changes); + const stats = await this.syncdb.fs.stat(this.path); + const stillNotChanged = this.syncdb.last_changed() == before; + this.setIpynbMtime(stats.mtime.valueOf()); + if (stillNotChanged) { + this.savedVersion = this.syncdb.last_changed(); + } else { + this.savedVersion = 0; + } + }; + + private isIpynbDeleted = false; + private loadFromDiskIfChanged = async () => { + const mtime = this.syncdb.get_one({ type: "fs" })?.get("mtime"); + if (mtime == null) { + await this.loadFromDisk(); + return; + } + const fs = this.syncdb.fs; + const stats = await fs.stat(this.path); + this.isIpynbDeleted = false; + + if (stats.mtime != mtime) { + await this.loadFromDisk(); + } + }; + + // TODO: The following code is very similar to code in + // @cocalc/sync/editor/generic/sync-doc + // and it seems to work very well. After writing some more + // tests, I should refactor it into a separate module that both use. + private fileWatcher?; + watchIpynb = async () => { + // one initial load right when we open the document + await until( + async () => { + if (this.isClosed()) return true; + try { + await this.loadFromDiskIfChanged(); + return true; + } catch (err) { + console.warn(`Issue watching ipynb file`, err); + return false; + } + }, + { min: 3000 }, + ); + if (this.isClosed()) return true; + const fs = this.syncdb.fs; + + await until( + async () => { + if (this.isClosed()) return true; + try { + this.fileWatcher = await fs.watch(this.path, { unique: true }); + for await (const { eventType, ignore } of this.fileWatcher) { + if (this.isClosed()) return true; + if (!ignore) { + await this.loadFromDiskIfChanged(); + } + if (eventType == "rename") { + break; + } + } + } catch {} + // check if file was deleted + this.signalIfFileDeleted(); + this.fileWatcher?.close(); + delete this.fileWatcher; + return false; + }, + { + min: this.syncdb.opts.watchRecreateWait ?? WATCH_RECREATE_WAIT, + max: this.syncdb.opts.watchRecreateWait ?? WATCH_RECREATE_WAIT, + }, + ); + }; + + private signalIfFileDeleted = async (): Promise => { + if (this.isClosed()) return; + const start = Date.now(); + const threshold = this.syncdb.opts.deletedThreshold ?? DELETED_THRESHOLD; + while (!this.isClosed()) { + try { + if (await this.syncdb.fs.exists(this.path)) { + // file definitely exists right now -- NOT deleted. + return; + } + // file definitely does NOT exist right now. + } catch { + // network not working or project off -- no way to know. + return; + } + const elapsed = Date.now() - start; + if (elapsed > threshold) { + // out of time to appear again, and definitely concluded + // it does not exist above + // file still doesn't exist -- consider it deleted -- browsers + // should close the tab and possibly notify user. + if (!this.isIpynbDeleted) { + this.isIpynbDeleted = true; + this.syncdb.emit("deleted"); + } + return; + } + await delay( + Math.min( + this.syncdb.opts.deletedCheckInterval ?? DELETED_CHECK_INTERVAL, + threshold - elapsed, + ), + ); + } + }; } function getCompletionGroup(x: string): number { diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index 72cab00030..e7a3b43d0f 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -2316,14 +2316,6 @@ export class JupyterActions extends Actions { this._state = "ready"; }; - // load the ipynb version of this notebook from disk - loadFromDisk = async () => { - const fs = this.syncdb.fs; - const ipynb = JSON.parse( - Buffer.from(await fs.readFile(this.path)).toString(), - ); - await this.setToIpynb(ipynb); - }; } function extractLabel(content: string): string { diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index a95d5b82c3..c75b489d63 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -28,15 +28,15 @@ const MAX_FILE_SIZE_MB = 32; const CURSOR_THROTTLE_MS = 150; // If file does not exist for this long, then syncdoc emits a 'deleted' event. -const DELETED_THRESHOLD = 2000; -const DELETED_CHECK_INTERVAL = 750; -const WATCH_RECREATE_WAIT = 3000; +export const DELETED_THRESHOLD = 2000; +export const DELETED_CHECK_INTERVAL = 750; +export const WATCH_RECREATE_WAIT = 3000; // all clients ignore file changes from when a save starts until this // amount of time later, so they avoid loading a file just because it was // saved by themself or another client. This is especially important for // large files that can take a long time to save. -const IGNORE_ON_SAVE_INTERVAL = 7500; +export const IGNORE_ON_SAVE_INTERVAL = 5000; // reading file when it changes on disk is deboucned this much, e.g., // if the file keeps changing you won't see those changes until it diff --git a/src/packages/util/relative-time.ts b/src/packages/util/relative-time.ts index c23f810f77..d3a8c24e56 100644 --- a/src/packages/util/relative-time.ts +++ b/src/packages/util/relative-time.ts @@ -3,6 +3,12 @@ * License: MS-RSL – see LICENSE.md for details */ +// ATTENTION/TODO: this is deprecated -- we no longer do any clock sync and server_time() +// is just the time!!! **This is still called in various places.** The historical reason +// for tracking server time was for the sync algorithm. However, we replaced it by a better +// one and no longer worry about server time. Just a heads up! + + declare var window; function getSkew(): number { From 024e9f9bc3167146586efb3766a43e5522eb0d55 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 16 Aug 2025 14:40:32 +0000 Subject: [PATCH 326/798] fix running jupyter cells in a whiteboard --- src/packages/frontend/client/welcome-file.ts | 2 +- .../jupyter-editor/cell-notebook/actions.ts | 6 ++++- .../jupyter-editor/snippets/main.tsx | 6 ++--- .../whiteboard-editor/elements/code/run.ts | 15 +++++++------ src/packages/jupyter/redux/actions.ts | 22 ++++--------------- 5 files changed, 21 insertions(+), 30 deletions(-) diff --git a/src/packages/frontend/client/welcome-file.ts b/src/packages/frontend/client/welcome-file.ts index 824894546a..7f87037bc4 100644 --- a/src/packages/frontend/client/welcome-file.ts +++ b/src/packages/frontend/client/welcome-file.ts @@ -147,7 +147,7 @@ export class WelcomeFile { if (cell.type == "markdown") { jactions.set_cell_type(cell_id, "markdown"); } else { - jactions.run_code_cell(cell_id); + jactions.runCells([cell_id]); } cell_id = jactions.insert_cell_adjacent(cell_id, +1); }); diff --git a/src/packages/frontend/frame-editors/jupyter-editor/cell-notebook/actions.ts b/src/packages/frontend/frame-editors/jupyter-editor/cell-notebook/actions.ts index c0cb91e9d5..0ae42f7c34 100644 --- a/src/packages/frontend/frame-editors/jupyter-editor/cell-notebook/actions.ts +++ b/src/packages/frontend/frame-editors/jupyter-editor/cell-notebook/actions.ts @@ -259,7 +259,11 @@ export class NotebookFrameActions { } public enable_key_handler(force?): void { - if (!force && this.jupyter_actions.store.get("stdin")) { + const store = this.jupyter_actions?.store; + if (store == null) { + return; + } + if (!force && store.get("stdin")) { // do not enable when getting input from stdin, since user may be typing // and keyboard shortcuts would be very confusing. return; diff --git a/src/packages/frontend/frame-editors/jupyter-editor/snippets/main.tsx b/src/packages/frontend/frame-editors/jupyter-editor/snippets/main.tsx index 6f7542de37..f8f8251e6f 100644 --- a/src/packages/frontend/frame-editors/jupyter-editor/snippets/main.tsx +++ b/src/packages/frontend/frame-editors/jupyter-editor/snippets/main.tsx @@ -222,7 +222,7 @@ export const JupyterSnippets: React.FC = React.memo((props: Props) => { id = jupyter_actions.insert_cell_adjacent(id, +1); jupyter_actions.set_cell_input(id, c); notebook_frame_actions.set_cur_id(id); - jupyter_actions.run_code_cell(id); + jupyter_actions.runCells([id]); } notebook_frame_actions.scroll("cell visible"); } @@ -258,8 +258,8 @@ export const JupyterSnippets: React.FC = React.memo((props: Props) => { doc[0] === "" ? undefined : typeof doc[0] === "string" - ? [doc[0]] - : doc[0]; + ? [doc[0]] + : doc[0]; if (code != null && insertSetup) { const setup = generateSetupCode({ code, data }); if (setup != "") code.unshift(setup); diff --git a/src/packages/frontend/frame-editors/whiteboard-editor/elements/code/run.ts b/src/packages/frontend/frame-editors/whiteboard-editor/elements/code/run.ts index 47b8dc4c62..28b07df1f8 100644 --- a/src/packages/frontend/frame-editors/whiteboard-editor/elements/code/run.ts +++ b/src/packages/frontend/frame-editors/whiteboard-editor/elements/code/run.ts @@ -13,8 +13,7 @@ interface Opts { set: (object) => void; } -export default async function run(opts: Opts) { - const { project_id, path, input, id, set } = opts; +export default async function run({ project_id, path, input, id, set }: Opts) { const jupyter_actions = await getJupyterActions({ project_id, path }); const store = jupyter_actions.store; let cell = store.get("cells").get(id); @@ -24,14 +23,12 @@ export default async function run(opts: Opts) { const pos = store.getIn(["cells", last_cell_id])?.get("pos", 0) + 1; jupyter_actions.insert_cell_at(pos, false, id); } + const previousEnd = cell?.get("end"); jupyter_actions.clear_outputs([id], false); jupyter_actions.set_cell_input(id, input, false); - jupyter_actions.run_code_cell(id); - //console.log("starting running ", id); - //window.jupyter_actions = jupyter_actions; + jupyter_actions.runCells([id]); function onChange() { const cell = store.get("cells").get(id); - //console.log("onChange", cell?.toJS()); if (cell == null) return; set({ @@ -42,7 +39,11 @@ export default async function run(opts: Opts) { start: cell.get("start"), end: cell.get("end"), }); - if (cell.get("state") == "done") { + if ( + cell.get("state") == "done" && + cell.get("end") && + cell.get("end") != previousEnd + ) { store.removeListener("change", onChange); // Useful for debugging since can then open the ipynb and see. // However, NOT needed normally. We might even come up with diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index e7a3b43d0f..89688f262b 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -318,7 +318,10 @@ export class JupyterActions extends Actions { const cell = cells.get(id); if (cell == null) continue; if (cell.get("output") != null || cell.get("exec_count")) { - this._set({ type: "cell", id, output: null, exec_count: null }, false); + this._set( + { type: "cell", id, output: null, exec_count: null, done: null }, + false, + ); } } if (save) { @@ -877,10 +880,6 @@ export class JupyterActions extends Actions { return this.insert_cell_at(pos, save); } - delete_selected_cells = (sync = true): void => { - this.deprecated("delete_selected_cells", sync); - }; - delete_cells(cells: string[], sync: boolean = true): void { let not_deletable: number = 0; for (const id of cells) { @@ -938,14 +937,6 @@ export class JupyterActions extends Actions { return this.syncdb?.in_undo_mode() ?? false; } - public run_code_cell( - _id: string, - _save: boolean = true, - _no_halt: boolean = false, - ): void { - console.log("run_code_cell: deprecated"); - } - clear_cell = (id: string, save = true) => { const cell = this.store.getIn(["cells", id]); @@ -968,10 +959,6 @@ export class JupyterActions extends Actions { ); }; - run_selected_cells = (): void => { - this.deprecated("run_selected_cells"); - }; - runCells(_ids: string[], _opts?: { noHalt?: boolean }): Promise { // defined in derived class (e.g., frontend browser). throw Error("DEPRECATED"); @@ -2315,7 +2302,6 @@ export class JupyterActions extends Actions { this.ensure_backend_kernel_setup(); this._state = "ready"; }; - } function extractLabel(content: string): string { From f77fc19b0e49c6754b05e19ae00ce6c7b862193c Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 16 Aug 2025 15:06:42 +0000 Subject: [PATCH 327/798] fix crash --- src/packages/frontend/jupyter/browser-actions.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index a9381d724a..385c55433e 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -2003,12 +2003,16 @@ export class JupyterActions extends JupyterActions0 { const before = this.syncdb.last_changed(); const ipynb = await this.toIpynb(); + if(this.isClosed()) return; + const serialize = JSON.stringify(ipynb, undefined, 2); this.syncdb.fs.writeFile(this.path, serialize); const has_unsaved_changes = this.syncdb.last_changed() != before; this.setState({ has_unsaved_changes }); this.store.emit("has-unsaved-changes", has_unsaved_changes); const stats = await this.syncdb.fs.stat(this.path); + if(this.isClosed()) return; + const stillNotChanged = this.syncdb.last_changed() == before; this.setIpynbMtime(stats.mtime.valueOf()); if (stillNotChanged) { From d73666b3098f3b1f7d06c1240aa6a1352225de9c Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 16 Aug 2025 15:35:27 +0000 Subject: [PATCH 328/798] don't save whiteboard ipynb to disk; fix images sometimes being broken --- .../frontend/app-framework/redux-hooks.ts | 2 +- .../elements/code/actions.ts | 2 + .../elements/code/output.tsx | 1 - .../frontend/jupyter/browser-actions.ts | 20 ++++--- .../jupyter/output-messages/image.tsx | 1 + .../jupyter/output-messages/use-blob.ts | 52 ++++++++++--------- 6 files changed, 45 insertions(+), 33 deletions(-) diff --git a/src/packages/frontend/app-framework/redux-hooks.ts b/src/packages/frontend/app-framework/redux-hooks.ts index 1fc9449825..498a277a8b 100644 --- a/src/packages/frontend/app-framework/redux-hooks.ts +++ b/src/packages/frontend/app-framework/redux-hooks.ts @@ -50,7 +50,7 @@ export function useReduxNamedStore(path: string[]) { }); useEffect(() => { - if (path[0] == "") { + if (!path[0]) { // Special case -- we allow passing "" for the name of the store and get out undefined. // This is useful when using the useRedux hook but when the name of the store isn't known initially. return undefined; diff --git a/src/packages/frontend/frame-editors/whiteboard-editor/elements/code/actions.ts b/src/packages/frontend/frame-editors/whiteboard-editor/elements/code/actions.ts index 7bf22069b6..8040c77ceb 100644 --- a/src/packages/frontend/frame-editors/whiteboard-editor/elements/code/actions.ts +++ b/src/packages/frontend/frame-editors/whiteboard-editor/elements/code/actions.ts @@ -53,6 +53,8 @@ export async function getJupyterFrameEditorActions({ if (actions == null) { throw Error("bug -- actions must be defined"); } + // do not waste effort on saving the aux ipynb to disk... + actions.jupyter_actions.noSaveToDisk = true; return actions; } diff --git a/src/packages/frontend/frame-editors/whiteboard-editor/elements/code/output.tsx b/src/packages/frontend/frame-editors/whiteboard-editor/elements/code/output.tsx index 55eca2606a..ca9c18c26d 100644 --- a/src/packages/frontend/frame-editors/whiteboard-editor/elements/code/output.tsx +++ b/src/packages/frontend/frame-editors/whiteboard-editor/elements/code/output.tsx @@ -5,7 +5,6 @@ import { fromJS } from "immutable"; import { useEffect, useRef, useState } from "react"; - import { useIsMountedRef, useRedux } from "@cocalc/frontend/app-framework"; import type { JupyterActions } from "@cocalc/frontend/jupyter/browser-actions"; import { CellOutput } from "@cocalc/frontend/jupyter/cell-output"; diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 385c55433e..f33e4d2b1f 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -92,6 +92,10 @@ export class JupyterActions extends JupyterActions0 { private lastCursorMoveTime: number = 0; public jupyterEditorActions?; + // if true, never saves to disk or loads from disk -- this is NOT + // ephemeral -- the history is tracked in the conat database! + public noSaveToDisk?: boolean; + protected init2(): void { this.syncdbPath = syncdbPath(this.path); this.setState({ @@ -2003,15 +2007,15 @@ export class JupyterActions extends JupyterActions0 { const before = this.syncdb.last_changed(); const ipynb = await this.toIpynb(); - if(this.isClosed()) return; - + if (this.isClosed()) return; + const serialize = JSON.stringify(ipynb, undefined, 2); this.syncdb.fs.writeFile(this.path, serialize); const has_unsaved_changes = this.syncdb.last_changed() != before; this.setState({ has_unsaved_changes }); this.store.emit("has-unsaved-changes", has_unsaved_changes); const stats = await this.syncdb.fs.stat(this.path); - if(this.isClosed()) return; + if (this.isClosed()) return; const stillNotChanged = this.syncdb.last_changed() == before; this.setIpynbMtime(stats.mtime.valueOf()); @@ -2044,10 +2048,12 @@ export class JupyterActions extends JupyterActions0 { // tests, I should refactor it into a separate module that both use. private fileWatcher?; watchIpynb = async () => { + const done = () => this.isClosed() || this.noSaveToDisk; + if (done()) return; // one initial load right when we open the document await until( async () => { - if (this.isClosed()) return true; + if (this.isClosed() || this.noSaveToDisk) return true; try { await this.loadFromDiskIfChanged(); return true; @@ -2058,16 +2064,16 @@ export class JupyterActions extends JupyterActions0 { }, { min: 3000 }, ); - if (this.isClosed()) return true; + if (done()) return; const fs = this.syncdb.fs; await until( async () => { - if (this.isClosed()) return true; + if (done()) return true; try { this.fileWatcher = await fs.watch(this.path, { unique: true }); for await (const { eventType, ignore } of this.fileWatcher) { - if (this.isClosed()) return true; + if (done()) return true; if (!ignore) { await this.loadFromDiskIfChanged(); } diff --git a/src/packages/frontend/jupyter/output-messages/image.tsx b/src/packages/frontend/jupyter/output-messages/image.tsx index 815b9c2bcc..cc949ae314 100644 --- a/src/packages/frontend/jupyter/output-messages/image.tsx +++ b/src/packages/frontend/jupyter/output-messages/image.tsx @@ -109,6 +109,7 @@ function RenderBlobImage({ }) { const [error, setError] = useState(""); const src = useBlob({ sha1, actions, type, setError }); + console.log({ src }); if (error) { return ( diff --git a/src/packages/frontend/jupyter/output-messages/use-blob.ts b/src/packages/frontend/jupyter/output-messages/use-blob.ts index 84f4d18fd4..917c4b3a50 100644 --- a/src/packages/frontend/jupyter/output-messages/use-blob.ts +++ b/src/packages/frontend/jupyter/output-messages/use-blob.ts @@ -1,6 +1,7 @@ import useIsMountedRef from "@cocalc/frontend/app-framework/is-mounted-hook"; import { useEffect, useState } from "react"; import LRU from "lru-cache"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; // max number of recent blob url's to save - older ones will // silently be removed and data has to be re-downloaded from server. @@ -16,31 +17,34 @@ const cache = new LRU({ }, }); -async function blobToUrl({ actions, sha1, type, leaveAsString }) { - if (cache.has(sha1)) { - return cache.get(sha1)!; - } - const buf = await actions.asyncBlobStore.get(sha1, { - timeout: BLOB_WAIT_TIMEOUT, - }); - if (buf == null) { - throw Error("Not available"); - } - let src; - if (leaveAsString != null) { - const t = new TextDecoder("utf8"); - const str = t.decode(buf); - if (leaveAsString(str)) { - src = str; - cache.set(sha1, src); - return src; +const blobToUrl = reuseInFlight( + async ({ actions, sha1, type, leaveAsString }) => { + if (cache.has(sha1)) { + return cache.get(sha1)!; } - } - const blob = new Blob([buf], { type }); - src = URL.createObjectURL(blob); - cache.set(sha1, src); - return src; -} + const buf = await actions.asyncBlobStore.get(sha1, { + timeout: BLOB_WAIT_TIMEOUT, + }); + if (buf == null) { + throw Error("Not available"); + } + let src; + if (leaveAsString != null) { + const t = new TextDecoder("utf8"); + const str = t.decode(buf); + if (leaveAsString(str)) { + src = str; + cache.set(sha1, src); + return src; + } + } + const blob = new Blob([buf], { type }); + src = URL.createObjectURL(blob); + cache.set(sha1, src); + return src; + }, + { createKey: (args: any[]) => args[0].sha1 }, +); export default function useBlob({ sha1, From ed4d8ff9e4183e1196f7332a6d51011c48471be7 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 16 Aug 2025 16:26:44 +0000 Subject: [PATCH 329/798] fix whiteboard with unsafe html... that said, it would be better to just turn it off since what is the point? --- .../whiteboard-editor/canvas.tsx | 399 +++++++++--------- src/packages/frontend/jupyter/cell-list.tsx | 2 +- 2 files changed, 205 insertions(+), 196 deletions(-) diff --git a/src/packages/frontend/frame-editors/whiteboard-editor/canvas.tsx b/src/packages/frontend/frame-editors/whiteboard-editor/canvas.tsx index 931b028eb6..3efbfb19e1 100644 --- a/src/packages/frontend/frame-editors/whiteboard-editor/canvas.tsx +++ b/src/packages/frontend/frame-editors/whiteboard-editor/canvas.tsx @@ -119,6 +119,7 @@ import { encodeForCopy, decodeForPaste } from "./tools/clipboard"; import { aspectRatioToNumber } from "./tools/frame"; import useIsMountedRef from "@cocalc/frontend/app-framework/is-mounted-hook"; import { extendToIncludeEdges } from "./actions"; +import { StableHtmlContext } from "@cocalc/frontend/jupyter/cell-list"; import Cursors from "./cursors"; @@ -1355,6 +1356,12 @@ export default function Canvas({ frame.actions.deleteElements(getOverlappingElements(elements, rect)); } }; + const scrollOrResize = useMemo(() => { + return {}; + }, []); + useEffect(() => { + Object.values(scrollOrResize).map((f: Function) => f()); + }, [canvasScale, offsetRef.current.left, offsetRef.current.top]); if (isNavigator && !isBoard) { return null; @@ -1372,221 +1379,223 @@ export default function Canvas({ // } return ( -
+
{ + mousePath.current = null; + if (isNavigator) { + if (ignoreNextClick.current) { + ignoreNextClick.current = false; + return; } - : undefined), - }} - onClick={(evt) => { - mousePath.current = null; - if (isNavigator) { - if (ignoreNextClick.current) { - ignoreNextClick.current = false; + frame.actions.setViewportCenter(frame.id, evtToData(evt)); return; } - frame.actions.setViewportCenter(frame.id, evtToData(evt)); - return; + if (!readOnly) { + handleClick(evt); + } + }} + onScroll={() => { + saveViewport(); + }} + onMouseDown={!isNavigator ? onMouseDown : undefined} + onMouseMove={!isNavigator ? onMouseMove : undefined} + onMouseUp={!isNavigator ? onMouseUp : undefined} + onTouchStart={!isNavigator ? onTouchStart : undefined} + onTouchMove={!isNavigator ? onTouchMove : undefined} + onTouchEnd={!isNavigator ? onTouchEnd : undefined} + onTouchCancel={!isNavigator ? onTouchCancel : undefined} + onPointerMove={!isNavigator ? onPointerMove : undefined} + onCopy={ + isNavigator + ? undefined + : (event: ClipboardEvent) => { + if (editFocus) return; + event.preventDefault(); + const selectedElements = getSelectedElements({ + elements, + selection, + }); + extendToIncludeEdges(selectedElements, elements); + const encoded = encodeForCopy(selectedElements); + event.clipboardData.setData( + "application/x-cocalc-whiteboard", + encoded, + ); + } } - if (!readOnly) { - handleClick(evt); + onCut={ + isNavigator || readOnly + ? undefined + : (event: ClipboardEvent) => { + if (editFocus) return; + event.preventDefault(); + const selectedElements = getSelectedElements({ + elements, + selection, + }); + extendToIncludeEdges(selectedElements, elements); + const encoded = encodeForCopy(selectedElements); + event.clipboardData.setData( + "application/x-cocalc-whiteboard", + encoded, + ); + frame.actions.deleteElements(selectedElements); + frame.actions.clearSelection(frame.id); + } } - }} - onScroll={() => { - saveViewport(); - }} - onMouseDown={!isNavigator ? onMouseDown : undefined} - onMouseMove={!isNavigator ? onMouseMove : undefined} - onMouseUp={!isNavigator ? onMouseUp : undefined} - onTouchStart={!isNavigator ? onTouchStart : undefined} - onTouchMove={!isNavigator ? onTouchMove : undefined} - onTouchEnd={!isNavigator ? onTouchEnd : undefined} - onTouchCancel={!isNavigator ? onTouchCancel : undefined} - onPointerMove={!isNavigator ? onPointerMove : undefined} - onCopy={ - isNavigator - ? undefined - : (event: ClipboardEvent) => { - if (editFocus) return; - event.preventDefault(); - const selectedElements = getSelectedElements({ - elements, - selection, - }); - extendToIncludeEdges(selectedElements, elements); - const encoded = encodeForCopy(selectedElements); - event.clipboardData.setData( - "application/x-cocalc-whiteboard", - encoded, - ); - } - } - onCut={ - isNavigator || readOnly - ? undefined - : (event: ClipboardEvent) => { - if (editFocus) return; - event.preventDefault(); - const selectedElements = getSelectedElements({ - elements, - selection, - }); - extendToIncludeEdges(selectedElements, elements); - const encoded = encodeForCopy(selectedElements); - event.clipboardData.setData( - "application/x-cocalc-whiteboard", - encoded, - ); - frame.actions.deleteElements(selectedElements); - frame.actions.clearSelection(frame.id); - } - } - onPaste={ - isNavigator || readOnly - ? undefined - : (event: ClipboardEvent) => { - if (editFocus) return; - const encoded = event.clipboardData.getData( - "application/x-cocalc-whiteboard", - ); - if (encoded) { - // copy/paste between whiteboards of their own structured data - const pastedElements = decodeForPaste(encoded); - /* TODO: should also get where mouse is? */ - let target: Point | undefined = undefined; - const pos = getMousePos(mousePosRef.current); - if (pos != null) { - const { x, y } = pos; - target = transformsRef.current.windowToDataNoScale(x, y); - } else { - const point = getCenterPositionWindow(); - if (point != null) { - target = windowToData(point); + onPaste={ + isNavigator || readOnly + ? undefined + : (event: ClipboardEvent) => { + if (editFocus) return; + const encoded = event.clipboardData.getData( + "application/x-cocalc-whiteboard", + ); + if (encoded) { + // copy/paste between whiteboards of their own structured data + const pastedElements = decodeForPaste(encoded); + /* TODO: should also get where mouse is? */ + let target: Point | undefined = undefined; + const pos = getMousePos(mousePosRef.current); + if (pos != null) { + const { x, y } = pos; + target = transformsRef.current.windowToDataNoScale(x, y); + } else { + const point = getCenterPositionWindow(); + if (point != null) { + target = windowToData(point); + } } - } - const ids = frame.actions.insertElements( - frame.id, - pastedElements, - target, - ); - frame.actions.setSelectionMulti(frame.id, ids); - } else { - // nothing else implemented yet! + const ids = frame.actions.insertElements( + frame.id, + pastedElements, + target, + ); + frame.actions.setSelectionMulti(frame.id, ids); + } else { + // nothing else implemented yet! + } } - } - } - > - {!isNavigator && selectedTool == "pen" && ( - - )} -
- {selectRect != null && ( -
-
-
+ /> )}
- {!isNavigator && mainFrameType == "whiteboard" && ( - - )} - {!isNavigator && mainFrameType == "slides" && ( - + {selectRect != null && ( +
+
+
)} - {renderedElements} +
+ {!isNavigator && mainFrameType == "whiteboard" && ( + + )} + {!isNavigator && mainFrameType == "slides" && ( + + )} + {renderedElements} +
-
+ ); } diff --git a/src/packages/frontend/jupyter/cell-list.tsx b/src/packages/frontend/jupyter/cell-list.tsx index 4702558137..0480bd1e35 100644 --- a/src/packages/frontend/jupyter/cell-list.tsx +++ b/src/packages/frontend/jupyter/cell-list.tsx @@ -40,7 +40,7 @@ interface StableHtmlContextType { cellListDivRef?: MutableRefObject; scrollOrResize?: { [key: string]: () => void }; } -const StableHtmlContext = createContext({}); +export const StableHtmlContext = createContext({}); export const useStableHtmlContext: () => StableHtmlContextType = () => { return useContext(StableHtmlContext); }; From d8be845edd1b52e4afcd6dea39de31c349e6ee0a Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 16 Aug 2025 16:53:16 +0000 Subject: [PATCH 330/798] disable "stable unsafe html" when needed --- .../account/editor-settings/checkboxes.tsx | 2 +- .../whiteboard-editor/canvas.tsx | 399 +++++++++--------- src/packages/frontend/jupyter/cell-list.tsx | 5 +- .../output-messages/stable-unsafe-html.tsx | 32 +- src/packages/util/db-schema/accounts.ts | 2 +- 5 files changed, 231 insertions(+), 209 deletions(-) diff --git a/src/packages/frontend/account/editor-settings/checkboxes.tsx b/src/packages/frontend/account/editor-settings/checkboxes.tsx index b40c611d89..421b8df674 100644 --- a/src/packages/frontend/account/editor-settings/checkboxes.tsx +++ b/src/packages/frontend/account/editor-settings/checkboxes.tsx @@ -87,7 +87,7 @@ const EDITOR_SETTINGS_CHECKBOXES = { disable_jupyter_virtualization: defineMessage({ id: "account.editor-setting.checkbox.disable_jupyter_virtualization", defaultMessage: - "render entire Jupyter Notebook instead of just visible part (slower and not recommended)", + "render entire Jupyter Notebook instead of just visible part (slower but more reliable)", }), } as const; diff --git a/src/packages/frontend/frame-editors/whiteboard-editor/canvas.tsx b/src/packages/frontend/frame-editors/whiteboard-editor/canvas.tsx index 3efbfb19e1..931b028eb6 100644 --- a/src/packages/frontend/frame-editors/whiteboard-editor/canvas.tsx +++ b/src/packages/frontend/frame-editors/whiteboard-editor/canvas.tsx @@ -119,7 +119,6 @@ import { encodeForCopy, decodeForPaste } from "./tools/clipboard"; import { aspectRatioToNumber } from "./tools/frame"; import useIsMountedRef from "@cocalc/frontend/app-framework/is-mounted-hook"; import { extendToIncludeEdges } from "./actions"; -import { StableHtmlContext } from "@cocalc/frontend/jupyter/cell-list"; import Cursors from "./cursors"; @@ -1356,12 +1355,6 @@ export default function Canvas({ frame.actions.deleteElements(getOverlappingElements(elements, rect)); } }; - const scrollOrResize = useMemo(() => { - return {}; - }, []); - useEffect(() => { - Object.values(scrollOrResize).map((f: Function) => f()); - }, [canvasScale, offsetRef.current.left, offsetRef.current.top]); if (isNavigator && !isBoard) { return null; @@ -1379,223 +1372,221 @@ export default function Canvas({ // } return ( - -
{ - mousePath.current = null; - if (isNavigator) { - if (ignoreNextClick.current) { - ignoreNextClick.current = false; - return; +
{ + mousePath.current = null; + if (isNavigator) { + if (ignoreNextClick.current) { + ignoreNextClick.current = false; return; } - if (!readOnly) { - handleClick(evt); - } - }} - onScroll={() => { - saveViewport(); - }} - onMouseDown={!isNavigator ? onMouseDown : undefined} - onMouseMove={!isNavigator ? onMouseMove : undefined} - onMouseUp={!isNavigator ? onMouseUp : undefined} - onTouchStart={!isNavigator ? onTouchStart : undefined} - onTouchMove={!isNavigator ? onTouchMove : undefined} - onTouchEnd={!isNavigator ? onTouchEnd : undefined} - onTouchCancel={!isNavigator ? onTouchCancel : undefined} - onPointerMove={!isNavigator ? onPointerMove : undefined} - onCopy={ - isNavigator - ? undefined - : (event: ClipboardEvent) => { - if (editFocus) return; - event.preventDefault(); - const selectedElements = getSelectedElements({ - elements, - selection, - }); - extendToIncludeEdges(selectedElements, elements); - const encoded = encodeForCopy(selectedElements); - event.clipboardData.setData( - "application/x-cocalc-whiteboard", - encoded, - ); - } + frame.actions.setViewportCenter(frame.id, evtToData(evt)); + return; } - onCut={ - isNavigator || readOnly - ? undefined - : (event: ClipboardEvent) => { - if (editFocus) return; - event.preventDefault(); - const selectedElements = getSelectedElements({ - elements, - selection, - }); - extendToIncludeEdges(selectedElements, elements); - const encoded = encodeForCopy(selectedElements); - event.clipboardData.setData( - "application/x-cocalc-whiteboard", - encoded, - ); - frame.actions.deleteElements(selectedElements); - frame.actions.clearSelection(frame.id); - } + if (!readOnly) { + handleClick(evt); } - onPaste={ - isNavigator || readOnly - ? undefined - : (event: ClipboardEvent) => { - if (editFocus) return; - const encoded = event.clipboardData.getData( - "application/x-cocalc-whiteboard", - ); - if (encoded) { - // copy/paste between whiteboards of their own structured data - const pastedElements = decodeForPaste(encoded); - /* TODO: should also get where mouse is? */ - let target: Point | undefined = undefined; - const pos = getMousePos(mousePosRef.current); - if (pos != null) { - const { x, y } = pos; - target = transformsRef.current.windowToDataNoScale(x, y); - } else { - const point = getCenterPositionWindow(); - if (point != null) { - target = windowToData(point); - } - } - - const ids = frame.actions.insertElements( - frame.id, - pastedElements, - target, - ); - frame.actions.setSelectionMulti(frame.id, ids); + }} + onScroll={() => { + saveViewport(); + }} + onMouseDown={!isNavigator ? onMouseDown : undefined} + onMouseMove={!isNavigator ? onMouseMove : undefined} + onMouseUp={!isNavigator ? onMouseUp : undefined} + onTouchStart={!isNavigator ? onTouchStart : undefined} + onTouchMove={!isNavigator ? onTouchMove : undefined} + onTouchEnd={!isNavigator ? onTouchEnd : undefined} + onTouchCancel={!isNavigator ? onTouchCancel : undefined} + onPointerMove={!isNavigator ? onPointerMove : undefined} + onCopy={ + isNavigator + ? undefined + : (event: ClipboardEvent) => { + if (editFocus) return; + event.preventDefault(); + const selectedElements = getSelectedElements({ + elements, + selection, + }); + extendToIncludeEdges(selectedElements, elements); + const encoded = encodeForCopy(selectedElements); + event.clipboardData.setData( + "application/x-cocalc-whiteboard", + encoded, + ); + } + } + onCut={ + isNavigator || readOnly + ? undefined + : (event: ClipboardEvent) => { + if (editFocus) return; + event.preventDefault(); + const selectedElements = getSelectedElements({ + elements, + selection, + }); + extendToIncludeEdges(selectedElements, elements); + const encoded = encodeForCopy(selectedElements); + event.clipboardData.setData( + "application/x-cocalc-whiteboard", + encoded, + ); + frame.actions.deleteElements(selectedElements); + frame.actions.clearSelection(frame.id); + } + } + onPaste={ + isNavigator || readOnly + ? undefined + : (event: ClipboardEvent) => { + if (editFocus) return; + const encoded = event.clipboardData.getData( + "application/x-cocalc-whiteboard", + ); + if (encoded) { + // copy/paste between whiteboards of their own structured data + const pastedElements = decodeForPaste(encoded); + /* TODO: should also get where mouse is? */ + let target: Point | undefined = undefined; + const pos = getMousePos(mousePosRef.current); + if (pos != null) { + const { x, y } = pos; + target = transformsRef.current.windowToDataNoScale(x, y); } else { - // nothing else implemented yet! + const point = getCenterPositionWindow(); + if (point != null) { + target = windowToData(point); + } } + + const ids = frame.actions.insertElements( + frame.id, + pastedElements, + target, + ); + frame.actions.setSelectionMulti(frame.id, ids); + } else { + // nothing else implemented yet! } - } - > - {!isNavigator && selectedTool == "pen" && ( - + {!isNavigator && selectedTool == "pen" && ( + + )} +
+ {selectRect != null && ( +
+ > +
+
)}
- {selectRect != null && ( -
-
-
+ {!isNavigator && mainFrameType == "whiteboard" && ( + )} -
- {!isNavigator && mainFrameType == "whiteboard" && ( - - )} - {!isNavigator && mainFrameType == "slides" && ( - - )} - {renderedElements} -
+ {!isNavigator && mainFrameType == "slides" && ( + + )} + {renderedElements}
- +
); } diff --git a/src/packages/frontend/jupyter/cell-list.tsx b/src/packages/frontend/jupyter/cell-list.tsx index 0480bd1e35..56e54390a7 100644 --- a/src/packages/frontend/jupyter/cell-list.tsx +++ b/src/packages/frontend/jupyter/cell-list.tsx @@ -37,6 +37,7 @@ import { Cell } from "./cell"; import HeadingTagComponent from "./heading-tag"; interface StableHtmlContextType { + enabled?: boolean; cellListDivRef?: MutableRefObject; scrollOrResize?: { [key: string]: () => void }; } @@ -577,7 +578,9 @@ export const CellList: React.FC = (props: CellListProps) => { if (use_windowed_list) { body = ( - +
+ ); + } else { + return ; + } +} + +export function DisabledStableUnsafeHtml({ html }) { + const elt = useMemo( + () =>
, + [html], + ); + return elt; +} + +export function EnabledStableUnsafeHtml({ + docId, + html, + zIndex = Z_INDEX, // todo: support changing? + stableHtmlContext, +}) { const divRef = useRef(null); const cellOutputDivRef = useRef(null); const intervalRef = useRef(null); const { isVisible, project_id, path, id } = useFrameContext(); - const stableHtmlContext = useStableHtmlContext(); const htmlRef = useRef(html); const globalKeyRef = useRef( sha1(`${project_id}-${id}-${docId}-${path}-${html}`), diff --git a/src/packages/util/db-schema/accounts.ts b/src/packages/util/db-schema/accounts.ts index 27d8ad9a2d..e6f4ebf85a 100644 --- a/src/packages/util/db-schema/accounts.ts +++ b/src/packages/util/db-schema/accounts.ts @@ -477,7 +477,7 @@ Table({ keyboard_variant: "", ask_jupyter_kernel: true, show_my_other_cursors: false, - disable_jupyter_virtualization: false, + disable_jupyter_virtualization: true, }, other_settings: { katex: true, From 7db487334531b97964db4528ffbe10f19c50ec12 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 16 Aug 2025 18:54:57 +0000 Subject: [PATCH 331/798] fix subtle mutation issue involving export to ipynb --- .../frontend/jupyter/browser-actions.ts | 17 +++---- src/packages/jupyter/ipynb/export-to-ipynb.ts | 49 ++++++++----------- src/packages/jupyter/util/misc.ts | 4 +- src/packages/util/db-schema/accounts.ts | 2 +- 4 files changed, 32 insertions(+), 40 deletions(-) diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index f33e4d2b1f..a4fc9f688f 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -1288,19 +1288,17 @@ export class JupyterActions extends JupyterActions0 { } } - const blobsBase64: { [sha1: string]: string | null } = {}; - const blobsString: { [sha1: string]: string | null } = {}; + const blobsBase64 = new Set(); + const blobsString = new Set(); const blob_store = { getBase64: (hash) => { - blobsBase64[hash] = null; + blobsBase64.add(hash); }, getString: (hash) => { - blobsString[hash] = null; + blobsString.add(hash); }, }; - // export_to_ipynb mutates its input... mostly not a problem, since - // we're toJS'ing most of it, but be careful with more_output. const options = { cells: store.get("cells").toJS(), cell_list: cell_list.toJS(), @@ -1311,11 +1309,12 @@ export class JupyterActions extends JupyterActions0 { more_output: cloneDeep(more_output), }; - const pass1 = export_to_ipynb(options); + // clone deep because export_to_ipynb mutates its input! + const pass1 = export_to_ipynb(cloneDeep(options)); let n = 0; const blobs: { [sha1: string]: string | null } = {}; - for (const hash in blobsBase64) { + for (const hash of blobsBase64) { try { const ar = await this.asyncBlobStore.get(hash); if (ar) { @@ -1327,7 +1326,7 @@ export class JupyterActions extends JupyterActions0 { } } const t = new TextDecoder(); - for (const hash in blobsString) { + for (const hash of blobsString) { try { const ar = await this.asyncBlobStore.get(hash); if (ar) { diff --git a/src/packages/jupyter/ipynb/export-to-ipynb.ts b/src/packages/jupyter/ipynb/export-to-ipynb.ts index 21867a679c..4f4066a282 100644 --- a/src/packages/jupyter/ipynb/export-to-ipynb.ts +++ b/src/packages/jupyter/ipynb/export-to-ipynb.ts @@ -279,39 +279,32 @@ function processOutputN( if (output_n.text != null) { output_n.text = diff_friendly(output_n.text); } - if (output_n.data != null) { + if (output_n.data != null && blob_store != null) { for (let k in output_n.data) { + const isShaType = + k.startsWith("image/") || k == "application/pdf" || k == "iframe"; const v = output_n.data[k]; - if (k.slice(0, 5) === "text/") { - output_n.data[k] = diff_friendly(output_n.data[k]); - } - if ( - isSha1(v) && - (k.startsWith("image/") || k === "application/pdf" || k === "iframe") - ) { - if (blob_store != null) { - let value; - if (isJupyterBase64MimeType(k)) { - value = blob_store.getBase64(v); - } else { - value = blob_store.getString(v); - value = value?.split("\n").map((x) => x + "\n"); - if (k === "iframe") { - delete output_n.data[k]; - k = "text/html"; - } - } - if (value == null) { - // The image is no longer known; this could happen if the user reverts in the history - // browser and there is an image in the output that was not saved in the latest version. - // TODO: instead return an error. - return; - } - output_n.data[k] = value; + + if (isShaType && isSha1(v)) { + // value was replaced by sha1 version of it so swap back the original + // content + const value = isJupyterBase64MimeType(k) + ? blob_store.getBase64(v) + : blob_store.getString(v); + if (value == null) { + delete output_n.data[k]; + continue; } else { - return; // impossible to include in the output without blob_store + output_n[k] = value; + if (k == "iframe") { + output_n["text/html"] = value; + delete output_n["iframe"]; + } } } + if (k.slice(0, 5) === "text/") { + output_n.data[k] = diff_friendly(output_n.data[k]); + } } output_n.output_type = "execute_result"; if (output_n.metadata == null) { diff --git a/src/packages/jupyter/util/misc.ts b/src/packages/jupyter/util/misc.ts index cff641b391..7dbe854764 100644 --- a/src/packages/jupyter/util/misc.ts +++ b/src/packages/jupyter/util/misc.ts @@ -40,7 +40,7 @@ export const JUPYTER_MIMETYPES_SET = new Set(JUPYTER_MIMETYPES); export function isJupyterBase64MimeType(type: string) { type = type.toLowerCase(); - if (type.startsWith("text")) { + if (type.startsWith("text") || type == "iframe") { // no text ones are base64 encoded return false; } @@ -50,7 +50,7 @@ export function isJupyterBase64MimeType(type: string) { if (type == "image/svg+xml") { return false; } - + // what remains should be application/pdf and the image types return true; } diff --git a/src/packages/util/db-schema/accounts.ts b/src/packages/util/db-schema/accounts.ts index 6388fbe04b..e6f4ebf85a 100644 --- a/src/packages/util/db-schema/accounts.ts +++ b/src/packages/util/db-schema/accounts.ts @@ -471,7 +471,7 @@ Table({ undo_depth: 300, jupyter_classic: false, jupyter_window: false, - disable_jupyter_windowing: true, + disable_jupyter_windowing: false, show_exec_warning: true, physical_keyboard: "default", keyboard_variant: "", From a9b6e3b298c48d8a1410df01945a0bfb8edbbeeb Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 16 Aug 2025 19:41:36 +0000 Subject: [PATCH 332/798] make math for sagemath appear --- src/packages/jupyter/redux/actions.ts | 4 ++++ src/packages/jupyter/util/iframe.ts | 17 +++-------------- src/packages/jupyter/util/misc.ts | 2 +- 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index 89688f262b..781cd2d689 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -51,6 +51,7 @@ import { } from "@cocalc/jupyter/util/misc"; import { remove_redundant_reps } from "@cocalc/jupyter/ipynb/import-from-ipynb"; import { isSha1, sha1 } from "@cocalc/util/misc"; +import { shouldUseIframe } from "@cocalc/jupyter/util/iframe"; const { close, required, defaults } = misc; @@ -2171,6 +2172,9 @@ export class JupyterActions extends Actions { // processing output closes and we cutoff to the project processing output. continue; } + if (type == "text/html" && !shouldUseIframe(content.data[type])) { + continue; + } const s = saveBlob(content.data[type], type); if (s) { dbg("put content in blob store: ", { type, sha1: s }); diff --git a/src/packages/jupyter/util/iframe.ts b/src/packages/jupyter/util/iframe.ts index 46d71e31d9..3831e77097 100644 --- a/src/packages/jupyter/util/iframe.ts +++ b/src/packages/jupyter/util/iframe.ts @@ -11,17 +11,9 @@ MOTIVATION: Sage 3d graphics. import { decode } from "he"; -// use iframe for anything at all large (reduce strain on ) -const MAX_HTML_SIZE = 10000; +// use iframe for really large html (reduce strain on sync) +const MAX_HTML_SIZE = 1_000_000; -// We use iframes to render html in a number of cases: -// - if it starts with iframe -// - if it has a whole page doctype -// - if it has a - - - - -
-

#{data.title}

- - - - - - - -
Author#{data.author}
Date#{data.timestamp}
Project#{data.project_id}
Location#{data.filename}
Original file#{data.basename}
-
- #{data.content} - - - """ - return @_html_tmpl - - html_process_output_mesg: (mesg, mark) -> - # Note: each mesg could contain more than one output type - out = '' - # if DEBUG console.log 'html_process_output_mesg', mesg, mark - if mesg.stdout?.length > 0 - # assertion: for stdout, `mark` might be undefined - out += "
#{mesg.stdout}
" - if mesg.stderr?.length > 0 - out += "
#{mesg.stderr}
" - if mesg.html? - $html = $("
#{mesg.html}
") - @editor.syncdoc.process_html_output($html) - out += "
#{$html.html()}
" - if mesg.md? - s = markdown.markdown_to_html(mesg.md) - $out = $("
") - $out.html_noscript(s) # also, don't process mathjax! - @editor.syncdoc.process_html_output($out) - out += "
#{$out.html()}
" - if mesg.interact? - out += "
#{mark.widgetNode.innerHTML}
" - if mesg.file? - if mesg.file.show ? true - ext = misc.filename_extension(mesg.file.filename).toLowerCase() - if ext == 'sage3d' - for el in $(mark.replacedWith).find(".webapp-3d-container") - $3d = $(el) - scene = $3d.data('webapp-threejs') - if not scene? - # when the document isn't fully processed, there is no scene data - continue - scene.set_static_renderer() - data_url = scene.static_image - out ?= '' - out += "
" - else if ext == 'webm' - # 'raw' url. later, embed_videos will be replace by the data-uri if there is no error - out += "" - else - # if DEBUG then console.log 'msg.file', mark, mesg - if not @_output_ids[mark.id] # avoid duplicated outputs - @_output_ids[mark.id] = true - # if DEBUG then console.log "output.file", mark, mesg - $images = $(mark.widgetNode) - for el in $images.find('.sagews-output-image') - # innerHTML should just be the element - out += el.innerHTML - out += "
#{out}
" - if mesg.code? # what's that actually? - code = mesg.code.source - out += "
#{code}
" - if mesg.javascript? - # mesg.javascript.coffeescript is true iff coffeescript - $output = $(mark.replacedWith) - $output.find('.sagews-output-container').remove() # TODO what's that? - out += "
#{$output.html()}
" - if mesg.done? - # ignored - out += ' ' - - return @html_post_process(out) - - html_post_process: (html) -> - # embedding images and detecting a title - if not html? - return html - $html = $('
').html(html) - if not @_title - for tag in ['h1', 'h2', 'h3'] - $hx = $html.find(tag + ':first') - if $hx.length > 0 - @_title = $hx.text() - break - for img in $html.find('img') - if img.src.startsWith('data:') - continue - c = document.createElement("canvas") - scaling = img.getAttribute('smc-image-scaling') ? 1 - c.width = img.width - c.height = img.height - c.getContext('2d').drawImage(img, 0, 0) - img.width = scaling * img.width - img.height = scaling * img.height - ext = misc.filename_extension(img.src).toLowerCase() - ext = ext.split('?')[0] - ext = if ext == 'jpg' then 'jpeg' else ext - if ext == 'svg' - ext = 'svg+xml' - else if ext in ['png', 'jpeg'] - _ - else - console.warn("printing sagews2html image file extension of '#{img.src}' not supported") - continue - try - img.src = c.toDataURL("image/#{ext}") - catch e - # ignore a potential CORS security error, when the image comes from another domain. - # SecurityError: Failed to execute 'toDataURL' on 'HTMLCanvasElement': Tainted canvases may not be exported. - console.info('ignoring CORS error regarding reading the image content via "toDataURL"') - continue - return $html.html() - - html: (cb, progress) -> - # the following fits mentally into sagews.SynchronizedWorksheet - # progress takes two arguments: a float between 0 and 1 [%] and optionally a message - {MARKERS} = require('@cocalc/util/sagews') - _html = [] # list of elements - full_html = '' # end result of html content - @_title = null # for saving the detected title - @_output_ids = {} # identifies text marker elements, to avoid printing show-plots them more than once! - cm = @editor.codemirror - progress ?= _.noop - - # canonical modes in a sagews - {sagews_decorator_modes} = require('./editor') - canonical_modes = _.object(sagews_decorator_modes) - - # cell input lines are collected first and processed once lines with markers appear (i.e. output) - # the assumption is, that default_mode extends to all the consecutive cells until the next mode or default_mode - input_lines = [] - input_lines_mode = null - input_lines_default_mode = 'python' - - canonical_mode = (mode) -> - canonical_modes[mode] ? input_lines_default_mode - - detect_mode = (line) -> - line = line.trim() - if line.startsWith('%') # could be %auto, %md, %auto %default_mode, ... - i = line.indexOf('%default_mode') - if i >= 0 - input_lines_default_mode = canonical_mode(line[i..].split(/\s+/)[1]) - else - mode = line.split(" ")[0][1..] # worst case, this is an empty string - if _.has(canonical_modes, mode) - input_lines_mode = canonical_mode(mode) - - process_line = (line) -> - detect_mode(line) - # each line is in because of the css line numbering - code = document.createElement('code') - mode = input_lines_mode ? input_lines_default_mode - CodeMirror.runMode(line, mode, code) - return code.outerHTML - - input_lines_process = (final = false) => - # final: if true, filter out the empty lines at the bottom - while final and input_lines.length > 0 - line = input_lines[input_lines.length - 1] - if line.length == 0 - input_lines.pop() - else - break - if input_lines.length > 0 - input_lines = input_lines.map(process_line).join('') # no \n linebreaks! - #_html.push("
#{input_lines_mode ? input_lines_default_mode} mode") - _html.push("
#{input_lines}
") - input_lines = [] - input_lines_mode = null - - process_lines = (cb) => - line = 0 - lines_total = cm.lineCount() - while line < lines_total - progress(.1 + .8 * line / lines_total, "Converting line #{line}") - x = cm.getLine(line) - marks = cm.findMarks({line:line, ch:0}, {line:line, ch:x.length}) - if not marks? or marks.length == 0 - input_lines.push(x) - else - input_lines_process() - mark = marks[0] # assumption it's always length 1 - switch x[0] # first char is the marker - when MARKERS.cell - _ - when MARKERS.output - # assume, all cells are evaluated and hence mark.rendered contains the html - output_msgs = [] - for mesg_ser in mark.rendered.split(MARKERS.output) - if mesg_ser.length == 0 - continue - try - mesg = misc.from_json(mesg_ser) - catch e - console.warn("invalid output message '#{mesg_ser}' in line '#{line}'") - continue - - #if DEBUG then console.log 'sagews2html: output message', mesg - if mesg.clear - output_msgs = [] - else if mesg.delete_last - output_msgs.pop() - else - output_msgs.push(mesg) - # after for-loop over rendered output cells - for mesg in output_msgs - om = @html_process_output_mesg(mesg, mark) - _html.push(om) if om? - - line++ - input_lines_process(final = true) - # combine all html snippets to one html block - full_html = (h for h in _html).join('\n') - cb() - - sagews_data = (cb) => - dl_url = webapp_client.project_client.read_file - project_id : @editor.project_id - path : @editor.filename - - data_base64 = null - f = (cb) -> - $.get(dl_url).done((data) -> - # console.log "data", data - data_enc = window.btoa(window.unescape(encodeURIComponent(data))) - data_base64 = 'data:application/octet-stream;base64,' + data_enc - cb(null) - ).fail(-> cb(true)) - - misc.retry_until_success - f : f - max_time : 60*1000 - cb : (err) -> - cb(err, data_base64) - - $html = null - embed_videos = (cb) => - # downloading and embedding all video files (especially for animations) - # full_html is a string and we have to wrap this into a div - $html = $('
' + full_html + '
') - vids = (vid for vid in $html.find('video')) - vids_num = 0 - vids_tot = vids.length - vembed = (vid, cb) -> - # if DEBUG then console.log "embedding #{vids_num}/#{vids_tot}", vid - vids_num += 1 - progress(.4 + (5 / 10) * (vids_num / vids_tot), "video #{vids_num}/#{vids_tot}") - xhr = new XMLHttpRequest() - xhr.open('GET', vid.src) - xhr.responseType = 'blob' - xhr.onreadystatechange = -> - if this.readyState == 4 # it's DONE - if this.status == 200 # all OK - blob = this.response - reader = new FileReader() - reader.addEventListener "load", -> - # if DEBUG then console.log(reader.result[..100]) - vid.src = reader.result - cb(null) - reader.readAsDataURL(blob) - else - # error handling - cb("Error embedding video: HTTP status #{this.status}") - xhr.send() - async.mapLimit vids, 2, vembed, (err, results) -> - full_html = $html.html() - cb(err) - - finalize = (err, results) => - data = results[0] - if err - cb?(err) - return - if not data? - cb?('Unable to download and serialize the Sage Worksheet.') - return - - file_url = url_fullpath(@editor.project_id, @editor.filename) - content = @generate_html - title : @_title ? @editor.filename - filename : @editor.filename - content : full_html - timestamp : "#{(new Date()).toISOString()}".split('.')[0] - project_id : @editor.project_id - author : redux.getStore('account').get_fullname() - file_url : file_url - basename : misc.path_split(@editor.filename).tail - sagews_data : data - - progress(.95, "Saving to #{@output_file} ...") - try - await webapp_client.project_client.write_text_file - project_id : @editor.project_id - path : @output_file - content : content - console.debug("write_text_file") - cb?() - catch err - cb?(err) - - # parallel is tempting, but videos depend on process lines - async.series([sagews_data, process_lines, embed_videos], finalize) - -# registering printers -printers = {} -for printer_cls in [PandocPrinter, LatexPrinter, SagewsPrinter] - for ext in printer_cls.supported - printers[ext] = printer_cls - -### -# Public API -# Printer, usually used like that: -# p = Printer(@, input_file, output_file, opts) -# p.print(cb) -# -# can_print(ext) → true or false -### - -# returns the printer class for a given file extension -exports.Printer = (editor, output_file, opts) -> - ext = misc.filename_extension_notilde(editor.filename).toLowerCase() - return new printers[ext](editor, output_file, opts) - -# returns true, if we know how to print it -exports.can_print = (ext) -> - return _.has(printers, ext) diff --git a/src/packages/frontend/project/explorer/new-button.tsx b/src/packages/frontend/project/explorer/new-button.tsx index a9e37355c7..12152e0174 100644 --- a/src/packages/frontend/project/explorer/new-button.tsx +++ b/src/packages/frontend/project/explorer/new-button.tsx @@ -5,14 +5,12 @@ import { Button, type MenuProps, Space } from "antd"; import { useIntl } from "react-intl"; - import { DropdownMenu, Icon } from "@cocalc/frontend/components"; import { labels } from "@cocalc/frontend/i18n"; import { ProjectActions } from "@cocalc/frontend/project_store"; import { COLORS } from "@cocalc/util/theme"; import { EXTs as ALL_FILE_BUTTON_TYPES } from "./file-listing/utils"; - -const { file_options } = require("@cocalc/frontend/editor"); +import { file_options } from "@cocalc/frontend/editor-tmp"; interface Props { file_search: string; diff --git a/src/packages/frontend/project/new/new-file-dropdown.tsx b/src/packages/frontend/project/new/new-file-dropdown.tsx index 23eee1497b..a7fa6c46f0 100644 --- a/src/packages/frontend/project/new/new-file-dropdown.tsx +++ b/src/packages/frontend/project/new/new-file-dropdown.tsx @@ -16,9 +16,7 @@ import { file_associations } from "@cocalc/frontend/file-associations"; import { EXTs } from "@cocalc/frontend/project/explorer/file-listing/utils"; import { keys } from "@cocalc/util/misc"; import { COLORS } from "@cocalc/util/theme"; - -// eslint-disable-next-line @typescript-eslint/no-var-requires -const { file_options } = require("@cocalc/frontend/editor"); +import { file_options } from "@cocalc/frontend/editor-tmp"; interface Props { create_file: (ext?: string) => void; diff --git a/src/packages/frontend/project/page/file-tab.tsx b/src/packages/frontend/project/page/file-tab.tsx index 20eaaebdf3..21d342c60f 100644 --- a/src/packages/frontend/project/page/file-tab.tsx +++ b/src/packages/frontend/project/page/file-tab.tsx @@ -51,8 +51,7 @@ import { ActiveFlyout } from "./flyouts/active"; import { shouldOpenFileInNewWindow } from "./utils"; import { getValidActivityBarOption } from "./activity-bar"; import { ACTIVITY_BAR_KEY } from "./activity-bar-consts"; - -const { file_options } = require("@cocalc/frontend/editor"); +import { file_options } from "@cocalc/frontend/editor-tmp"; export type FixedTab = | "active" @@ -163,7 +162,7 @@ export const FIXED_PROJECT_TABS: FixedTabs = { icon: "microchip", flyout: ProjectInfoFlyout, noAnonymous: false, - noLite: true, // process monitor doesn't work at all yet for some reason + noLite: true, // process monitor doesn't work at all yet for some reason }, settings: { label: labels.settings, diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index edcc8c3f29..097633c599 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -101,7 +101,6 @@ import * as misc from "@cocalc/util/misc"; import { reduxNameToProjectId } from "@cocalc/util/redux/name"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { client_db } from "@cocalc/util/schema"; -import { get_editor } from "./editors/react-wrapper"; import { type FilesystemClient } from "@cocalc/conat/files/fs"; import { getCacheId, @@ -1220,40 +1219,13 @@ export class ProjectActions extends Actions { // moves to the given line. Otherwise, does nothing. public goto_line(path, line, cursor?: boolean, focus?: boolean): void { const actions: any = redux.getEditorActions(this.project_id, path); - if (actions == null) { - // try non-react editor - const editor = get_editor(this.project_id, path); - if ( - editor != null && - typeof editor.programmatical_goto_line === "function" - ) { - editor.programmatical_goto_line(line); - // TODO: For an old non-react editor (basically just sage worksheets at this point!) - // we have to just use this flaky hack, since we are going to toss all this - // code soon. This is needed since if editor is just *loading*, should wait until it - // finishes before actually jumping to line, but that's not implemented in editor.coffee. - setTimeout(() => { - editor.programmatical_goto_line(line); - }, 1000); - setTimeout(() => { - editor.programmatical_goto_line(line); - }, 2000); - } - } else if (actions.programmatical_goto_line != null) { - actions.programmatical_goto_line(line, cursor, focus); - } + actions?.programmatical_goto_line?.(line, cursor, focus); } // Called when a file tab is shown. private show_file(path): void { const a: any = redux.getEditorActions(this.project_id, path); - if (a == null) { - // try non-react editor - const editor = get_editor(this.project_id, path); - if (editor != null) editor.show(); - } else { - a.show?.(); - } + a?.show?.(); const fragmentId = this.open_files?.get(path, "fragmentId"); if (fragmentId) { // have to wait for next render so that local store is updated and @@ -1268,12 +1240,8 @@ export class ProjectActions extends Actions { // another tab being made active. private hide_file(path): void { const a: any = redux.getEditorActions(this.project_id, path); - if (a == null) { - // try non-react editor - const editor = get_editor(this.project_id, path); - if (editor != null) editor.hide(); - } else { - if (typeof a.hide === "function") a.hide(); + if (typeof a?.hide === "function") { + a.hide(); } } diff --git a/src/packages/frontend/project_store.ts b/src/packages/frontend/project_store.ts index a93847bb9f..0aefbf830e 100644 --- a/src/packages/frontend/project_store.ts +++ b/src/packages/frontend/project_store.ts @@ -3,18 +3,6 @@ * License: MS-RSL – see LICENSE.md for details */ -let wrapped_editors; - -// TODO: we should refactor our code to now have these window/document references -// in *this* file. This very code (all the redux/store stuff) is used via node.js -// in projects, so should not reference window or document. - -declare let window, document; -if (typeof window !== "undefined" && window !== null) { - // don't import in case not in browser (for testing) - wrapped_editors = require("./editors/react-wrapper"); -} - import * as immutable from "immutable"; import { AppRedux, @@ -59,6 +47,7 @@ import { import { type PublicPath } from "@cocalc/util/db-schema/public-paths"; import { DirectoryListing } from "@cocalc/frontend/project/explorer/types"; export { FILE_ACTIONS as file_actions, type FileAction, ProjectActions }; +import { SCHEMA, client_db } from "@cocalc/util/schema"; export type ModalInfo = TypedMap<{ title: string | React.JSX.Element; @@ -350,9 +339,7 @@ export class ProjectStore extends Store { fn: () => { const project_id = this.project_id; return function (path) { - // (this exists because rethinkdb doesn't have compound primary keys) - const { SCHEMA, client_db } = require("@cocalc/util/schema"); - return SCHEMA.public_paths.user_query.set.fields.id( + return SCHEMA.public_paths.user_query?.set?.fields.id( { project_id, path }, client_db, ); @@ -366,17 +353,7 @@ export class ProjectStore extends Store { // is in react and has store with a cursors key. get_users_cursors = (path, account_id) => { const store: any = redux.getEditorStore(this.project_id, path); - if (store == null) { - // try non-react editor - const editors = wrapped_editors.get_editor(this.project_id, path); - if (editors && editors.get_users_cursors) { - return editors.get_users_cursors(account_id); - } else { - return undefined; - } - } else { - return store.get("cursors") && store.get("cursors").get(account_id); - } + return store?.get("cursors") && store.get("cursors").get(account_id); }; is_file_open = (path) => { diff --git a/src/packages/frontend/sagews/3d.coffee b/src/packages/frontend/sagews/3d.coffee deleted file mode 100644 index 2113402d77..0000000000 --- a/src/packages/frontend/sagews/3d.coffee +++ /dev/null @@ -1,964 +0,0 @@ -######################################################################### -# This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. -# License: MS-RSL – see LICENSE.md for details -######################################################################### - -$ = window.$ -async = require('async') - -misc = require('@cocalc/util/misc') -{defaults, required} = misc - -component_to_hex = (c) -> - hex = c.toString(16) - if hex.length == 1 - return "0" + hex - else - return hex - -rgb_to_hex = (r, g, b) -> - # Unfortunately, some code thinks the range is 0 to 255, and other (sage) - # thinks it is 0 to 1. So we use a *heuristic*, which is horrible, but - # I don't know an alternative, and this is LIKELY to be ok in practice. - if r <= 1 and g <= 1 and b <= 1 - r = Math.round(r*255) - g = Math.round(g*255) - b = Math.round(b*255) - return "#" + component_to_hex(r) + component_to_hex(g) + component_to_hex(b) - -_loading_threejs_callbacks = [] - -VERSION = '73' - -window.THREE = require("three-ancient") - -require("../node_modules/three-ancient/examples/js/controls/OrbitControls") -require("../node_modules/three-ancient/examples/js/renderers/CanvasRenderer") -require("../node_modules/three-ancient/examples/js/renderers/Projector") -Detector = require("../node_modules/three-ancient/examples/js/Detector") - -_scene_using_renderer = undefined -_renderer = {webgl:undefined, canvas:undefined} -dynamic_renderer_type = undefined - -get_renderer = (scene, type) -> - # if there is a scene currently using this renderer, tell it to switch to - # the static renderer. - if _scene_using_renderer? and _scene_using_renderer._id != scene._id - _scene_using_renderer.set_static_renderer() - - # now scene takes over using this renderer - _scene_using_renderer = scene - if Detector.webgl and (not type? or type == 'webgl') - type = 'webgl' - else - type = 'canvas' - dynamic_renderer_type = type - if not _renderer[type]? - # get the best-possible THREE.js renderer (once and for all) - if type == 'webgl' - _renderer[type] = new THREE.WebGLRenderer - antialias : true - alpha : true - preserveDrawingBuffer : true - else - _renderer[type] = new THREE.CanvasRenderer - antialias : true - alpha : true - $(_renderer[type].domElement).addClass("webapp-3d-dynamic-renderer") - return _renderer[type] - -MIN_WIDTH = MIN_HEIGHT = 16 - -class WebappThreeJS - constructor: (opts) -> - @opts = defaults opts, - element : required - container : required - width : undefined - height : undefined - renderer : undefined # 'webgl' or 'canvas' or undefined to choose best - background : "transparent" - foreground : undefined - spin : false # if true, image spins by itself when mouse is over it. - camera_distance : 10 - aspect_ratio : undefined # undefined does nothing or a triple [x,y,z] of length three, - # which scales the x,y,z coordinates of everything by the given values. - stop_when_gone : undefined # if given, animation, etc., stops when this html element (not jquery!) is no longer in the DOM - frame : undefined # if given call set_frame with opts.frame as input when init_done called - cb : undefined # opts.cb(undefined, this object) - - if misc.is_array(@opts.background) - @opts.background = rgb_to_hex(@opts.background[0], @opts.background[1], @opts.background[2]) - - @init_eval_note() - opts.cb?(undefined, @) - # window.w = @ # for debugging - - # client code should call this when start adding objects to the scene - init: () => - if @_init - return - @_init = true - - @_id = misc.uuid() - @init_aspect_ratio_functions() - - @scene = new THREE.Scene() - - # IMPORTANT: There is a major bug in three.js -- if you make the width below more than .5 of the window - # width, then after 8 3d renders, things get foobared in WebGL mode. This happens even with the simplest - # demo using the basic cube example from their site with R68. It even sometimes happens with this workaround, but - # at least retrying a few times can fix it. - if not @opts.width? or @opts.width < MIN_WIDTH - # ignore width/height less than a cutoff -- some graphics, - # e.g., "Polyhedron([(0,0,0),(0,1,0),(0,2,1),(1,0,0),(1,2,3),(2,1,1)]).plot()" - # weirdly set it very small. - @opts.width = $(window).width()*.5 - - @opts.height = if @opts.height? and @opts.height >= MIN_HEIGHT then @opts.height else @opts.width*2/3 - @opts.container.css(width:"#{@opts.width+50}px") - - @set_dynamic_renderer() - @init_orbit_controls() - @init_on_mouseover() - - # add a bunch of lights - @init_light() - - # set background color - @opts.element.find(".webapp-3d-canvas").css('background':@opts.background) - - if not @opts.foreground? - if @opts.background == 'transparent' - @opts.foreground = 'gray' - else - c = @opts.element.find(".webapp-3d-canvas").css('background') - if not c? or c.indexOf(')') == -1 - @opts.foreground = "#000" # e.g., on firefox - this is best we can do for now - else - i = c.indexOf(')') - z = [] - for a in c.slice(4,i).split(',') - b = parseInt(a) - if b < 128 - z.push(255) - else - z.push(0) - @opts.foreground = rgb_to_hex(z[0], z[1], z[2]) - - # client code should call this when done adding objects to the scene - init_done: () => - if @opts.frame? - @set_frame(@opts.frame) - - if @renderer_type != 'dynamic' - # if we don't have the renderer, swap it in, make a static image, then give it back to whoever had it. - owner = _scene_using_renderer - @set_dynamic_renderer() - @set_static_renderer() - owner?.set_dynamic_renderer() - - # possibly show the canvas warning. - if dynamic_renderer_type == 'canvas' - @opts.element.find(".webapp-3d-canvas-warning").show().tooltip() - - # show an "eval note" if we don't load the scene within a second. - init_eval_note: () => - f = () => - if not @_init - @opts.element.find(".webapp-3d-note").show() - setTimeout(f, 1000) - - set_dynamic_renderer: () => - # console.log "dynamic renderer" - if @renderer_type == 'dynamic' - # already have it - return - @renderer = get_renderer(@, @opts.renderer) - @renderer_type = 'dynamic' - # place renderer in correct place in the DOM - @opts.element.find(".webapp-3d-canvas").empty().append($(@renderer.domElement)) - if @opts.background == 'transparent' - @renderer.setClearColor(0x000000, 0) - else - @renderer.setClearColor(@opts.background, 1) - @renderer.setSize(@opts.width, @opts.height) - if @controls? - @controls.enabled = true - if @last_canvas_pos? - @controls.object.position.copy(@last_canvas_pos) - if @last_canvas_target? - @controls.target.copy(@last_canvas_target) - if @opts.spin - @animate(render:false) - @render_scene(true) - - set_static_renderer: () => - # console.log "static renderer" - if @renderer_type == 'static' - # already have it - return - @static_image = @data_url() - @renderer_type = 'static' - if @controls? - @controls.enabled = false - @last_canvas_pos = @controls.object.position - @last_canvas_target = @controls.target - img = $("").attr(src:@static_image).width(@opts.width).height(@opts.height) - @opts.element.find(".webapp-3d-canvas").empty().append(img) - - # On mouseover, we switch the renderer out to use webgl, if available, and also enable spin animation. - init_on_mouseover: () => - @opts.element.mouseenter () => - @set_dynamic_renderer() - - @opts.element.mouseleave () => - @set_static_renderer() - - @opts.element.click () => - @set_dynamic_renderer() - - # initialize functions to create new vectors, which take into account the scene's 3d frame aspect ratio, - # and also the change of coordinates from THREE.js coords to "math coordinates". - init_aspect_ratio_functions: () => - if @opts.aspect_ratio? - x = @opts.aspect_ratio[0]; y = @opts.aspect_ratio[1]; z = @opts.aspect_ratio[2] - @vector3 = (a,b,c) => new THREE.Vector3( -y*b , x*a , z*c ) - @vector = (v) => new THREE.Vector3( -y*v[1], x*v[0], z*v[2] ) - @aspect_ratio_scale = (v) => [ -y*v[1], x*v[0], z*v[2] ] - else - @vector3 = (a,b,c) => new THREE.Vector3( -b , a, c ) - @vector = (v) => new THREE.Vector3( -v[1], v[0], v[2]) - @aspect_ratio_scale = (v) => [ -v[1], v[0], v[2]] - - show_canvas: () => - @init() - @opts.element.find(".webapp-3d-note").hide() - @opts.element.find(".webapp-3d-canvas").show() - - data_url: (opts) => - opts = defaults opts, - type : 'png' # 'png' or 'jpeg' or 'webp' (the best) - quality : undefined # 1 is best quality; 0 is worst; only applies for jpeg or webp - s = @renderer.domElement.toDataURL("image/#{opts.type}", opts.quality) - # console.log("took #{misc.to_json(opts)} snapshot (length=#{s.length})") - return s - - init_orbit_controls: () => - if not @camera? - @add_camera(distance:@opts.camera_distance) - - # console.log 'set_orbit_controls' - # set up camera controls - @controls = new THREE.OrbitControls(@camera, @renderer.domElement) - @controls.damping = 2 - @controls.enableKeys = false # see https://github.com/mrdoob/three.js/blob/master/examples/js/controls/OrbitControls.js#L962 - @controls.zoomSpeed = 0.4 - if @_center? - @controls.target = @_center - if @opts.spin - if typeof(@opts.spin) == "boolean" - @controls.autoRotateSpeed = 2.0 - else - @controls.autoRotateSpeed = @opts.spin - @controls.autoRotate = true - - @controls.addEventListener 'change', () => - if @renderer_type=='dynamic' - @rescale_objects() - @renderer.render(@scene, @camera) - - add_camera: (opts) => - opts = defaults opts, - distance : 10 - - if @camera? - return - - view_angle = 45 - aspect = @opts.width/@opts.height - near = 0.1 - far = Math.max(20000, opts.distance*2) - - @camera = new THREE.PerspectiveCamera(view_angle, aspect, near, far) - @scene.add(@camera) - @camera.position.set(opts.distance, opts.distance, opts.distance) - @camera.lookAt(@scene.position) - @camera.up = new THREE.Vector3(0,0,1) - - init_light: (color= 0xffffff) => - - ambient = new THREE.AmbientLight(0x404040) - @scene.add(ambient) - - color = 0xffffff - d = 10000000 - intensity = 0.5 - - for p in [[d,d,d], [d,d,-d], [d,-d,d], [d,-d,-d],[-d,d,d], [-d,d,-d], [-d,-d,d], [-d,-d,-d]] - directionalLight = new THREE.DirectionalLight(color, intensity) - directionalLight.position.set(p[0], p[1], p[2]).normalize() - @scene.add(directionalLight) - - @light = new THREE.PointLight(color) - @light.position.set(0,d,0) - - add_text: (opts) => - o = defaults opts, - pos : [0,0,0] - text : required - fontsize : 12 - fontface : 'Arial' - color : "#000000" # anything that is valid to canvas context, e.g., "rgba(249,95,95,0.7)" is also valid. - constant_size : true # if true, then text is automatically resized when the camera moves - # WARNING: if constant_size, don't remove text from scene (or if you do, note that it is slightly inefficient still.) - - #console.log("add_text: #{misc.to_json(o)}") - @show_canvas() - # make an HTML5 2d canvas on which to draw text - width = 300 # this determines max text width; beyond this, text is cut off. - height = 150 - canvas = document.createElement( 'canvas' ) - canvas.width = width - canvas.height = height - context = canvas.getContext("2d") # get the drawing context - - # set the fontsize and fix for our text. - context.font = "Normal " + o.fontsize + "px " + o.fontface - context.textAlign = 'center' - - # set the color of our text - context.fillStyle = o.color - - # actually draw the text -- right in the middle of the canvas. - context.fillText(o.text, width/2, height/2) - - # Make THREE.js texture from our canvas. - texture = new THREE.Texture(canvas) - texture.needsUpdate = true - texture.minFilter = THREE.LinearFilter - - # Make a material out of our texture. - spriteMaterial = new THREE.SpriteMaterial(map: texture) - - # Make the sprite itself. (A sprite is a 3d plane that always faces the camera.) - sprite = new THREE.Sprite(spriteMaterial) - - # Move the sprite to its position - p = @aspect_ratio_scale(o.pos) - sprite.position.set(p[0],p[1],p[2]) - - # If the text is supposed to stay constant size, add it to the list of constant size text, - # which gets resized on scene update. - if o.constant_size - if not @_text? - @_text = [sprite] - else - @_text.push(sprite) - - # Finally add the sprite to our scene - @scene.add(sprite) - - return sprite - - add_line: (opts) => - o = defaults opts, - points : required - thickness : 1 - color : "#000000" - arrow_head : false - if o.points.length <= 1 - # nothing to do... - return - - @show_canvas() - - if o.arrow_head - # Draw an arrowhead using the ArrowHelper: https://github.com/mrdoob/three.js/blob/master/src/extras/helpers/ArrowHelper.js - n = o.points.length - 1 - orig = @vector(o.points[n-1]) - p1 = @vector(o.points[n]) - dir = new THREE.Vector3(); dir.subVectors(p1, orig) - length = dir.length() - dir.normalize() - headLength = Math.max(1, o.thickness/4.0) * 0.2 * length - headWidth = 0.2 * headLength - @scene.add(new THREE.ArrowHelper(dir, orig, length, o.color, headLength, headWidth)) - - # always render the full line, in case there are extra points, or the thickness isn't 1 (note that ArrowHelper has no line thickness option). - geometry = new THREE.Geometry() - for a in o.points - geometry.vertices.push(@vector(a)) - @scene.add(new THREE.Line(geometry, new THREE.LineBasicMaterial(color:o.color, linewidth:o.thickness))) - - add_point: (opts) => - o = defaults opts, - loc : [0,0,0] - size : 5 - color: "#000000" - @show_canvas() - if not @_points? - @_points = [] - - # IMPORTANT: Below we use sprites instead of the more natural/faster PointCloudMaterial. - # Why? Because usually people don't plot a huge number of points, and PointCloudMaterial is SQUARE. - # By using sprites, our points are round, which is something people really care about. - - switch dynamic_renderer_type - - when 'webgl' - width = 50 - height = 50 - canvas = document.createElement('canvas') - canvas.width = width - canvas.height = height - - context = canvas.getContext('2d') # get the drawing context - centerX = width/2 - centerY = height/2 - radius = 25 - - context.beginPath() - context.arc(centerX, centerY, radius, 0, 2*Math.PI, false) - context.fillStyle = o.color - context.fill() - - texture = new THREE.Texture(canvas) - texture.needsUpdate = true - texture.minFilter = THREE.LinearFilter - spriteMaterial = new THREE.SpriteMaterial(map: texture) - particle = new THREE.Sprite(spriteMaterial) - - p = @aspect_ratio_scale(o.loc) - particle.position.set(p[0],p[1],p[2]) - @_points.push([particle, o.size/200]) - - when 'canvas' - # inspired by http://mrdoob.github.io/three.js/examples/canvas_particles_random.html - PI2 = Math.PI * 2 - program = (context) -> - context.beginPath() - context.arc(0, 0, 0.5, 0, PI2, true) - context.fill() - material = new THREE.SpriteCanvasMaterial - color : new THREE.Color(o.color) - program : program - particle = new THREE.Sprite(material) - p = @aspect_ratio_scale(o.loc) - particle.position.set(p[0],p[1],p[2]) - @_points.push([particle, 4*o.size/@opts.width]) - else - throw Error("bug -- unkown dynamic_renderer_type = #{dynamic_renderer_type}") - - @scene.add(particle) - - add_obj: (myobj)=> - @show_canvas() - - if myobj.type == 'index_face_set' - if myobj.has_local_colors == 0 - has_local_colors = false - else - has_local_colors = true - # then we will assume that every face is a triangle or a square - else - has_local_colors = false - - - vertices = myobj.vertex_geometry - for objects in [0...myobj.face_geometry.length] - #console.log("object=", misc.to_json(myobj)) - face3 = myobj.face_geometry[objects].face3 - face4 = myobj.face_geometry[objects].face4 - face5 = myobj.face_geometry[objects].face5 - - faces = myobj.face_geometry[objects].faces - if not faces? - faces = [] - - # backwards compatibility with old scenes - if face3? - for k in [0...face3.length] by 3 - faces.push(face3.slice(k,k+3)) - if face4? - for k in [0...face4.length] by 4 - faces.push(face4.slice(k,k+4)) - if face5? - for k in [0...face5.length] by 6 # yep, 6 :-() - faces.push(face5.slice(k,k+6)) - - geometry = new THREE.Geometry() - - for k in [0...vertices.length] by 3 - geometry.vertices.push(@vector(vertices.slice(k, k+3))) - - push_face3 = (a, b, c) => - geometry.faces.push(new THREE.Face3(a-1, b-1, c-1)) - #geometry.faces.push(new THREE.Face3(b-1, a-1, c-1)) # both sides of faces, so material is visible from inside -- but makes some things like look really crappy; disable. Better to just set a property of the material/light, which fixes the same problem. - - push_face3_with_color = (a, b, c, col) => - face = new THREE.Face3(a-1, b-1, c-1) - face.color.setStyle(col) - geometry.faces.push(face) - #geometry.faces.push(new THREE.Face3(b-1, a-1, c-1)) # both sides of faces, so material is visible from inside -- but makes some things like look really crappy; disable. Better to just set a property of the material/light, which fixes the same problem. - - # *polygonal* faces defined by 4 vertices (squares), which for THREE.js we must define using two triangles - push_face4 = (a, b, c, d) => - push_face3(a,b,c) - push_face3(a,c,d) - - push_face4_with_color = (a, b, c, d, col) => - push_face3_with_color(a,b,c,col) - push_face3_with_color(a,c,d,col) - - # *polygonal* faces defined by 5 vertices - push_face5 = (a, b, c, d, e) => - push_face3(a, b, c) - push_face3(a, c, d) - push_face3(a, d, e) - - # *polygonal* faces defined by 6 vertices (see http://people.cs.clemson.edu/~dhouse/courses/405/docs/brief-obj-file-format.html) - push_face6 = (a, b, c, d, e, f) => - push_face3(a, b, c) - push_face3(a, c, d) - push_face3(a, d, e) - push_face3(a, e, f) - - # include all faces - if has_local_colors - for v in faces - switch v.length - when 4 - push_face3_with_color(v...) - when 5 - push_face4_with_color(v...) - else - console.log("WARNING: rendering colored face with #{v.length - 1} vertices not implemented") - push_face4_with_color(v[0], v[1], v[2], v[3], v[-1]) # might as well render most of the face... - else - for v in faces - switch v.length - when 3 - push_face3(v...) - when 4 - push_face4(v...) - when 5 - push_face5(v...) - when 6 - push_face6(v...) - else - console.log("WARNING: rendering face with #{v.length} vertices not implemented") - push_face6(v...) # might as well render most of the face... - - geometry.mergeVertices() - #geometry.computeCentroids() - geometry.computeFaceNormals() - #geometry.computeVertexNormals() - geometry.computeBoundingSphere() - - #finding material key(mk) - name = myobj.face_geometry[objects].material_name - mk = 0 - for item in [0..myobj.material.length-1] - if name == myobj.material[item].name - mk = item - break - - if @opts.wireframe or myobj.wireframe - if myobj.color - color = myobj.color - else - c = myobj.material[mk].color - color = "rgb(#{c[0]*255},#{c[1]*255},#{c[2]*255})" - if typeof myobj.wireframe == 'number' - line_width = myobj.wireframe - else if typeof @opts.wireframe == 'number' - line_width = @opts.wireframe - else - line_width = 1 - - material = new THREE.MeshBasicMaterial - wireframe : true - color : color - wireframeLinewidth : line_width - side : THREE.DoubleSide - else if not myobj.material[mk]? - console.log("BUG -- couldn't get material for ", myobj) - material = new THREE.MeshBasicMaterial - wireframe : false - color : "#000000" - else - - m = myobj.material[mk] - - if has_local_colors - material = new THREE.MeshPhongMaterial - shininess : "1" - wireframe : false - transparent : m.opacity < 1 - vertexColors: THREE.FaceColors - else - material = new THREE.MeshPhongMaterial - shininess : "1" - wireframe : false - transparent : m.opacity < 1 - material.color.setRGB(m.color[0], m.color[1], m.color[2]) - material.specular.setRGB(m.specular[0], m.specular[1], m.specular[2]) - material.opacity = m.opacity - material.side = THREE.DoubleSide - - mesh = new THREE.Mesh(geometry, material) - mesh.position.set(0,0,0) - @scene.add(mesh) - - # always call this after adding things to the scene to make sure track - # controls are sorted out, etc. Set draw:false, if you don't want to - # actually *see* a frame. - set_frame: (opts) => - o = defaults opts, - xmin : required - xmax : required - ymin : required - ymax : required - zmin : required - zmax : required - color : @opts.foreground - thickness : .4 - labels : true # whether to draw three numerical labels along each of the x, y, and z axes. - fontsize : 14 - draw : true - @show_canvas() - - @_frame_params = o - eps = 0.1 - x0 = o.xmin; x1 = o.xmax; y0 = o.ymin; y1 = o.ymax; z0 = o.zmin; z1 = o.zmax - # console.log("set_frame: #{misc.to_json(o)}") - if Math.abs(x1-x0) - if not b? - z = a - else - z = (a+b)/2 - z = z.toFixed(2) - return (z*1).toString() - - txt = (x,y,z,text) => - @_frame_labels.push(@add_text(pos:[x,y,z], text:text, fontsize:o.fontsize, color:o.color, constant_size:false)) - - offset = 0.15 - if o.draw - e = (x1 - x0)*offset - txt(x0 - e, y0, z0, l(z0)) - txt(x0 - e, y0, mz, "z = #{l(z0,z1)}") - txt(x0 - e, y0, z1, l(z1)) - - e = (x1 - x0)*offset - txt(x1 + e, y0, z0, l(y0)) - txt(x1 + e, my, z0, "y = #{l(y0,y1)}") - txt(x1 + e, y1, z0, l(y1)) - - e = (y1 - y0)*offset - txt(x1, y0 - e, z0, l(x1)) - txt(mx, y0 - e, z0, "x = #{l(x0,x1)}") - txt(x0, y0 - e, z0, l(x0)) - - v = @vector3(mx, my, mz) - @camera.lookAt(v) - if @controls? - @controls.target = @_center - @render_scene() - - add_3dgraphics_obj: (opts) => - opts = defaults opts, - obj : required - wireframe : undefined - set_frame : undefined - @show_canvas() - - for o in opts.obj - try - switch o.type - when 'text' - @add_text - pos : o.pos - text : o.text - color : o.color - fontsize : o.fontsize - fontface : o.fontface - constant_size : o.constant_size - when 'index_face_set' - if opts.wireframe? - o.wireframe = opts.wireframe - @add_obj(o) - if o.mesh and not o.wireframe # draw a wireframe mesh on top of the surface we just drew. - o.color='#000000' - o.wireframe = o.mesh - @add_obj(o) - when 'line' - delete o.type - @add_line(o) - when 'point' - delete o.type - @add_point(o) - else - console.log("ERROR: no renderer for model number = #{o.id}") - return - catch err - # This can happen in some obscure case: https://github.com/sagemathinc/cocalc/issues/5654 - # It's better to gracefully skip than to crash. - console.warn("WARNING -- #{err}") - - if opts.set_frame? - @set_frame(opts.set_frame) - - @render_scene(true) - - - animate: (opts={}) => - opts = defaults opts, - fps : undefined - stop : false - mouseover : undefined # ignored now - render : true - #console.log("@animate #{@_animate_started}") - if @_animate_started and not opts.stop - return - @_animate_started = true - @_animate(opts) - - _animate: (opts) => - #console.log("anim?", @opts.element.length, @opts.element.is(":visible")) - - if @renderer_type == 'static' - # will try again when we switch to dynamic renderer - @_animate_started = false - return - - if not @opts.element.is(":visible") - if @opts.stop_when_gone? and not $.contains(document, @opts.stop_when_gone) - # console.log("stop_when_gone removed from document -- quit animation completely") - @_animate_started = false - else if not $.contains(document, @opts.element[0]) - # console.log("element removed from document; wait 5 seconds") - setTimeout((() => @_animate(opts)), 5000) - else - # console.log("check again after a second") - setTimeout((() => @_animate(opts)), 1000) - return - - if opts.stop - @_stop_animating = true - # so next time around will start - return - if @_stop_animating - @_stop_animating = false - @_animate_started = false - return - @render_scene(opts.render) - delete opts.render - f = () => - requestAnimationFrame((()=>@_animate(opts))) - if opts.fps? and opts.fps - setTimeout(f , 1000/opts.fps) - else - f() - - - render_scene: (force=false) => - # console.log('render', @opts.element.length) - # FUTURE: Render static - if @renderer_type == 'static' - console.log 'render static -- not implemented yet' - return - - if not @camera? - return # nothing to do yet - - @controls?.update() - - pos = @camera.position - if not @_last_pos? - new_pos = true - @_last_pos = pos.clone() - else if @_last_pos.distanceToSquared(pos) > .05 - new_pos = true - @_last_pos.copy(pos) - else - new_pos = false - - if not new_pos and not force - return - - # rescale all text in scene - @rescale_objects() - - @renderer.render(@scene, @camera) - - _rescale_factor: () => - if not @_center? - return undefined - else - return @camera.position.distanceTo(@_center) / 3 - - rescale_objects: (force=false) => - s = @_rescale_factor() - if not s? or (Math.abs(@_last_scale - s) < 0.000001 and not force) - return - @_last_scale = s - if @_text? - for sprite in @_text - sprite.scale.set(s,s,s) - if @_frame_labels? - for sprite in @_frame_labels - sprite.scale.set(s,s,s) - if @_points? - for z in @_points - c = z[1] - z[0].scale.set(s*c,s*c,s*c) - - -exports.render_3d_scene = (opts) -> - opts = defaults opts, - url : undefined # url from which to download (via ajax) a JSON string that parses to {opts:?,obj:?} - scene : undefined # {opts:?, obj:?} - element : required # DOM element - cb : undefined # cb(err, scene object) - # Render a 3-d scene - #console.log("render_3d_scene: url='#{opts.url}'") - - if not opts.scene? and not opts.url? - opts.cb?("one of url or scene must be defined") - return - - scene_obj = undefined - e = $(".webapp-3d-templates .webapp-3d-loading").clone() - opts.element.append(e) - async.series([ - (cb) => - if opts.scene? - cb() - else - f = (cb) -> - $.ajax( - url : opts.url - timeout : 30000 - success : (data) -> - try - opts.scene = misc.from_json(data) - cb() - catch e - console.log("ERROR", e) - cb(e) - ).fail () -> - console.log("FAIL") - cb(true) - misc.retry_until_success - f : f - max_tries : 10 - max_delay : 5 - cb : (err) -> - if err - msg = "error downloading #{opts.url} - #{err}" - console.warn(msg) - cb(msg) - else - cb() - (cb) => - e.remove() - # do this initialization *after* we create the 3d renderer - init = (err, s) -> - if err - cb(err) - else - scene_obj = s - s.init() - s.add_3dgraphics_obj - obj : opts.scene.obj - s.init_done() - cb() - # create the 3d renderer - opts.scene.opts.cb = init - opts.element.webapp_threejs(opts.scene.opts) - ], (err) -> - opts.cb?(err, scene_obj) - ) - - - -# jQuery plugin for making a DOM object into a 3d renderer - -$.fn.webapp_threejs = (opts={}) -> - @each () -> - # console.log("applying official .webapp_threejs plugin") - elt = $(this) - e = $(".webapp-3d-templates .webapp-3d-viewer").clone() - elt.empty().append(e) - e.find(".webapp-3d-canvas").hide() - opts.element = e - opts.container = elt - - # WARNING -- this explicit reference is brittle -- it is just an animation efficiency, but still... - opts.stop_when_gone = e.closest(".webapp-editor-codemirror")[0] - - f = () -> - obj = new WebappThreeJS(opts) - elt.data('webapp-threejs', obj) - if not THREE? - load_threejs (err) => - if not err - f() - else - msg = "Error loading THREE.js -- #{err}" - if not opts.cb? - console.log(msg) - else - opts.cb?(msg) - else - f() diff --git a/src/packages/frontend/sagews/cell.tsx b/src/packages/frontend/sagews/cell.tsx deleted file mode 100644 index b0271c85ae..0000000000 --- a/src/packages/frontend/sagews/cell.tsx +++ /dev/null @@ -1,27 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -Rendering a Sage worksheet cell -*/ - -import CellInput from "./input"; -import CellOutput from "./output"; -import type { OutputMessages } from "./parse-sagews"; - -interface Props { - input: string; - output: OutputMessages; - flags: string; -} - -export default function Cell({ input, output, flags }: Props) { - return ( -
- - {output && } -
- ); -} diff --git a/src/packages/frontend/sagews/chatgpt.ts b/src/packages/frontend/sagews/chatgpt.ts deleted file mode 100644 index 5aa996b430..0000000000 --- a/src/packages/frontend/sagews/chatgpt.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { redux } from "@cocalc/frontend/app-framework"; -import { getHelp } from "@cocalc/frontend/frame-editors/llm/help-me-fix"; -import { getValidLanguageModelName } from "@cocalc/util/db-schema/llm-utils"; -import { MARKERS } from "@cocalc/util/sagews"; -import { SETTINGS_LANGUAGE_MODEL_KEY } from "../account/useLanguageModelSetting"; - -export function isEnabled(project_id: string): boolean { - return redux - .getStore("projects") - .hasLanguageModelEnabled(project_id, "help-me-fix-solution"); -} - -export function isHintEnabled(project_id: string): boolean { - return redux - .getStore("projects") - .hasLanguageModelEnabled(project_id, "help-me-fix-hint"); -} - -interface HelpParams { - codemirror: any; - stderr: string; - uuid: string; - project_id: string; - path: string; -} - -function getHelpCommon(params: HelpParams, isHint: boolean): void { - const { codemirror, stderr, uuid, project_id, path } = params; - - // Show confirmation dialog - const action = isHint ? "get a hint" : "get help to fix this error"; - const confirmMessage = `This will query a language model to ${action}. The error message and your code will be sent to the AI service for analysis. Do you want to continue?`; - - if (!window.confirm(confirmMessage)) { - return; // User cancelled - } - - const val = codemirror.getValue(); - const i = val.indexOf(uuid); - if (i == -1) return; - const j = val.lastIndexOf(MARKERS.cell, i); - const k = val.lastIndexOf(MARKERS.output, i); - const input = val.slice(j + 1, k).trim(); - - // use the currently set language model from the account store - // https://github.com/sagemathinc/cocalc/pull/7278 - const other_settings = redux.getStore("account").get("other_settings"); - - const projectsStore = redux.getStore("projects"); - const enabled = projectsStore.whichLLMareEnabled(); - const ollama = redux.getStore("customize").get("ollama")?.toJS() ?? {}; - const customOpenAI = - redux.getStore("customize").get("custom_openai")?.toJS() ?? {}; - const selectableLLMs = - redux.getStore("customize").get("selectable_llms")?.toJS() ?? []; - - const model = getValidLanguageModelName({ - model: other_settings?.get(SETTINGS_LANGUAGE_MODEL_KEY), - filter: enabled, - ollama: Object.keys(ollama), - custom_openai: Object.keys(customOpenAI), - selectable_llms: selectableLLMs, - }); - - getHelp({ - project_id, - path, - tag: "sagews", - error: stderr, - input, - task: "ran a cell in a Sage Worksheet", - language: "sage", - extraFileInfo: "SageMath Worksheet", - redux, - prioritize: "end", - model, - isHint, - }); -} - -export function helpMeFix(params: HelpParams): void { - getHelpCommon(params, false); -} - -export function giveMeAHint(params: HelpParams): void { - getHelpCommon(params, true); -} diff --git a/src/packages/frontend/sagews/d3.coffee b/src/packages/frontend/sagews/d3.coffee deleted file mode 100644 index ffca06a6ed..0000000000 --- a/src/packages/frontend/sagews/d3.coffee +++ /dev/null @@ -1,238 +0,0 @@ -######################################################################### -# This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. -# License: MS-RSL – see LICENSE.md for details -######################################################################### - -$ = window.$ -misc = require('@cocalc/util/misc') -{defaults, required} = misc - -d3 = require('d3') - -# Make d3 available to users in general. -window?.d3 = d3 - -$.fn.extend - d3: (opts={}) -> - opts = defaults opts, - viewer : required - data : required - @each () -> - t = $(this) - elt = $("
") - t.replaceWith(elt) - switch opts.viewer - when 'graph' - d3_graph(elt, opts.data) - else - elt.append($("unknown d3 viewer '#{opts.viewer}'")) - return elt - -# Rewrite of code in Sage by Nathann Cohen. -d3_graph = (elt, graph) -> - color = d3.scale.category20() # List of colors - width = graph.width - if not width? - width = Math.min(elt.width(), 700) - height = graph.height - if not height? - height = .6*width - elt.width(width); elt.height(height) - elt.addClass("smc-d3-graph") - - #dbg = (m) -> console.log("d3_graph: #{JSON.stringify(m)}") - #dbg([width, height]) - - force = d3.layout.force() - .charge(graph.charge) - .linkDistance(graph.link_distance) - .linkStrength(graph.link_strength) - .gravity(graph.gravity) - .size([width, height]) - .links(graph.links) - .nodes(graph.nodes) - - # Returns the coordinates of a point located at distance d from the - # barycenter of two points pa, pb. - third_point_of_curved_edge = (pa, pb, d) -> - dx = pb.x - pa.x - dy = pb.y - pa.y - ox = pa.x - oy = pa.y - dx = pb.x - dy = pb.y - cx = (dx + ox)/2 - cy = (dy + oy)/2 - ny = -(dx - ox) - nx = dy - oy - nn = Math.sqrt(nx*nx + ny*ny) - return [cx+d*nx/nn, cy+d*ny/nn] - - # Applies a transformation to the points of the graph respecting the - # aspect ratio, so that the graph takes the whole rendering target - # and is centered - center_and_scale = (graph) -> - minx = graph.pos[0][0] - maxx = graph.pos[0][0] - miny = graph.pos[0][1] - maxy = graph.pos[0][1] - - graph.nodes.forEach (d,i) -> - maxx = Math.max(maxx, graph.pos[i][0]) - minx = Math.min(minx, graph.pos[i][0]) - maxy = Math.max(maxy, graph.pos[i][1]) - miny = Math.min(miny, graph.pos[i][1]) - - border = 60 - xspan = maxx - minx - yspan = maxy - miny - - scale = Math.min((height - border)/yspan, (width - border)/xspan) - xshift = (width - scale*xspan)/2 - yshift = (height - scale*yspan)/2 - - force.nodes().forEach (d,i) -> - d.x = scale*(graph.pos[i][0] - minx) + xshift - d.y = scale*(graph.pos[i][1] - miny) + yshift - - # Adapts the graph layout to the window's dimensions - if graph.pos.length != 0 - center_and_scale(graph) - - # SVG - id = 'a' + misc.uuid() - elt.attr('id', id) - svg = d3.select("##{id}").append("svg") - .attr("width", width) - .attr("height", height) - - # Edges - link = svg.selectAll(".link") - .data(force.links()) - .enter().append("path") - .attr("class", (d) -> "link directed") - .attr("marker-end", (d) -> "url(#directed)") - .style("stroke", (d) -> d.color) - .style("stroke-width", graph.edge_thickness+"px") - - # Loops - loops = svg.selectAll(".loop") - .data(graph.loops) - .enter().append("circle") - .attr("class", "link") - .attr("r", (d) -> d.curve) - .style("stroke", (d) -> d.color) - .style("stroke-width", graph.edge_thickness+"px") - - # Nodes - node = svg.selectAll(".node") - .data(force.nodes()) - .enter().append("circle") - .attr("class", "node") - .attr("r", graph.vertex_size) - .style("fill", (d) -> color(d.group)) - .call(force.drag) - - node.append("title").text((d) -> d.name) - - # Vertex labels - if graph.vertex_labels - v_labels = svg.selectAll(".v_label") - .data(force.nodes()) - .enter() - .append("svg:text") - .attr("vertical-align", "middle") - .text((d)-> return d.name) - - # Edge labels - if graph.edge_labels - e_labels = svg.selectAll(".e_label") - .data(force.links()) - .enter() - .append("svg:text") - .attr("text-anchor", "middle") - .text((d) -> d.name) - - l_labels = svg.selectAll(".l_label") - .data(graph.loops) - .enter() - .append("svg:text") - .attr("text-anchor", "middle") - .text((d,i) -> graph.loops[i].name) - - # Arrows, for directed graphs - if graph.directed - svg.append("svg:defs").selectAll("marker") - .data(["directed"]) - .enter().append("svg:marker") - .attr("id", String) - # viewbox is a rectangle with bottom-left corder (0,-2), width 4 and height 4 - .attr("viewBox", "0 -2 4 4") - # This formula took some time ... :-P - .attr("refX", Math.ceil(2*Math.sqrt(graph.vertex_size))) - .attr("refY", 0) - .attr("markerWidth", 4) - .attr("markerHeight", 4) - .attr("orient", "auto") - .append("svg:path") - # triangles with endpoints (0,-2), (4,0), (0,2) - .attr("d", "M0,-2L4,0L0,2") - - #.attr("preserveAspectRatio",false) # SMELL: this gives an error. - - # The function 'line' takes as input a sequence of tuples, and returns a - # curve interpolating these points. - line = d3.svg.line() - .interpolate("cardinal") - .tension(.2) - .x((d) -> d.x) - .y((d) -> d.y) - - # This is where all movements are defined - force.on "tick", () -> - - # Position of vertices - node.attr("cx", (d) -> d.x) - .attr("cy", (d) -> d.y) - - # Position of edges - link.attr "d", (d) -> - # Straight edges - if d.curve == 0 - return "M#{d.source.x},#{d.source.y} L#{d.target.x},#{d.target.y}" - # Curved edges - else - p = third_point_of_curved_edge(d.source,d.target,d.curve) - return line([{'x':d.source.x,'y':d.source.y}, - {'x':p[0],'y':p[1]}, - {'x':d.target.x,'y':d.target.y}]) - - # Position of Loops - if graph.loops.length != 0 - loops - .attr("cx", (d) -> return force.nodes()[d.source].x) - .attr("cy", (d) -> return force.nodes()[d.source].y-d.curve) - - # Position of vertex labels - if graph.vertex_labels - v_labels - .attr("x", (d) -> d.x+graph.vertex_size) - .attr("y", (d) -> return d.y) - - # Position of the edge labels - if graph.edge_labels - e_labels - .attr("x", (d) -> third_point_of_curved_edge(d.source,d.target,d.curve+3)[0]) - .attr("y", (d) -> third_point_of_curved_edge(d.source,d.target,d.curve+3)[1]) - l_labels - .attr("x", (d,i) -> force.nodes()[d.source].x) - .attr("y", (d,i) -> force.nodes()[d.source].y-2*d.curve-1) - - # Starts the automatic force layout - force.start() - if graph.pos.length != 0 - force.tick() - force.stop() - graph.nodes.forEach (d,i) -> - d.fixed = true - diff --git a/src/packages/frontend/sagews/input.tsx b/src/packages/frontend/sagews/input.tsx deleted file mode 100644 index 8c29c52b36..0000000000 --- a/src/packages/frontend/sagews/input.tsx +++ /dev/null @@ -1,31 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -Rendering input part of a Sage worksheet cell -*/ - -import { CodeMirrorStatic } from "@cocalc/frontend/jupyter/codemirror-static"; -import { FLAGS } from "@cocalc/util/sagews"; - -const OPTIONS = { mode: "sagews" }; - -interface Props { - input?: string; - flags?: string; -} - -export default function CellInput({ input, flags }: Props) { - if (flags?.includes(FLAGS.hide_input)) { - return ; - } - return ( - - ); -} diff --git a/src/packages/frontend/sagews/interact.coffee b/src/packages/frontend/sagews/interact.coffee deleted file mode 100644 index 41078c553d..0000000000 --- a/src/packages/frontend/sagews/interact.coffee +++ /dev/null @@ -1,478 +0,0 @@ -######################################################################### -# This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. -# License: MS-RSL – see LICENSE.md for details -######################################################################### - -# Interact -- Client side of interact implementation. -# -# This file defines a jQuery plugin ".sage_interact(...)" that replaces a DOM element -# by one with interactive controls and output. - -$ = window.$ - -misc = require('@cocalc/util/misc') - -{defaults, required} = misc - -# Interact jQuery plugin -$.fn.extend - sage_interact: (opts) -> - opts = defaults opts, # see comments for Interact below. - desc : required - execute_code : required - process_output_mesg : required - process_html_output : required - start : undefined - stop : undefined - - @each () -> - elt = $(this) - opts.elt = elt - interact = new Interact(opts) - elt.data("interact", interact) - return interact.element - -templates = $(".webapp-interact-templates") - -class Interact - constructor: (opts) -> - @opts = defaults opts, - # elt = a jQuery wrapped DOM element that will be replaced by the interact - elt : required - # desc = an object that describes the interact controls, etc. - desc : required - # execute_code = sage code executor; function that can be called like this - # (see the execute_code message in message.coffee): - # - # id = execute_code(code:?, data:?, preparse:?, cb:(mesg) => ) - # - execute_code : required - # - # process_output_mesg = message to deal with output from execute_code - # process_output_mesg(element:jQuery wrapped output DOM element, mesg:message output from execute_code) - - process_output_mesg : required - process_html_output : required - - # start(@) called when execution of code starts due to user manipulating a control - start : undefined - # stop(@) called when execution stops - stop : undefined - - @element = templates.find(".webapp-interact-container").clone() - @element.attr('id', opts.desc.id).data('interact', @) - @opts.elt.replaceWith(@element) - @initialize_interact() - - set_interact_var: (control_desc) => - var0 = control_desc.var - - controls = @element.find(".webapp-interact-var-#{var0}") - if controls.length > 0 - # There is already (at least) one control location with this name - for C in controls - control = $(C).find(':first-child') - if control.length > 0 - control.data("set")(control_desc.default) - else - # No control yet, so make one. - new_control = interact_control(control_desc, @element.data('update'), @opts.process_html_output) - $(C).append(new_control) - new_control.data('refresh')?() - else - # No controls with this name or even place to put it. - row = $("
") - container = $("
") - row.append(container) - new_control = interact_control(control_desc, @element.data('update'), @opts.process_html_output) - if new_control? - container.append(new_control) - @element.append(row) - new_control.data('refresh')?() - - del_interact_var: (arg) => - @element.find(".webapp-interact-var-#{arg}").remove() - - initialize_interact: () => - desc = @opts.desc - - # Canonicalize width - desc.width = parse_width(desc.width) - - # Create the fluid bootstrap layout canvas. - labels = {} - for row in desc.layout - fluid_row = $("
") - if row.length == 0 # empty row -- user wants space - fluid_row.append($("
")) - else - for x in row - arg = x[0]; span = x[1]; label = x[2] - if label? - labels[arg] = label - t = $("
") - fluid_row.append(t) - @element.append(fluid_row) - - # Create cell for the output stream from the function to appear in, if it is defined above - output = @element.find(".webapp-interact-var-") # empty string is output - - # Define the update function, which communicates with the server. - done = true - update = (vals) => - # FUTURE: flicker? - #for output_cell in output_cells - # if not desc.flicker - # height = output_cell._output.height() - # output_cell._output.css('min-height', height) - # output_cell.delete_output() - output.html("") - - done = false - first = true - @opts.execute_code - code : 'salvus._execute_interact(salvus.data["id"], salvus.data["vals"])' - data : {id:desc.id, vals:vals} - preparse : false - cb : (mesg) => - if first - @opts.start?() - first = false - - @opts.process_output_mesg(mesg:mesg, element:output) - - if mesg.done - # stop the stopwatch - @opts.stop?() - done = true - - # Define the controls. - created_controls = [] - for control_desc in desc.controls - containing_div = @element.find(".webapp-interact-var-#{control_desc.var}") - if labels[control_desc.var]? - control_desc.label = labels[control_desc.var] - for X in containing_div - c = interact_control(control_desc, update, @opts.process_html_output) - created_controls.push(c) - $(X).append(c) - - # Refresh any controls that need refreshing - for c in created_controls - c.data('refresh')?() - - @element.attr('style', desc.style) - @element.data('update', update) - - if desc.width? - @element.width(desc.width) - - update({}) - - - -parse_width = (width) -> - if width? - if typeof width == 'number' - return "#{width}ex" - else - return width - -interact_control = (desc, update, process_html_output) -> - # Create and return a detached DOM element elt that represents - # the interact control described by desc. It will call update - # when it changes. If @element.data('refresh') is defined, it will - # be called after the control is inserted into the DOM. - - # Generic initialization code - control = templates.find(".webapp-interact-control-#{desc.control_type}").clone() - if control.length == 0 - # nothing to do -- the control no longer exists (deprecated?) - # WARNING: we should probably send a message somewhere saying this no longer exists. - return - control.processIcons() - if desc.label? - control.find(".webapp-interact-label").html(desc.label).katex({preProcess:true}) - - # Initialization specific to each control type - set = undefined - send = (val) -> - vals = {} - vals[desc.var] = val - update(vals) - - desc.width = parse_width(desc.width) - - switch desc.control_type - when 'input-box' - last_sent_val = undefined - do_send = () -> - val = input.val() - last_sent_val = val - send(val) - - if desc.nrows <= 1 - input = control.find("input").show() - input.keypress (evt) -> - if evt.which == 13 - do_send() - else - input = control.find("textarea").show().attr('rows', desc.nrows) - desc.submit_button = true - input.keypress (evt) -> - if evt.shiftKey and evt.which == 13 - do_send() - return false - - set = (val) -> - input.val(val) - process_html_output(input) - - input.on 'blur', () -> - if input.val() != last_sent_val - do_send() - - if desc.submit_button - submit = control.find(".webapp-interact-control-input-box-submit-button").show() - submit.find("a").click(() -> send(input.val())) - - if desc.readonly - input.attr('readonly', 'readonly') - input.width(desc.width) - - - when 'checkbox' - input = control.find("input") - set = (val) -> - input.attr('checked', val) - input.click (evt) -> - send(input.is(':checked')) - if desc.readonly - input.attr('disabled', 'disabled') - - when 'button' - button = control.find("a") - if desc.classes - for cls in desc.classes.split(/\s+/g) - button.addClass(cls) - if desc.width - button.width(desc.width) - if desc.icon - button.find('i').addClass(desc.icon) - else - button.find('i').hide() - button.click (evt) -> send(null) - set = (val) -> - button.find("span").html(val).katex({preProcess:true}) - - when 'text' - text = control.find(".webapp-interact-control-content") - if desc.classes - for cls in desc.classes.split(/\s+/g) - text.addClass(cls) - - # This is complicated because we shouldn't run mathjax until - # the element is visible. - set = (val) -> - if text.data('val')? - # it has already appeared, so safe to mathjax immediately - text.html(val) - process_html_output(text) - text.katex({preProcess:true}) - - text.data('val', val) - - control.data 'refresh', () -> - text.katex({preProcess:true, tex:text.data('val')}) - - when 'input-grid' - grid = control.find(".webapp-interact-control-grid") - - entries = [] - for i in [0...desc.nrows] - for j in [0...desc.ncols] - cell = $('').css("margin","0") - if desc.width - cell.width(desc.width) - cell.keypress (evt) -> - if evt.which == 13 - send_all() - grid.append(cell) - entries.push(cell) - grid.append($('
')) - - send_all = () -> - send( (cell.val() for cell in entries) ) - - control.find("a").click () -> - send_all() - - set = (val) -> - cells = grid.find("input") - i = 0 - for r in val - for c in r - $(cells[i]).val(c).data('last',c) - i += 1 - - when 'color-selector' - input = control.find("input").colorpicker() - sample = control.find("i") - input.change (ev) -> - hex = input.val() - input.colorpicker('setValue', hex) - input.on "changeColor", (ev) -> - hex = ev.color.toHex() - sample.css("background-color", hex) - send(hex) - sample.click (ev) -> input.colorpicker('show') - set = (val) -> - input.val(val) - sample.css("background-color", val) - if desc.hide_box - input.parent().width('1px') - else - input.parent().width('9em') - - when 'slider' - content = control.find(".webapp-interact-control-content") - slider = content.find(".webapp-interact-control-slider") - value = control.find(".webapp-interact-control-value") - if desc.width? - slider.width(desc.width) - slider.slider - animate : desc.animate - min : 0 - max : desc.vals.length-1 - step : 1 - value : desc.default - change : (event, ui) -> - if desc.display_value - value.text(desc.vals[ui.value]) - if event.altKey? - # This is a genuine event by user, not the result of calling "set" below. - send(ui.value) - - set = (val) -> - slider.slider('value', val) - - when 'range-slider' - content = control.find(".webapp-interact-control-content") - slider = content.find(".webapp-interact-control-slider") - value = control.find(".webapp-interact-control-value") - if desc.width - content.width(desc.width) - slider.slider - animate : desc.animate - range : true - min : 0 - max : desc.vals.length-1 - step : 1 - values : desc.default - change : (event, ui) -> - if desc.display_value - v = slider.slider("values") - value.text("#{desc.vals[v[0]]} - #{desc.vals[v[1]]}") - if event.altKey? - # This is a genuine event by user, not calling "set" below. - send(slider.slider("values")) - - set = (val) -> - slider.slider('values', val) - - when 'selector' - content = control.find(".webapp-interact-control-content") - if desc.buttons or desc.nrows != null or desc.ncols != null - content.addClass('webapp-interact-control-selector-buttonbox') - ######################## - # Buttons. - ######################## - if desc.ncols != null - ncols = desc.ncols - else if desc.nrows != null - ncols = Math.ceil(desc.lbls.length/desc.nrows) - else - ncols = desc.lbls.length - - multi_row = (desc.lbls.length > ncols) - - bar = $('') - if multi_row - bar.addClass('btn-group') - content.append(bar) - - i = 0 - for lbl in desc.lbls - button = $("").data('value',i).text(lbl) - if desc.button_classes != null - if typeof desc.button_classes == "string" - c = desc.button_classes - else - c = desc.button_classes[i] - for cls in c.split(/\s+/g) - button.addClass(cls) - if desc.width - button.width(desc.width) - button.click () -> - val = $(@).data('value') - send(val) - set(val) - bar.append(button) - i += 1 - if i % ncols == 0 and i < desc.lbls.length - # start a new row in the button bar - content.append($('
')) - bar = $('') - content.append(bar) - - control.data 'refresh', () -> - if ncols != desc.lbls.length and not desc.width - # If no width param is specified and the - # button bar will take up multiple lines, make - # all buttons the same width as the widest, so - # the buttons look nice. - w = Math.max.apply @, ($(x).width() for x in content.find("a")) - content.find("a").width(w) - - set = (val) -> - content.find("a.active").removeClass("active") - $(content.find("a")[val]).addClass("active") - else - # A standard drop down selector box. - select = $(" -
- ); - }, -}; - -export default RENDERERS; diff --git a/src/packages/frontend/sagews/output.tsx b/src/packages/frontend/sagews/output.tsx deleted file mode 100644 index 4447537544..0000000000 --- a/src/packages/frontend/sagews/output.tsx +++ /dev/null @@ -1,87 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -Rendering output part of a Sage worksheet cell -*/ - -import { keys, cmp, len } from "@cocalc/util/misc"; -import { FLAGS } from "@cocalc/util/sagews"; -import type { OutputMessage, OutputMessages } from "./parse-sagews"; -import RENDERERS from "./output-renderers"; - -interface Props { - output: OutputMessages; - flags?: string; -} - -export default function CellOutput({ output, flags }: Props) { - if (flags != null && flags.indexOf(FLAGS.hide_output) != -1) { - return ; - } - - function renderOutputMesg(elts: React.JSX.Element[], mesg: object): void { - for (const type in mesg) { - let value: any = mesg[type]; - let f = RENDERERS[type]; - if (f == null) { - f = RENDERERS.stderr; - value = `unknown message type '${type}'`; - } - elts.push(f(value, elts.length)); - } - } - - function renderOutput(): React.JSX.Element[] { - const elts: React.JSX.Element[] = []; - for (const mesg of processMessages(output)) { - renderOutputMesg(elts, mesg); - } - return elts; - } - - return
{renderOutput()}
; -} - -// sort in order to a list and combine adjacent stdout/stderr messages. -const STRIP = ["done", "error", "once", "javascript", "hide", "show"]; // these are just deleted -- make no sense for static rendering. - -function processMessages(output: OutputMessages): object[] { - const v: string[] = keys(output); - v.sort((a, b) => cmp(parseInt(a), parseInt(b))); - let r: OutputMessage[] = []; - for (const a of v) { - const m = output[a]; - for (const s of STRIP) { - if (m[s] != null) { - delete m[s]; - } - } - const n = len(m); - if (n === 0) { - continue; - } - if (m.clear) { - r = []; - continue; - } - if (m.delete_last) { - r.pop(); - continue; - } - if (r.length > 0 && n === 1) { - if (m.stdout != null && r[r.length - 1].stdout != null) { - r[r.length - 1] = { stdout: r[r.length - 1].stdout + m.stdout }; - continue; - } - if (m.stderr != null && r[r.length - 1].stderr != null) { - r[r.length - 1] = { stderr: r[r.length - 1].stderr + m.stderr }; - continue; - } - } - r.push(m); - } - return r; -} diff --git a/src/packages/frontend/sagews/parse-sagews.ts b/src/packages/frontend/sagews/parse-sagews.ts deleted file mode 100644 index 3acc56c37e..0000000000 --- a/src/packages/frontend/sagews/parse-sagews.ts +++ /dev/null @@ -1,111 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -Take a sagews file and produce a structured object representation of it. - -Why? - -- This is used for our public share server, to have sane input to a React-based renderer. -- This will be used for a syncdb based version of sage worksheets, someday. - -*/ - -import { MARKERS } from "@cocalc/util/sagews"; - -// Input: a string that is the contents of a .sagews file -// Output: a list of objects -// [{type:'cell', pos:0, id:'...', flags:'...', input:'...', output:{0:mesg, 1:mesg, ...}}] - -export type OutputMessage = any; -export type OutputMessages = { [n: number]: OutputMessage }; - -type CellTypes = "cell"; - -export interface Cell { - type: CellTypes; - pos: number; - id: string; - flags?: string; - input?: string; - output?: OutputMessages; -} - -export function parse_sagews(sagews: string): Cell[] { - const obj: Cell[] = []; - let pos: number = 0; - let i: number = 0; - while (true) { - const meta_start: number = sagews.indexOf(MARKERS.cell, i); - if (meta_start === -1) { - break; - } - const meta_end: number = sagews.indexOf(MARKERS.cell, meta_start + 1); - if (meta_end === -1) { - break; - } - const id: string = sagews.slice(meta_start + 1, meta_start + 1 + 36); - const flags: string = sagews.slice(meta_start + 1 + 36, meta_end); - let output_start: number = sagews.indexOf(MARKERS.output, meta_end + 2); - let output_end: number; - if (output_start === -1) { - output_start = sagews.length; - output_end = sagews.length; - } else { - const n: number = sagews.indexOf(MARKERS.cell, output_start + 1); - if (n === -1) { - output_end = sagews.length; - } else { - output_end = n - 1; - } - } - const input: string = sagews.slice(meta_end + 2, output_start - 1); - let n: number = 0; - const output: OutputMessages = {}; - for (const s of sagews - .slice(output_start + 38, output_end) - .split(MARKERS.output)) { - if (!s) { - continue; - } - try { - const mesg: OutputMessage = JSON.parse(s); - output[`${n}`] = mesg; - n += 1; - } catch (err) { - console.warn(`exception parsing '${s}'; ignoring -- ${err}`); - } - } - const cell: Cell = { - type: "cell", - pos, - id, - }; - if (flags) { - cell.flags = flags; - } - if (n > 0) { - cell.output = output; - } - if (input) { - cell.input = input; - } - obj.push(cell); - pos += 1; - i = output_end + 1; - } - - if (pos === 0 && sagews.trim().length > 0) { - // special case -- no defined cells, e.g., just code that hasn't been run - obj.push({ - type: "cell", - pos: 0, - id: "", - input: sagews, - }); - } - - return obj; -} diff --git a/src/packages/frontend/sagews/sagews-eval.coffee b/src/packages/frontend/sagews/sagews-eval.coffee deleted file mode 100644 index c7da963028..0000000000 --- a/src/packages/frontend/sagews/sagews-eval.coffee +++ /dev/null @@ -1,97 +0,0 @@ -######################################################################### -# This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. -# License: MS-RSL – see LICENSE.md for details -######################################################################### - -### -Used for potentially dangerous -code evaluation in Sage worksheets. -### - - -### -Cell and Worksheet below are used when eval'ing %javascript blocks. -### - -{defaults} = require('@cocalc/util/misc') - -log = (s) -> console.log(s) - -class Cell - constructor: (opts) -> - @opts = defaults opts, - output : undefined # jquery wrapped output area - cell_id : undefined - @output = opts.output - @cell_id = opts.cell_id - -class Worksheet - constructor: (sagews_doc, redux) -> - # Copy over exactly the methods we need rather than everything. - # This is a token attempt ot make this slightly less dangerous. - # Obviously, execute_code is quite dangerous... for a particular project on the backend. - @worksheet = {} - for x in ['execute_code', 'interrupt', 'kill', 'element'] - @worksheet[x] = sagews_doc[x] - - # The following project_page functions are assumed to be available - # by the backend sage_server.py, which generates code that uses them. - actions = redux.getProjectActions(sagews_doc.editor.project_id) - @project_page = - open_file : actions.open_file - close_file : actions.close_file - open_directory : actions.open_directory - set_current_path : actions.set_current_path - - execute_code: (opts) => - if typeof opts == "string" - opts = {code:opts} - @worksheet.execute_code(opts) - - interrupt: () => - @worksheet.interrupt() - - kill: () => - @worksheet.kill() - - set_interact_var: (opts) => - elt = @worksheet.element.find("#" + opts.id) - if elt.length == 0 - log("BUG: Attempt to set var of interact with id #{opts.id} failed since no such interact known.") - else - i = elt.data('interact') - if not i? - log("BUG: interact with id #{opts.id} doesn't have corresponding data object set.", elt) - else - i.set_interact_var(opts) - - del_interact_var: (opts) => - elt = @worksheet.element.find("#" + opts.id) - if elt.length == 0 - log("BUG: Attempt to del var of interact with id #{opts.id} failed since no such interact known.") - else - i = elt.data('interact') - if not i? - log("BUG: interact with id #{opts.id} doesn't have corresponding data object del.", elt) - else - i.del_interact_var(opts.name) - - - -exports.sagews_eval = (code, sagews_doc, element, id, obj, redux) -> - if element? - cell = new Cell(output : element, cell_id : id) - worksheet = new Worksheet(sagews_doc, redux) - sagews_doc = redux = undefined # so not visible in eval - print = (s...) -> - for i in [0...s.length] - if typeof(s[i]) != 'string' - s[i] = JSON.stringify(s[i] ? 'undefined') - cell.output.append($("
").text("#{s.join(' ')}")) - try - eval(code) - catch js_error - cell.output.append($("
").text("#{js_error}\n(see the Javascript console for more details)")) - console.warn("ERROR evaluating code '#{code}' in Sage worksheet", js_error) - console.trace() - diff --git a/src/packages/frontend/sagews/sagews.coffee b/src/packages/frontend/sagews/sagews.coffee deleted file mode 100644 index 906f93eadd..0000000000 --- a/src/packages/frontend/sagews/sagews.coffee +++ /dev/null @@ -1,2525 +0,0 @@ -######################################################################### -# This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. -# License: MS-RSL – see LICENSE.md for details -######################################################################### - - -$ = window.$ -async = require('async') -stringify = require('json-stable-stringify') -CodeMirror = require('codemirror') -{ appBasePath } = require("@cocalc/frontend/customize/app-base-path"); - -{MARKERS, FLAGS, ACTION_FLAGS, ACTION_SESSION_FLAGS} = require('@cocalc/util/sagews') - -{SynchronizedDocument2} = require('../syncdoc') - -misc = require('@cocalc/util/misc') -{defaults, required} = misc - -message = require('@cocalc/util/message') -markdown = require('../markdown') -{webapp_client} = require('../webapp-client') -{redux} = require('../app-framework') -{alert_message} = require('../alerts') -{join} = require('path') - -{sagews_eval} = require('./sagews-eval') - -# Define interact jQuery plugins - used only by sage worksheets -require('./interact') - - -{IS_TOUCH} = require('../feature') - -templates = $("#webapp-editor-templates") -cell_start_template = templates.find(".sagews-input") -output_template = templates.find(".sagews-output") -chatgpt = require('./chatgpt') - -log = (s) -> console.log(s) - -CLIENT_SIDE_MODE_LINES = {} -for mode in ['md', 'html', 'coffeescript', 'javascript', 'cjsx'] - CLIENT_SIDE_MODE_LINES["%#{mode}"] = {mode:mode} - CLIENT_SIDE_MODE_LINES["%#{mode}(hide=false)"] = {mode:mode, hide:false} - CLIENT_SIDE_MODE_LINES["%#{mode}(hide=true)"] = {mode:mode, hide:true} - CLIENT_SIDE_MODE_LINES["%#{mode}(hide=0)"] = {mode:mode, hide:false} - CLIENT_SIDE_MODE_LINES["%#{mode}(hide=1)"] = {mode:mode, hide:true} - CLIENT_SIDE_MODE_LINES["%#{mode}(once=false)"] = {mode:mode} - CLIENT_SIDE_MODE_LINES["%#{mode}(once=0)"] = {mode:mode} - -MARKERS_STRING = MARKERS.cell + MARKERS.output -is_marked = (c) -> - if not c? - return false - return c.indexOf(MARKERS.cell) != -1 or c.indexOf(MARKERS.output) != -1 - -# Create gutter elements -open_gutter_elt = $('
') -folded_gutter_elt = $('
') -line_number_elt = $("
") - -class SynchronizedWorksheet extends SynchronizedDocument2 - constructor: (editor, opts) -> - if opts.static_viewer - super(editor, opts) - @readonly = true - @project_id = editor.project_id - @filename = editor.filename - return - - # We set a custom rangeFinder that is output cell marker aware. - # See https://github.com/sagemathinc/cocalc/issues/966 - foldOptions = - rangeFinder : (cm, start) -> - helpers = cm.getHelpers(start, "fold") - for h in helpers - cur = h(cm, start) - if cur - i = start.line - while i < cur.to.line and cm.getLine(i+1)?[0] != MARKERS.output - i += 1 - if cm.getLine(i+1)?[0] == MARKERS.output - cur.to.line = i - cur.to.ch = cm.getLine(i).length - return cur - - opts0 = - cursor_interval : opts.cursor_interval - sync_interval : opts.sync_interval - cm_foldOptions : foldOptions - persistent : true # so sage session **not** killed when everybody closes the tab. - - super(editor, opts0) - - # Code execution queue. - @execution_queue = new ExecutionQueue(@_execute_cell_server_side, @) - - # Since we can't use in super cbs, use _init_cb as the function which - # will be called by the parent - _init_cb: => - @readonly = @_syncstring.is_read_only() # TODO: harder problem -- if file state flips between read only and not, need to rerender everything... - - @init_hide_show_gutter() # must be after @readonly set - - @process_sage_updates(caller:"constructor") # MUST be after @readonly is set. - if not @readonly - @status cb: (err, status) => - if not status?.running - # nobody has started the worksheet running yet, so enqueue the %auto cells - @execute_auto_cells() - else - # worksheet is running, but do something just to ensure it works - # Kick the worksheet process into gear if it isn't running already - @introspect_line - line : "return?" - timeout : 30 - preparse : false - cb : (err) => - - @on 'sync', () => - #console.log("sync") - @process_sage_update_queue() - - @editor.on 'show', (height) => - @set_all_output_line_classes() - - @editor.on 'toggle-split-view', => - @process_sage_updates(caller:"toggle-split-view") - - @init_worksheet_buttons() - - v = [@codemirror, @codemirror1] - for cm in v - cm.on 'beforeChange', (instance, changeObj) => - #console.log("beforeChange (#{instance.name}): #{misc.to_json(changeObj)}") - # Set the evaluated flag to false for the cell that contains the text - # that just changed (if applicable) - if changeObj.origin == 'redo' - return - if changeObj.origin == 'undo' - return - if changeObj.origin? and changeObj.origin != 'setValue' - @remove_this_session_flags_from_changeObj_range(changeObj) - - if changeObj.origin == 'paste' - changeObj.cancel() - # WARNING: The Codemirror manual says "Note: you may not do anything - # from a "beforeChange" handler that would cause changes to the - # document or its visualization." I think this is OK below though - # since we just canceled the change. - @remove_cell_flags_from_changeObj(changeObj, ACTION_SESSION_FLAGS) - @_apply_changeObj(changeObj) - @process_sage_updates(caller:"paste") - @sync() - - cm.on 'change', (instance, changeObj) => - #console.log('changeObj=', changeObj) - if changeObj.origin == 'undo' or changeObj.origin == 'redo' - return - start = changeObj.from.line - stop = changeObj.to.line + changeObj.text.length + 1 # changeObj.text is an array of lines - - if @editor.opts.line_numbers - # If stop isn't at a marker, extend stop to include the rest of the input, - # so relative line numbers for this cell get updated. - x = cm.getLine(stop)?[0] - if x != MARKERS.cell and x != MARKERS.output - n = cm.lineCount() - 1 - while stop < n and x != MARKERS.output and x != MARKERS.cell - stop += 1 - x = cm.getLine(stop)?[0] - - # Similar for start - x = cm.getLine(start)?[0] - if x != MARKERS.cell and x != MARKERS.output - while start > 0 and x != MARKERS.cell and x != MARKERS.output - start -= 1 - x = cm.getLine(start)?[0] - - if not @_update_queue_start? or start < @_update_queue_start - @_update_queue_start = start - if not @_update_queue_stop? or stop > @_update_queue_stop - @_update_queue_stop = stop - @process_sage_update_queue() - - close: => - @execution_queue?.close() - super() - - init_hide_show_gutter: () => - gutters = ["CodeMirror-linenumbers", "CodeMirror-foldgutter", "smc-sagews-gutter-hide-show"] - for cm in [@codemirror, @codemirror1] - continue if not cm? - cm.setOption('gutters', gutters) - cm.on 'gutterClick', @_handle_input_hide_show_gutter_click - - _handle_input_hide_show_gutter_click: (cm, line, gutter) => - if gutter != 'smc-sagews-gutter-hide-show' - return - x = cm.getLine(line) - if not x? - return - switch x[0] - when MARKERS.cell - @action(pos:{line:line, ch:0}, toggle_input:true) - when MARKERS.output - @action(pos:{line:line, ch:0}, toggle_output:true) - - _apply_changeObj: (changeObj) => - @codemirror.replaceRange(changeObj.text, changeObj.from, changeObj.to) - if changeObj.next? - @_apply_changeObj(changeObj.next) - - # Get cell at current line or return undefined if create=false - # and there is no complete cell with input and output. - cell: (line, create=true) => - {start, end} = @current_input_block(line) - if not create - cm = @focused_codemirror() - if cm.getLine(start)?[0] != MARKERS.cell or cm.getLine(end)?[0] != MARKERS.output - return - return new SynchronizedWorksheetCell(@, start, end) - # CRITICAL: We do **NOT** cache cells. The reason is that client code should create - # a cell for a specific purpose then forget about it as soon as that is done!!! - # The reason is that at any time new input cell lines can be added in the - # middle of a cell, and in general the document can change arbitrarily. - # Keeping a big list of cells in sync with the document would be - # extremely difficult and inefficient. Instead, this cell class just provides - # a clean abstraction for doing specific things with cells. - - # Return list of all cells that are touched by the current selection - # or contain any cursors. - get_current_cells: (create=true) => - cm = @focused_codemirror() - cells = [] - top = undefined - process_line = (n) => - if not top? or n < top - cell = @cell(n, create) - if cell? - cells.push(cell) - top = cell.start_line() - # "These [selections] will always be sorted, and never overlap (overlapping selections are merged)." - for sel in cm.listSelections().reverse() - n = sel.anchor.line; m = sel.head.line - if n == m - process_line(n) - else if n < m - for i in [m..n] - process_line(i) - else - for i in [n..m] - process_line(i) - return cells.reverse() - - get_all_cells: => - cm = @focused_codemirror() - if not cm? - return [] - cells = [] - top = undefined - process_line = (n) => - if not top? or n < top - cell = @cell(n) - cells.push(cell) - top = cell.start_line() - n = cm.lineCount() - 1 - while n > 0 and cm.getLine(n)[0] != MARKERS.output # skip empty lines at end so don't create another cell - n -= 1 - if n == 0 - # no cells yet - return [] - while n >= 0 - process_line(n) - n -= 1 - return cells.reverse() - - process_sage_update_queue: => - #console.log("process, start=#{@_update_queue_start}, stop=#{@_update_queue_stop}") - @process_sage_updates - start : @_update_queue_start - stop : @_update_queue_stop - caller : 'queue' - @_update_queue_start = undefined - @_update_queue_stop = undefined - - init_worksheet_buttons: () => - buttons = @element.find(".webapp-editor-codemirror-worksheet-buttons") - buttons.show() - buttons.find("a").tooltip(delay:{ show: 500, hide: 100 }) - buttons.find("a[href=\"#execute\"]").click () => - @action(execute:true, advance:false) - @focused_codemirror().focus() - return false - buttons.find("a[href=\"#toggle-input\"]").click () => - @action(execute:false, toggle_input:true) - @focused_codemirror().focus() - return false - buttons.find("a[href=\"#toggle-output\"]").click () => - @action(execute:false, toggle_output:true) - @focused_codemirror().focus() - return false - buttons.find("a[href=\"#delete-output\"]").click () => - @action(execute:false, delete_output:true) - @focused_codemirror().focus() - return false - - if IS_TOUCH - buttons.find("a[href=\"#tab\"]").click () => - @editor.press_tab_key(@editor.codemirror_with_last_focus) - @focused_codemirror().focus() - return false - @element.find("a[href=\"#copy\"]").remove() - @element.find("a[href=\"#replace\"]").remove() - @element.find("a[href=\"#paste\"]").remove() - @element.find("a[href=\"#goto-line\"]").remove() - @element.find("a[href=\"#sagews2ipynb\"]").find("span").remove() - else - @element.find("a[href=\"#tab\"]").remove() - @element.find("a[href=\"#undo\"]").remove() - @element.find("a[href=\"#redo\"]").remove() - - buttons.find("a[href=\"#new-html\"]").click () => - cm = @focused_codemirror() - line = cm.lineCount()-1 - while line >= 0 and cm.getLine(line) == "" - line -= 1 - if line >= 0 and cm.getLine(line)[0] == MARKERS.cell - cm.replaceRange("%html\n", {line:line+1,ch:0}) - cm.setCursor(line:line+1, ch:0) - else - cm.replaceRange("\n\n\n", {line:line+1,ch:0}) - @cell_start_marker(line+1) - @cell_start_marker(line+3) - cm.replaceRange("%html\n", {line:line+2,ch:0}) - cm.setCursor(line:line+2, ch:0) - @action - execute : true - advance : true - @focused_codemirror().focus() - - interrupt_button = buttons.find("a[href=\"#interrupt\"]").click () => - interrupt_button.find("i").addClass('fa-spin') - @interrupt - maxtime : 15 - cb : (err) => - interrupt_button.find("i").removeClass('fa-spin') - if err - alert_message(type:"error", message:"Unable to interrupt worksheet; try restarting the worksheet instead.") - @focused_codemirror().focus() - return false - - kill_button = buttons.find("a[href=\"#kill\"]").click () => - kill_button.find("i").addClass('fa-spin') - @_restarting = true - @kill - restart : true - cb : (err) => - delete @_restarting # must happen *before* emiting the restarted event - @emit('restarted', err) - kill_button.find("i").removeClass('fa-spin') - if err - alert_message(type:"error", message:"Unable to restart worksheet (the system might be heavily loaded causing Sage to take a while to restart -- try again in a minute)") - @focused_codemirror().focus() - return false - - _is_dangerous_undo_step: (cm, changes) => - if not changes? - return false - for c in changes - if c.from.line == c.to.line - if c.from.line < cm.lineCount() # ensure we have such line in document - if is_marked(cm.getLine(c.from.line)) - return true - for t in c.text - if is_marked(t) - return true - return false - - on_undo: (cm, changeObj) => - u = cm.getHistory().undone - if u.length > 0 and @_is_dangerous_undo_step(cm, u[u.length-1].changes) - #console.log("on_undo(repeat)") - try - cm.undo() - catch e - console.warn("skipping undo: ",e) - @process_sage_updates() # reprocess entire buffer -- e.g., output could change in strange ways - @set_all_output_line_classes() - - on_redo: (cm, changeObj) => - u = cm.getHistory().done - if u.length > 0 and @_is_dangerous_undo_step(cm, u[u.length-1].changes) - #console.log("on_redo(repeat)") - try - cm.redo() - # TODO: having to do this is potentially very bad/slow if document has large number - # to do this. This is temporary anyways, since we plan to get rid of using codemirror - # undo entirely. - catch e - console.warn("skipping redo: ",e) - @process_sage_updates() # reprocess entire buffer - @set_all_output_line_classes() - - interrupt: (opts={}) => - opts = defaults opts, - maxtime : 15 - cb : undefined - if @readonly - opts.cb?(); return - @close_on_action() - t = misc.walltime() - @execution_queue?.clear() - @clear_action_flags(false) - async.series([ - (cb) => - @send_signal - signal : 2 - cb : cb - (cb) => - @start - maxtime : opts.maxtime - misc.walltime(t) - cb : cb - ], (err) => opts.cb?(err)) - - clear_action_flags: (this_session) => - flags = if this_session then ACTION_SESSION_FLAGS else ACTION_FLAGS - for cell in @get_all_cells() - for flag in flags - cell.remove_cell_flag(flag) - - kill: (opts={}) => - opts = defaults opts, - restart : false - maxtime : 60 - cb : undefined - if @readonly - opts.cb?(); return - t = misc.walltime() - @close_on_action() - @clear_action_flags(true) - # Empty the execution queue. - @execution_queue?.clear() - @process_sage_updates(caller:"kill") - if opts.restart - @restart(cb:opts.cb) - else - @send_signal - signal : 9 - cb : opts.cb - - # ensure that the sage process is working and responding to compute requests - start: (opts={}) => - opts = defaults opts, - maxtime : 60 # (roughly) maximum amount of time to try to restart - cb : undefined - if @readonly - opts.cb?(); return - - if opts.maxtime <= 0 - opts.cb?("timed out trying to start Sage worksheet - system may be heavily loaded or Sage is broken.") - return - - timeout = 0.5 - f = (cb) => - timeout = Math.min(10, 1.4*timeout) - @introspect_line - line : "return?" - timeout : timeout - preparse : false - cb : (resp) => - cb() - - misc.retry_until_success - f : f - max_time : opts.maxtime*1000 - cb : opts.cb - - restart: (opts) => - opts = defaults opts, - cb : undefined - @sage_call - input : {event:'restart'} - cb : => - @execute_auto_cells() - opts.cb?() - - send_signal: (opts) => - opts = defaults opts, - signal : 2 - cb : undefined - @sage_call - input : {event:'signal', signal:opts.signal} - cb : () => opts.cb?() - - introspect_line: (opts) => - opts = defaults opts, - line : required - top : undefined - preparse : true - timeout : undefined - cb : required - @sage_call - input : - event : 'introspect' - line : opts.line - top : opts.top - preparse : opts.preparse - cb : opts.cb - - introspect: () => - if @opts.static_viewer - return - if @readonly - return - # TODO: obviously this wouldn't work in both sides of split worksheet. - cm = @focused_codemirror() - pos = cm.getCursor() - cib = @current_input_block(pos.line) - if cib.start == cib.end - toplineno = cib.start - else - toplineno = cib.start + 1 - # added topline for jupyter decorator autocompletion - topline = cm.getLine(toplineno) - line = cm.getLine(pos.line).slice(0, pos.ch) - if pos.ch == 0 or line[pos.ch-1] in ")]}'\"\t " - if @editor.opts.spaces_instead_of_tabs - cm.tab_as_space() - else - CodeMirror.commands.defaultTab(cm) - return - @introspect_line - line : line - top : topline - cb : (mesg) => - if mesg.event == "error" or not mesg?.target? # some other sort of error, e.g., mesg = 'some error' ? - # First, there is no situation I can think of where this happens... though - # of course it does. - # Showing user an alert_message at this point isn't useful; but we do want to know - # about this. The user is just going to see no completion or popup, which is - # possibly reasonable behavior from their perspective. - # NOTE: we do get mesg.event not error, but mesg.target isn't defined: see https://github.com/sagemathinc/cocalc/issues/1685 - err = "sagews: unable to instrospect '#{line}' -- #{JSON.stringify(mesg)}" - console.log(err) # this is intentional... -- it's may be useful to know - webapp_client.tracking_client.log_error(err) - return - else - from = {line:pos.line, ch:pos.ch - mesg.target.length} - elt = undefined - switch mesg.event - when 'introspect_completions' - cm.showCompletions - from : from - to : pos - completions : mesg.completions - target : mesg.target - completions_size : @editor.opts.completions_size - - when 'introspect_docstring' - elt = cm.showIntrospect - from : from - content : mesg.docstring - target : mesg.target - type : "docstring" - - when 'introspect_source_code' - elt = cm.showIntrospect - from : from - content : mesg.source_code - target : mesg.target - type : "source-code" - - else - console.log("BUG -- introspect_line -- unknown event #{mesg.event}") - if elt? - @close_on_action(elt) - - elt_at_mark: (mark) => - return mark?.element - - cm_wrapper: () => - if @_cm_wrapper? - return @_cm_wrapper - return @_cm_wrapper = $(@codemirror.getWrapperElement()) - - cm_lines: () => - if @_cm_lines? - return @_cm_lines - return @_cm_lines = @cm_wrapper().find(".CodeMirror-lines") - - pad_bottom_with_newlines: (n) => - if @opts.static_viewer - return - cm = @codemirror - m = cm.lineCount() - if m <= 13 # don't bother until worksheet gets big - return - j = m-1 - while j >= 0 and j >= m-n and cm.getLine(j).length == 0 - j -= 1 - k = n - (m - (j + 1)) - if k > 0 - cursor = cm.getCursor() - cm.replaceRange(Array(k+1).join('\n'), {line:m+1, ch:0} ) - cm.setCursor(cursor) - - # change the codemirror editor to reflect the proper sagews worksheet markup. - process_sage_updates: (opts={}) => - opts = defaults opts, - start : undefined # process starting at this line (0-based); 0 if not given - stop : undefined # end at this line (0-based); last line if not given - cm : undefined # only markup changes, etc., using the given editor (uses all visible ones by default) - pad_bottom : 10 # ensure there are this many blank lines at bottom of document - caller : undefined - if @_closed - return - #console.log("process_sage_updates", @readonly, opts.caller) - # For each line in the editor (or starting at line start), check if the line - # starts with a cell or output marker and is not already marked. - # If not marked, mark it appropriately, and possibly process any - # changes to that line. - #tm = misc.mswalltime() - before = @editor.codemirror.getValue() - if opts.pad_bottom - @pad_bottom_with_newlines(opts.pad_bottom) - try - if not opts.cm? - @_process_sage_updates(@editor.codemirror, opts.start, opts.stop) - if @editor._layout > 0 - @_process_sage_updates(@editor.codemirror1, opts.start, opts.stop) - else - @_process_sage_updates(opts.cm, opts.start, opts.stop) - catch e - console.log("Error rendering worksheet", e) - - #console.log("process_sage_updates(opts=#{misc.to_json({caller:opts.caller, start:opts.start, stop:opts.stop})}): time=#{misc.mswalltime(tm)}ms") - - after = @editor.codemirror.getValue() - if before != after and not @readonly - @_syncstring.from_str(after) - @_syncstring.save() - - _process_sage_updates: (cm, start, stop) => - #dbg = (m) -> console.log("_process_sage_updates: #{m}") - #dbg("start=#{start}, stop=#{stop}") - if @_closed - return - if not cm? - cm = @focused_codemirror() - if not start? - start = 0 - if not stop? - stop = cm.lineCount()-1 - context = {uuids:{}} - for line in [start..stop] - @_process_line(cm, line, context) - - _handle_input_cell_click0: (e, mark) => - @insert_new_cell(mark.find()?.from.line) - - _handle_input_cell_click: (e, mark) => - if IS_TOUCH - # It is way too easy to accidentally click on the insert new cell line on mobile. - bootbox.confirm "Create new cell?", (result) => - if result - @_handle_input_cell_click0(e, mark) - else # what the user really wants... - cm = @focused_codemirror() - cm.focus() - cm.setCursor({line:mark.find().from.line+1, ch:0}) - else - @_handle_input_cell_click0(e, mark) - return false - - # Process the codemirror gutter local SMC line number and input/output toggles - # cm = codemirror editor - # line = the line number (0 based) - # mode = the mode for the line: 'show', 'hide', 'number' - # relative_line = the relative line number - _process_line_gutter: (cm, line, mode, relative_line) => - # nb: I did implement a non-jQuery version of this function; the speed was *identical*. - want = mode + relative_line # we want the HTML node to have these params. - elt = cm.lineInfo(line).gutterMarkers?['smc-sagews-gutter-hide-show'] - if elt?.smc_cur == want # gutter is already defined and set as desired. - return - switch mode - when 'show' - # A show toggle triangle - elt = open_gutter_elt.clone()[0] - when 'hide' - # A hide triangle - elt = folded_gutter_elt.clone()[0] - when 'number' - # A line number - if not @editor.opts.line_numbers - # Ignore because line numbers are disabled - return - if elt?.className == '' - # Gutter elt is already a plain div, so just chnage innerHTML - elt.smc_cur = want - elt.innerHTML = relative_line - return - # New gutter element - elt = line_number_elt.clone().text(relative_line)[0] - else - console.warn("sagews unknown mode '#{mode}'") - if elt? - elt.smc_cur = want # elt will have this mode/line - # Now set it. - cm.setGutterMarker(line, 'smc-sagews-gutter-hide-show', elt) - - _process_line: (cm, line, context) => - ### - - Ensure that cell start line is properly marked so it looks like a horizontal - line, which can be clicked, and is colored to indicate state. - - - Ensure that cell output line is replaced by an output element, with the proper - output rendered in it. - ### - x = cm.getLine(line) - if not x? - return - - marks = (m for m in cm.findMarks({line:line, ch:0}, {line:line,ch:x.length}) when m.type != 'bookmark') - if marks.length > 1 - # There should never be more than 1 mark on a line - for m in marks.slice(1) - m.clear() - marks = [marks[0]] - - switch x[0] - when MARKERS.cell - uuid = x.slice(1, 37) - if context.uuids[uuid] - # seen this before -- so change it - uuid = misc.uuid() - cm.replaceRange(uuid, {line:line, ch:1}, {line:line, ch:37}) - context.uuids[uuid] = true - context.input_line = line - flagstring = x.slice(37, x.length-1) - - if FLAGS.hide_input in flagstring - @_process_line_gutter(cm, line, 'hide') - context.hide = line - else - @_process_line_gutter(cm, line, 'show') - delete context.hide - - # Record whether or not the output for this cell should be hidden. - context.hide_output = FLAGS.hide_output in flagstring - - # Determine the output line, if available, so we can toggle whether or not - # the output is hidden. Note that we are not doing something based on - # state change, as that is too hard to reason about, and are just always - # setting the line classes properly. This will have to be re-done someday. - n = line + 1 - output_line = undefined - while n < cm.lineCount() - z = cm.getLine(n) - if z?[0] == MARKERS.output - output_line = n - break - if z?[0] == MARKERS.input or not z?[0]? - break - n += 1 - - if output_line? # found output line -- properly set hide state - output_marks = cm.findMarks({line:output_line, ch:0}, {line:output_line, ch:z.length}) - if context.hide_output - @_process_line_gutter(cm, output_line, 'hide') - output_marks?[0]?.element.hide() - else - @_process_line_gutter(cm, output_line, 'show') - output_marks?[0]?.element.show() - - - if marks.length == 1 and (marks[0].type != 'input' or marks[0].uuid != uuid) - marks[0].clear() - marks = [] - if marks.length == 0 - # create the input mark here - #console.log("creating input mark at line #{line}") - input = cell_start_template.clone() - opts = - shared : false - inclusiveLeft : false - inclusiveRight : true - atomic : true - replacedWith : input[0] #$("
")[0] - mark = cm.markText({line:line, ch:0}, {line:line, ch:x.length}, opts) - marks.push(mark) - mark.element = input - mark.type = 'input' - mark.uuid = uuid - if not @readonly - input.addClass('sagews-input-live') - input.click((e) => @_handle_input_cell_click(e, mark)) - - if not @readonly - elt = marks[0].element - if FLAGS.waiting in flagstring - elt.data('execute',FLAGS.waiting) - @set_input_state(elt:elt, run_state:'waiting') - else if FLAGS.execute in flagstring - elt.data('execute',FLAGS.execute) - @set_input_state(elt:elt, run_state:'execute') - else if FLAGS.running in flagstring - elt.data('execute',FLAGS.running) - @set_input_state(elt:elt, run_state:'running') - else - # code is not running - elt.data('execute','done') - @set_input_state(elt:elt, run_state:'done') - # set marker of whether or not this cell was evaluated during this session - if FLAGS.this_session in flagstring - @set_input_state(elt:elt, eval_state:true) - else - @set_input_state(elt:elt, eval_state:false) - - - when MARKERS.output - - uuid = x.slice(1,37) - if context.uuids[uuid] - # seen this id before (in a previous cell!) -- so change it - uuid = misc.uuid() - cm.replaceRange(uuid, {line:line, ch:1}, {line:line, ch:37}) - context.uuids[uuid] = true - if marks.length == 1 and (marks[0].type != 'output' or marks[0].uuid != uuid) - marks[0].clear() - marks = [] - if marks.length == 0 - # create the output mark here - #console.log("creating output mark at line #{line}") - output = output_template.clone() - opts = - shared : false - inclusiveLeft : true - inclusiveRight : true - atomic : true - replacedWith : output[0] - mark = cm.markText({line:line, ch:0}, {line:line, ch:x.length}, opts) - mark.element = output - mark.type = 'output' - mark.uuid = uuid - mark.element.data('uuid',mark.uuid) - mark.rendered = '' - marks.push(mark) - - cm.addLineClass(line, 'gutter', 'sagews-output-cm-gutter') - cm.addLineClass(line, 'text', 'sagews-output-cm-text') - cm.addLineClass(line, 'wrap', 'sagews-output-cm-wrap') - - # To be sure, definitely properly set output state (should already be properly set when rendering input) - if context.hide_output - @_process_line_gutter(cm, line, 'hide') - marks[0].element.hide() - else - @_process_line_gutter(cm, line, 'show') - marks[0].element.show() - - @render_output(marks[0], x.slice(38), line) - - else - if @editor.opts.line_numbers - input_line = context.input_line - if not input_line? - input_line = line - 1 - while input_line >= 0 and cm.getLine(input_line)[0] != MARKERS.cell - input_line -= 1 - @_process_line_gutter(cm, line, 'number', line - input_line) # relative line number - else - @_process_line_gutter(cm, line) - - for b in [MARKERS.cell, MARKERS.output] - i = x.indexOf(b) - if i != -1 - cm.replaceRange('', {line:line,ch:i}, {line:line, ch:x.length}) - x = x.slice(0, i) - - if context.hide? - if marks.length > 0 and marks[0].type != 'hide' - marks[0].clear() - marks = [] - if marks.length == 0 and context.hide == line - 1 - opts = - shared : false - inclusiveLeft : true - inclusiveRight : true - atomic : true - collapsed : true - end = line+1 - while end < cm.lineCount() - if cm.getLine(end)[0] != MARKERS.output - end += 1 - else - break - mark = cm.markText({line:line, ch:0}, {line:end-1, ch:cm.getLine(end-1).length}, opts) - mark.type = 'hide' - #console.log("hide from #{line} to #{end}") - else - #console.log("line #{line}: No marks since line doesn't begin with a marker and not hiding") - if marks.length > 0 - for m in marks - m.clear() - - render_output: (mark, s, line) => - if mark.rendered == s - return - if s.slice(0, mark.rendered.length) != mark.rendered - mark.element.empty() - mark.element.data('stderr','') - mark.rendered = '' - for m in s.slice(mark.rendered.length).split(MARKERS.output) - if m.length == 0 - continue - try - mesg = misc.from_json(m) - catch e - #console.warn("invalid output message '#{m}' in line '#{s}' on line #{line}") - return - @process_output_mesg - mesg : mesg - element : mark.element - mark : mark - mark.rendered = s - - ################################################################################## - # Toggle visibility of input/output portions of cells - - # This is purely a client-side display function; it doesn't change - # the document or cause any sync to happen! - ################################################################################## - - set_input_state: (opts) => - opts = defaults opts, - elt : undefined - line : undefined - eval_state : undefined # undefined, true, false - run_state : undefined # undefined, 'execute', 'running', 'waiting', 'done' - #console.log("set_input_state", opts) - if opts.elt? - elt = opts.elt - else if opts.line? - mark = cm.findMarksAt({line:opts.line, ch:1})[0] - if not mark? - return - elt = @elt_at_mark(mark) - if opts.eval_state? - e = elt.find(".sagews-input-eval-state") - if opts.eval_state - e.addClass('sagews-input-evaluated').removeClass('sagews-input-unevaluated') - else - e.addClass('sagews-input-unevaluated').removeClass('sagews-input-evaluated') - if opts.run_state? - e = elt.find(".sagews-input-run-state") - for k in ['execute', 'running', 'waiting'] - e.removeClass("sagews-input-#{k}") - if opts.run_state == 'done' - e.removeClass('blink') - else - e.addClass("sagews-input-#{opts.run_state}").addClass('blink') - - # hide_input: hide input part of cell that has start marker at the given line. - hide_input: (line) => - end = line+1 - cm = @codemirror - while end < cm.lineCount() - c = cm.getLine(end)[0] - if c == MARKERS.cell or c == MARKERS.output - break - end += 1 - - line += 1 - - #hide = $("
") - opts = - shared : true - inclusiveLeft : true - inclusiveRight : true - atomic : true - #replacedWith : hide[0] - collapsed : true - marker = cm.markText({line:line, ch:0}, {line:end-1, ch:cm.getLine(end-1).length}, opts) - marker.type = 'hide_input' - #console.log("hide_input: ", {line:line, ch:0}, {line:end-1, ch:cm.getLine(end-1).length}) - for c in @codemirrors() - c.refresh() - - show_input: (line) => - for cm in [@codemirror, @codemirror1] - for marker in cm.findMarksAt({line:line+1, ch:0}) - if marker.type == 'hide_input' - marker.clear() - - hide_output: (line) => - for cm in [@codemirror, @codemirror1] - mark = @find_output_mark(line, cm) - if mark? - @elt_at_mark(mark).addClass('sagews-output-hide').find(".sagews-output-container").hide() - - show_output: (line) => - for cm in [@codemirror, @codemirror1] - mark = @find_output_mark(line, cm) - if mark? - @elt_at_mark(mark).removeClass('sagews-output-hide').find(".sagews-output-container").show() - - sage_call: (opts) => - opts = defaults opts, - input : required - cb : undefined - if @readonly - opts.cb?({done:true, error:'readonly'}) - return - if not @_syncstring?.evaluator? - opts.cb?({done:true, error:'closed'}) - return - @_syncstring.evaluator.call - program : 'sage' - input : opts.input - cb : opts.cb - return - - status: (opts) => - opts = defaults opts, - cb : required - @sage_call - input : - event : 'status' - cb : (resp) => - if resp.event == 'error' - opts.cb(resp.error) - else - opts.cb(undefined, resp) - - execute_code: (opts) => - opts = defaults opts, - code : required - cb : undefined - data : undefined - preparse : true - id : misc.uuid() - output_uuid : opts.output_uuid - timeout : undefined - @sage_call - input : - event : 'execute_code' - code : opts.code - data : opts.data - preparse : opts.preparse - id : opts.id - output_uuid : opts.output_uuid - timeout : opts.timeout - cb : opts.cb - return opts.id - - interact: (output, desc, mark) => - # Create and insert DOM objects corresponding to this interact - elt = $("
") - interact_elt = $("") - elt.append(interact_elt) - output.append(elt) - - if @readonly - interact_elt.text("(interacts not available)").addClass('lighten') - return - - f = (opts) => - opts.mark = mark - @process_output_mesg(opts) - - # Call jQuery plugin to make it all happen. - interact_elt.sage_interact - desc : desc - execute_code : @execute_code - process_output_mesg : f - process_html_output : @process_html_output - - jump_to_output_matching_jquery_selector: (selector) => - cm = @focused_codemirror() - for x in cm.getAllMarks() - t = $(x.replacedWith).find(selector) - if t.length > 0 - cm.scrollIntoView(x.find().from, cm.getScrollInfo().clientHeight/2) - return - - process_html_output: (e) => - # TODO: when redoing this using react, see the Markdown component in components - # and the process_smc_links jQuery plugin in misc_page.coffee - # makes tables look MUCH nicer - e.find("table").addClass('table') - - # handle a links - a = e.find('a') - - that = @ - for x in a - y = $(x) - href = y.attr('href') - if href? - if href[0] == '#' - # target is internal anchor to id - # make internal links in the same document scroll the target into view. - y.click (e) -> - that.jump_to_output_matching_jquery_selector($(e.target).attr('href')) - return false - else if href.indexOf(document.location.origin) == 0 - # target starts with cloud URL or is absolute, so we open the - # link directly inside this browser tab - y.click (e) -> - n = (document.location.origin + '/projects/').length - target = $(@).attr('href').slice(n) - redux.getActions('projects').load_target(decodeURI(target), not(e.which==2 or (e.ctrlKey or e.metaKey))) - return false - else if href.indexOf('http://') != 0 and href.indexOf('https://') != 0 - # internal link - y.click (e) -> - target = $(@).attr('href') - {join} = require('path') - if target.indexOf('/projects/') == 0 - # fully absolute (but without https://...) - target = decodeURI(target.slice('/projects/'.length)) - else if target[0] == '/' and target[37] == '/' and misc.is_valid_uuid_string(target.slice(1,37)) - # absolute path with /projects/ omitted -- /..project_id../files/.... - target = decodeURI(target.slice(1)) # just get rid of leading slash - else if target[0] == '/' - # absolute inside of project - target = join(that.project_id, 'files', decodeURI(target)) - else - # relative to current path - target = join(that.project_id, 'files', that.file_path(), decodeURI(target)) - redux.getActions('projects').load_target(target, not(e.which==2 or (e.ctrlKey or e.metaKey))) - return false - else - # make links open in a new tab - a.attr("target","_blank") - - # scale images - e.smc_image_scaling() - - # make relative links to images use the raw server - a = e.find("img") - for x in a - y = $(x) - src = y.attr('src') - # checking, if we need to fix the src path - is_fullurl = src.indexOf('://') != -1 - is_blob = misc.startswith(src, join(appBasePath, 'blobs/')) - # see https://github.com/sagemathinc/cocalc/issues/651 - is_data = misc.startswith(src, 'data:') - if is_fullurl or is_data or is_blob - continue - # see https://github.com/sagemathinc/cocalc/issues/1184 - file_path = @file_path() - if misc.startswith(src, '/') - file_path = ".smc/root/#{file_path}" - new_src = join(appBasePath, @project_id, 'raw', file_path, src) - y.attr('src', new_src) - - _post_save_success: () => - console.log("_post_save_success") - @remove_output_blob_ttls() - - # Return array of uuid's of blobs that might possibly be in the worksheet - # and have a ttl. - _output_blobs_with_possible_ttl: () => - v = [] - x = @_output_blobs_with_possible_ttl_done ?= {} - for c in @get_all_cells() - for output in c.output() - if output.file? - uuid = output.file.uuid - if uuid? - if not x[uuid] - v.push(uuid) - return v - - # mark these as having been successfully marked to never expire. - _output_blobs_ttls_removed: (uuids) => - for uuid in uuids - @_output_blobs_with_possible_ttl_done[uuid] = true - - remove_output_blob_ttls: (cb) => - # TODO: prioritize automatic testing of this highly... since it is easy to break by changing - # how worksheets render slightly. - uuids = @_output_blobs_with_possible_ttl() - console.log("remove_output_blob_ttls -- ", uuids) - if uuids? - try - await webapp_client.conat_client.hub.db.removeBlobTtls({uuids:uuids}) - catch err - console.log("WARNING: problem removing ttl from sage worksheet blobs ", err) - cb?(err) - return - # don't try again to remove ttls for these blobs -- since did so successfully - @_output_blobs_ttls_removed(uuids) - - raw_input: (raw_input) => - prompt = raw_input.prompt - value = raw_input.value - if not value? - value = '' - submitted = !!raw_input.submitted - elt = templates.find(".sagews-output-raw_input").clone() - label = elt.find(".sagews-output-raw_input-prompt") - label.text(prompt) - input = elt.find(".sagews-output-raw_input-value") - input.val(value) - - if raw_input.placeholder? - input.attr('placeholder', raw_input.placeholder) - - btn = elt.find(".sagews-output-raw_input-submit") - - if submitted or @readonly - btn.addClass('disabled') - input.attr('readonly', true) - else - submit_raw_input = () => - btn.addClass('disabled') - input.attr('readonly', true) - for cm in @codemirrors() - cm.setOption('readOnly',@readonly) - @sage_call - input : - event : 'raw_input' - value : input.val() - - input.keyup (evt) => - # if return, submit result - if evt.which == 13 - submit_raw_input() - - btn.click () => - submit_raw_input() - return false - - f = () => - input.focus() - setTimeout(f, 50) - - if raw_input.input_width? - input.width(raw_input.input_width) - - if raw_input.label_width? - label.width(raw_input.label_width) - - return elt - - process_output_mesg: (opts) => - opts = defaults opts, - mesg : required - element : required - mark : undefined - mesg = opts.mesg - output = opts.element - # mesg = object - # output = jQuery wrapped element - if mesg.stdout? - output.append($("").text(mesg.stdout)) - - if mesg.stderr? - # This is entirely for the ChatGPT help buttons: - cur = output.data('stderr') - if not cur - buttonsContainer = $("
") - - # Add hint button if enabled - if chatgpt.isHintEnabled(@project_id) - hintButton = $("Give me a hint...") - hintButton.click () => - chatgpt.giveMeAHint - codemirror : @focused_codemirror() - stderr : output.data('stderr') - uuid : output.data('uuid') - project_id : @project_id - path : @filename - buttonsContainer.append(hintButton) - - # Add solution button if enabled - if chatgpt.isEnabled(@project_id) - solutionButton = $("Help me fix this...") - solutionButton.click () => - chatgpt.helpMeFix - codemirror : @focused_codemirror() - stderr : output.data('stderr') - uuid : output.data('uuid') - project_id : @project_id - path : @filename - buttonsContainer.append(solutionButton) - - # Only append container if it has buttons - if buttonsContainer.children().length > 0 - output.append(buttonsContainer) - output.data('stderr', mesg.stderr) - else - output.data('stderr', cur + mesg.stderr) - - output.append($("").text(mesg.stderr)) - - if mesg.error? - error = """ERROR: '#{JSON.stringify(mesg.error)}' - - Communication with the Sage server is failing. - - Here are some actions you could try to resolve this problem: - - check your internet connection, - - run this cell again, - - close and reopen this file, - - restart the project (in project settings, wrench icon), - - reload the browser tab or even restart your browser, - - delete some of the content in the project's ~/.local directory, - (locally installed Python libraries might interfere with running this worksheet) - """ - output.append($("").text(error)) - - if mesg.code? - x = $("
") - output.append(x) - if mesg.code.mode - CodeMirror.runMode(mesg.code.source, mesg.code.mode, x[0]) - else - x.text(mesg.code.source) - - if mesg.html? - e = $("
") - if @editor.opts.allow_javascript_eval - e.html(mesg.html) - else - e.html_noscript(mesg.html) - e.katex({preProcess:true}) - output.append(e) - @process_html_output(e) - - if mesg.interact? - @interact(output, mesg.interact, opts.mark) - - if mesg.d3? - e = $("
") - output.append(e) - await import("./d3") - e.d3 - viewer : mesg.d3.viewer - data : mesg.d3.data - - if mesg.md? - # markdown - # we replace all backslashes by double backslashes since bizarely the markdown-it processes replaces \$ with $, which - # breaks later use of mathjax :-(. This will get deleted soon. - html = markdown.markdown_to_html(mesg.md) - t = $('
') - if @editor.opts.allow_javascript_eval - t.html(html) - else - t.html_noscript(html) - t.katex() - output.append(t) - @process_html_output(t) - - if mesg.tex? - # latex - val = mesg.tex - if val.display - delim = '$$' - else - delim = '$' - html = markdown.markdown_to_html(delim + val.tex + delim) - t = $("
") - t.html(html) - t.find('span.cocalc-katex-error').katex() - output.append(t) - - if mesg.raw_input? - output.append(@raw_input(mesg.raw_input)) - - if mesg.file? - val = mesg.file - if val.uuid? - blobs = opts.element.data('blobs') - if not blobs? - blobs = [val.uuid] - opts.element.data('blobs', blobs) - else - blobs.push(val.uuid) - - if not val.show? or val.show - if val.url? - target = val.url + "?nocache=#{Math.random()}" # randomize to dis-allow caching, since frequently used for images with one name that changes - else - target = join(appBasePath, "blobs", "#{misc.encode_path(val.filename)}?uuid=#{val.uuid}") - switch misc.filename_extension(val.filename).toLowerCase() - # TODO: harden DOM creation below? - - when 'webm' - if $.browser.safari or $.browser.ie - output.append($("
WARNING: webm animations not supported on Safari or IE; use an animated gif instead, e.g., the gif=True option to show.
")) - if $.browser.firefox - output.append($("
WARNING: Right click and select play.
")) - video = $("") - output.append(video) - - when 'sage3d' - elt = $("
") - elt.data('uuid',val.uuid) - output.append(elt) - {render_3d_scene} = await import("./3d") - render_3d_scene - url : target - element : elt - cb : (err, obj) => - if err - # TODO: red? - elt.append($("
").text("error rendering 3d scene -- #{err}")) - else - elt.data('width', obj.opts.width / $(window).width()) - - when 'svg', 'png', 'gif', 'jpg', 'jpeg' - img = $("
") - output.append(img) - - if mesg.events? - img.css(cursor:'crosshair') - location = (e) -> - offset = img.offset() - x = (e.pageX - offset.left) /img.width() - y = (e.pageY - offset.top) /img.height() - return [x,y] - - exec = (code) => - @execute_code - code : code - preparse : true - cb : (mesg) => - delete mesg.done - @process_output_mesg - mesg : mesg - element : output.find(".sagews-output-messages") - mark : opts.mark - - for event, function_name of mesg.events - img.data("webapp-events-#{event}", function_name) - switch event - when 'click' - img.click (e) => - p = location(e) - exec("#{img.data('webapp-events-click')}('click',(#{p}))") - when 'mousemove' - ignore_mouse_move = undefined - last_pos = undefined - img.mousemove (e) => - if ignore_mouse_move? - return - ignore_mouse_move = true - setTimeout( ( () => ignore_mouse_move=undefined ), 100 ) - p = location(e) - if last_pos? and p[0] == last_pos[0] and p[1] == last_pos[1] - return - last_pos = p - exec("#{img.data('webapp-events-mousemove')}('mousemove',(#{p}))") - else - console.log("unknown or unimplemented event -- #{event}") - - else - if val.text - text = val.text - else - text = "#{val.filename} (temporary link)" - output.append($("#{text} ")) - - if mesg.javascript? and @allow_javascript_eval() - code = mesg.javascript.code - if mesg.obj? - obj = JSON.parse(mesg.obj) - else - obj = undefined - if mesg.javascript.coffeescript - t = $('
') - t.html(''' - - ''') - output.append(t) - else - # The eval below is an intentional cross-site scripting vulnerability - # in the fundamental design of CoCalc. - # Note that there is an allow_javascript document option, which (at some point) users - # will be able to set. There is one more instance of eval below in _receive_broadcast. - sagews_eval(code, @, opts.element, undefined, obj, redux) - - if mesg.show? - if opts.mark? - line = opts.mark.find()?.from.line - if line? - cell = @cell(line) - if cell? - switch mesg.show - when 'input' - cell.remove_cell_flag(FLAGS.hide_input) - when 'output' - cell.remove_cell_flag(FLAGS.hide_output) - - if mesg.hide? - if opts.mark? - line = opts.mark.find()?.from.line - if line? - cell = @cell(line) - if cell? - switch mesg.hide - when 'input' - cell.set_cell_flag(FLAGS.hide_input) - when 'output' - cell.set_cell_flag(FLAGS.hide_output) - - # NOTE: Right now the "state object" is a just a list of messages in the output of a cell. - # It's viewed as something that should get rendered in order, with no dependence between them. - # Instead all thoose messages should get fed into one single state object, which then gets - # rendered each time it changes. React makes that approach easy and efficient. Without react - # (or something similar) it is basically impossible. When sage worksheets are rewritten - # using react, this will change. - if mesg.clear - output.empty() - output.data('stderr','') - - if mesg.delete_last - output.find(":last-child").remove() - - if mesg.done - output.removeClass('sagews-output-running') - output.addClass('sagews-output-done') - - allow_javascript_eval: () => - # TODO: Maybe better would be a button to click that re-renders - # with javascript enabled...? - if not @editor.opts.allow_javascript_eval - @javascript_block_mesg() - return false - else - return true - - javascript_block_mesg: () => - if @_javascript_block_mesg - return - @_javascript_block_mesg = true - alert_message - type : "info" - message : "Evaluation of arbitrary javascript is blocked in public worksheets, since it is dangerous; instead, open a copy of this worksheet in your own project." - timeout : 10 - - _receive_broadcast: (mesg) => - switch mesg.mesg.event - when 'execute_javascript' - if @allow_javascript_eval() - mesg = mesg.mesg - do () => - code = mesg.code - obj = JSON.parse(mesg.obj) - sagews_eval(code, @, undefined, mesg.cell_id, obj, redux) - - mark_cell_start: (cm, line) => - # Assuming the proper text is in the document for a new cell at this line, - # mark it as such. This hides control codes and places a cell separation - # element, which may be clicked to create a new cell. - if line >= cm.lineCount()-1 - # If at bottom, insert blank lines. - cm.replaceRange("\n\n\n", {line:line+1, ch:0}) - x = cm.getLine(line) - end = x.indexOf(MARKERS.cell, 1) - input = cell_start_template.clone() - if not @readonly - input.addClass('sagews-input-live') - input.click (e) => - f = () => - line = mark.find().from.line - @insert_new_cell(line) - if e.shiftKey - cm.replaceRange("%html\n", {line:line+1,ch:0}) - @action - execute : true - advance : false - if (e.altKey or e.metaKey) - cm.replaceRange("%md\n", {line:line+1,ch:0}) - @action - execute : true - advance : false - - if IS_TOUCH - # It is way too easy to accidentally click on the insert new cell line on mobile. - bootbox.confirm "Create new cell?", (result) => - if result - f() - else # what the user really wants... - cm.focus() - cm.setCursor({line:mark.find().from.line+1, ch:0}) - else - f() - return false - - opts = - shared : false - inclusiveLeft : false # CRITICAL: do not set this to true; it screws up undo/redo badly (maybe with undo/redo based on syncstring this will be fine again) - inclusiveRight : true - atomic : true - replacedWith : input[0] #$("
")[0] - - mark = cm.markText({line:line, ch:0}, {line:line, ch:end+1}, opts) - mark.type = MARKERS.cell - mark.element = input - return mark - - set_output_line_class: (line, check=true) => - #console.log("set_output_line_class #{line}") - for c in @codemirrors() - if check - info = c.lineInfo(line) - if not info? or info.textClass? - return - c.addLineClass(line, 'gutter', 'sagews-output-cm-gutter') - c.addLineClass(line, 'text', 'sagews-output-cm-text') - c.addLineClass(line, 'wrap', 'sagews-output-cm-wrap') - - set_all_output_line_classes: => - for cm in @codemirrors() - for m in cm.getAllMarks() - if m.type == MARKERS.output - line = m.find()?.from.line - if line? and not cm.lineInfo(line)?.textClass? - @set_output_line_class(line, false) - - mark_output_line: (cm, line) => - # Assuming the proper text is in the document for output to be displayed at this line, - # mark it as such. This hides control codes and creates a div into which output will - # be placed as it appears. - #console.log("mark_output_line, #{line}") - - @set_output_line_class(line) - - output = output_template.clone() - - if cm.lineCount() < line + 2 - cm.replaceRange('\n', {line:line+1,ch:0}) - start = {line:line, ch:0} - end = {line:line, ch:cm.getLine(line).length} - opts = - shared : false - inclusiveLeft : true - inclusiveRight : true - atomic : true - replacedWith : output[0] - # NOTE: I'm using markText, which is supposed to only be used inline, no divs, but I should be - # using .addLineWidget. However, I had **WAY** too many problems with line widgets, whereas cheating - # and using markText works. So there. - mark = cm.markText(start, end, opts) - mark.element = output - # mark.processed stores how much of the output line we - # have processed [marker]36-char-uuid[marker] - mark.processed = 38 - mark.uuid = cm.getRange({line:line, ch:1}, {line:line, ch:37}) - mark.type = MARKERS.output - mark.element.data('uuid',mark.uuid) - - if not @readonly - output.click (e) => - t = $(e.target) - if t.attr('href')? or t.hasParent('.sagews-output-editor').length > 0 - return - @edit_cell - line : mark.find().from.line - 1 - cm : cm - - return mark - - find_output_line: (line, cm) => - # Given a line number in the editor, return the nearest (greater or equal) line number that - # is an output line, or undefined if there is no output line before the next cell. - if not cm? - cm = @focused_codemirror() - if cm.getLine(line)?[0] == MARKERS.output - return line - line += 1 - while line < cm.lineCount() - 1 - x = cm.getLine(line) - if x.length > 0 - if x[0] == MARKERS.output - return line - if x[0] == MARKERS.cell - return undefined - line += 1 - return undefined - - find_output_mark: (line, cm) => - # Same as find_output_line, but returns the actual mark (or undefined). - if not cm? - cm = @focused_codemirror() - n = @find_output_line(line, cm) - if n? - for mark in cm.findMarksAt({line:n, ch:0}) - if mark.type == MARKERS.output - return mark - return undefined - - # Returns start and end lines of the current input block (if line is undefined), - # or of the block that contains the given line number. This does not chnage - # the document. - current_input_block: (line) => - cm = @focused_codemirror() - if not line? - line = cm.getCursor().line - - start = end = line - - # if start is on an output line, move up one line - x = cm.getLine(start) - if x? and x.length > 0 and x[0] == MARKERS.output - start -= 1 - # if end is on a start line, move down one line - x = cm.getLine(end) - if x? and x.length > 0 and x[0] == MARKERS.cell - end += 1 - - while start > 0 - x = cm.getLine(start) - if x? and x.length > 0 and (x[0] == MARKERS.cell or x[0] == MARKERS.output) - if x[0] == MARKERS.output - start += 1 - break - start -= 1 - while end < cm.lineCount()-1 - x = cm.getLine(end) - if x? and x.length > 0 and (x[0] == MARKERS.cell or x[0] == MARKERS.output) - if x[0] == MARKERS.cell - end -= 1 - break - end += 1 - if end == cm.lineCount() - 1 - # end is the last line -- if empty, go back up to line after last non-empty line - while end > start and cm.getLine(end).trim().length == 0 - end -= 1 - return {start:start, end:end} - - find_input_mark: (line) => - # Input mark containing the given line, or undefined - if line? - cm = @focused_codemirror() - if not cm? - return - while line >= 0 - for mark in cm.findMarksAt({line:line, ch:0}) - if mark.type == MARKERS.cell - return mark - line -= 1 - return - - # HTML editor for the cell whose input starts at the given 0-based line. - edit_cell: (opts) => - opts = defaults opts, - line : required - cm : required - # DISABLED! - return - - action: (opts={}) => - opts = defaults opts, - pos : undefined - advance : false - split : false # split cell at cursor (selection is ignored) - execute : false # if false, do whatever else we would do, but don't actually execute code. - toggle_input : false # if true; toggle whether input is displayed; ranges all toggle same as first - toggle_output : false # if true; toggle whether output is displayed; ranges all toggle same as first - delete_output : false # if true; delete all the the output in the range - - #console.log 'action ', opts - - if @readonly - # don't do any actions on a read-only file. - return - - if opts.split - # split at every cursor position before doing any other actions - for sel in @focused_codemirror().listSelections().reverse() # "These will always be sorted, and never overlap (overlapping selections are merged)." - @split_cell_at(sel.head) - - if opts.execute or opts.toggle_input or opts.toggle_output or opts.delete_output - # do actions on cells containing cursors or overlapping with selections - if opts.pos? - cells = [@cell(opts.pos.line)] - else - create = opts.execute - cells = @get_current_cells(create) - for cell in cells - cell.action - execute : opts.execute - toggle_input : opts.toggle_input - toggle_output : opts.toggle_output - delete_output : opts.delete_output - if opts.toggle_output - # toggling output requires explicitly processing due to distance between input line where - # state is stored and output line where displayed. - @process_sage_updates({start:cell.start_line(), stop:cell.end_line()}) - if cells.length == 1 and opts.advance - @move_cursor_to_next_cell() - if cells.length > 0 - @save_state_debounce?() - - @close_on_action() # close introspect popups - - # purely client-side markdown rendering for a markdown, javascript, html, etc. block -- an optimization - execute_cell_client_side: (opts) => - opts = defaults opts, - cell : required - mode : undefined - code : undefined - hide : undefined - if not opts.mode? or not opts.code? - x = opts.cell.client_side() - if not x? # cell can't be executed client side -- nothing to do - return - opts.mode = x.mode; opts.code = x.code; opts.hide = x.hide - if not opts.hide? and opts.mode in ['md', 'html'] - opts.hide = false - if opts.hide - opts.cell.set_cell_flag(FLAGS.hide_input) - cur_height = opts.cell.get_output_height() - opts.cell.set_output_min_height(cur_height) - opts.cell.set_output([]) - mesg = {done:true} - switch opts.mode - when 'javascript' - mesg.javascript = {code: opts.code} - when 'coffeescript' - mesg.javascript = {coffeescript: true, code: opts.code} - when 'cjsx' - mesg.javascript = {cjsx: true, code: opts.code} - else - mesg[opts.mode] = opts.code - opts.cell.append_output_message(mesg) - setTimeout(opts.cell.set_output_min_height, 1000) - setTimeout(@sync, 1) - - execute_cell_server_side: (opts) => - opts = defaults opts, - cell : required - cb : undefined # called when the execution is completely done (so no more output) - @execution_queue.push(opts) - - _execute_cell_server_side: (opts) => - opts = defaults opts, - cell : required - cb : undefined # called when the execution is completely done (so no more output) - - #dbg = (m...) -> console.log("execute_cell_server_side:", m...) - dbg = () -> - - cell = opts.cell - input = cell.input() - - if not input? - dbg("cell vanished/invalid") - opts.cb?("cell vanished/invalid") - return - - cur_height = cell.get_output_height() - output_uuid = cell.new_output_uuid() - if not output_uuid? - dbg("output_uuid not defined") - opts.cb?("output_uuid no longer defined") - return - - # set cell to running mode - cell.set_cell_flag(FLAGS.running) - - # used to reduce flicker, which again makes things feel slow/awkward - cell.set_output_min_height(cur_height) - - done = => - cell.remove_cell_flag(FLAGS.running) - cell.set_cell_flag(FLAGS.this_session) - # wait a second, e.g., for async loading of images to finish - setTimeout(cell.set_output_min_height, 1000) - @sync() - opts.cb?() - delete opts.cb # to be sure not called again. - - t0 = new Date() - cleared_output = false - clear_output = => - if not cleared_output - cleared_output = true - cell.set_output([]) - - # Give the cell one second to get output from backend. - # If not, then we clear output. - # These reduces "flicker", which makes things seem slow. - setTimeout(clear_output, 1000) - first_output = true - @execute_code - code : input - output_uuid : output_uuid - cb : (mesg) => - if first_output - # we *always* clear the first time, even if we - # cleared above via the setTimeout. - first_output = false - clear_output() - cell.append_output_message(mesg) - if mesg.done - done?() - done = undefined - @sync() - - # enqueue all of the auto cells for execution - execute_auto_cells: () => - for cell in @get_all_cells() - is_auto = cell.is_auto() - if is_auto? and is_auto - cell.action(execute:true) - - split_cell_at: (pos) => - # Split the cell at the given pos. - @cell_start_marker(pos.line) - @sync() - - # returns the line number where the previous cell ends - move_cursor_to_next_cell: () => - cm = @focused_codemirror() - line = cm.getCursor().line + 1 - while line < cm.lineCount() - x = cm.getLine(line) - if x.length > 0 and x[0] == MARKERS.cell - cm.setCursor(line:line+1, ch:0) - return line-1 - line += 1 - # there is no next cell, so we create one at the last non-whitespace line - while line > 0 and $.trim(cm.getLine(line)).length == 0 - line -= 1 - @cell_start_marker(line+1) - cm.setCursor(line:line+2, ch:0) - return line - - ########################################## - # Codemirror-based cell manipulation code - # This is tightly tied to codemirror, so only makes sense on the client. - ########################################## - get_input_line_flagstring: (line) => - if not line? - return '' - cm = @focused_codemirror() - x = cm.getLine(line) - if not x? - return '' - if not misc.is_valid_uuid_string(x.slice(1,37)) - # worksheet is somehow corrupt - # TODO: should fix things at this point, or make sure this is never hit; could be caused by - # undo conflicting with updates. - return undefined - return x.slice(37,x.length-1) - - get_cell_flagstring: (marker) => - if not marker? - return undefined - pos = marker.find() - if not pos? - return '' - return @get_input_line_flagstring(pos.from.line) - - set_input_line_flagstring: (line, value) => - cm = @focused_codemirror() - x = cm.getLine(line) - if x? - cm.replaceRange(value, {line:line, ch:37}, {line:line, ch:x.length-1}) - - set_cell_flagstring: (marker, value) => - if not marker? - return - pos = marker.find() - if pos? - @focused_codemirror().replaceRange(value, {line:pos.from.line, ch:37}, {line:pos.to.line, ch:pos.to.ch-1}) - - get_cell_uuid: (marker) => - if not marker? - return - pos = marker.find() - if not pos? - return '' - return @focused_codemirror().getLine(pos.line).slice(1,38) - - set_cell_flag: (marker, flag) => - if not marker? - return - s = @get_cell_flagstring(marker) - if s? and flag not in s - @set_cell_flagstring(marker, flag + s) - - remove_cell_flag: (marker, flag) => - if not marker? - return - s = @get_cell_flagstring(marker) - if s? and flag in s - s = s.replace(new RegExp(flag, "g"), "") - @set_cell_flagstring(marker, s) - - insert_new_cell: (line) => - pos = {line:line, ch:0} - cm = cm = @focused_codemirror() - cm.replaceRange('\n', pos) - @process_sage_updates(start:line, stop:line+1, caller:"insert_new_cell") - cm.focus() - cm.setCursor(pos) - @cell_start_marker(line) - @process_sage_updates(start:line, stop:line+1, caller:"insert_new_cell") - @sync() - - cell_start_marker: (line) => - if not line? - throw Error("cell_start_marker: line must be defined") - cm = @focused_codemirror() - current_line = cm.getLine(line) - if not current_line? - # line no longer exists (got removed) - return - if current_line.length < 38 or current_line[0] != MARKERS.cell or current_line[current_line.length-1] != MARKERS.cell - # insert marker uuid text, since it isn't there already - uuid = misc.uuid() - cm.replaceRange(MARKERS.cell + uuid + MARKERS.cell + '\n', {line:line, ch:0}) - else - uuid = current_line.slice(1,37) - x = cm.findMarksAt(line:line, ch:0) - if x.length > 0 and x[0].type == MARKERS.cell - # already properly marked - return - if cm.lineCount() < line + 2 - # insert a newline - cm.replaceRange('\n',{line:line+1,ch:0}) - # this creates the mark itself: - @process_sage_updates(start:line, stop:line+1, caller:"cell_start_marker") - x = cm.findMarksAt(line:line, ch:0) - if x.length > 0 and x[0].type == MARKERS.cell - # now properly marked - return - else - # didn't get marked for some reason - return - - # map from uuids in document to true. - doc_uuids: () => - uuids = {} - @focused_codemirror().eachLine (z) -> - if z.text[0] == MARKERS.cell or z.text[0] == MARKERS.output - uuids[z.text.slice(1,37)] = true - return false - return uuids - - remove_this_session_from_line: (n) => - s = @get_input_line_flagstring(n) - if s? and FLAGS.this_session in s - s = s.replace(new RegExp(FLAGS.this_session, "g"), "") - @set_input_line_flagstring(n, s) - - remove_this_session_flags_from_range: (start, end) => - {start} = @current_input_block(start) - n = start - @codemirror.eachLine start, end+1, (line) => - if line.text[0] == MARKERS.cell - @remove_this_session_from_line(n) - n += 1 - return false - - remove_this_session_flags_from_changeObj_range: (changeObj) => - @remove_this_session_flags_from_range(changeObj.from.line, changeObj.to.line) - if changeObj.next? - @remove_cell_flags_from_changeObj(changeObj.next) - - remove_cell_flags_from_changeObj: (changeObj, flags, uuids) => - if not uuids? - uuids = @doc_uuids() - # Remove cell flags from *contiguous* text in the changeObj. - # This is useful for cut/copy/paste. - # This function modifies changeObj in place. - @remove_cell_flags_from_text(changeObj.text, flags, uuids) - if changeObj.next? - @remove_cell_flags_from_changeObj(changeObj.next, flags, uuids) - - remove_cell_flags_from_text: (text, flags, uuids) => - # !! The input "text" is an array of strings, one for each line; - # this function modifies this array in place. - # Replace all lines of the form - # [MARKERS.cell][36-character uuid][flags][MARKERS.cell] - # by - # [MARKERS.cell][uuid][flags2][MARKERS.cell] - # where flags2 has the flags in the second argument (an array) removed, - # or all flags removed if the second argument is undefined - for i in [0...text.length] - s = text[i] - if s.length >= 38 - if s[0] == MARKERS.cell - if flags? - text[i] = s.slice(0,37) + (x for x in s.slice(37,s.length-1) when x not in flags) + MARKERS.cell - else - text[i] = s.slice(0,37) + MARKERS.cell - if (s[0] == MARKERS.cell or s[0] == MARKERS.output) and uuids?[text[i].slice(1,37)] - text[i] = text[i][0] + misc.uuid() + text[i].slice(37) - - output_elements: () => - cm = @editor.codemirror - v = [] - if not cm? - # See https://github.com/sagemathinc/cocalc/issues/2070 - return v - for line in [0...cm.lineCount()] - marks = cm.findMarksAt({line:line, ch:1}) - if not marks? or marks.length == 0 - continue - for mark in marks - elt = mark.replacedWith - if elt? - elt = $(elt) - if elt.hasClass('sagews-output') - v.push(elt) - return v - - print_to_pdf_data: () => - data = {} - sage3d = data.sage3d = {} - - # Useful extra data about 3d plots (a png data url) - for elt in @output_elements() - for e in elt.find(".webapp-3d-container") - f = $(e) - scene = f.data('webapp-threejs') - if not scene? - continue - scene.set_static_renderer() - data_url = scene.static_image - if data_url? - uuid = f.data('uuid') - if not sage3d[uuid]? - sage3d[uuid] = [] - sage3d[uuid].push({'data-url':data_url, 'width':f.data('width')}) - - if misc.len(sage3d) == 0 - return undefined - - return data - - refresh_soon: (wait) => - if not wait? - wait = 1000 - if @_refresh_soon? - # We have already set a timer to do a refresh soon. - #console.log("not refresh_soon since -- We have already set a timer to do a refresh soon.") - return - do_refresh = () => - delete @_refresh_soon - for cm in [@codemirror, @codemirror1] - cm?.refresh() - @_refresh_soon = setTimeout(do_refresh, wait) - - close_on_action: (element) => - # Close popups (e.g., introspection) that are set to be closed when an - # action, such as "execute", occurs. - if element? - if not @_close_on_action_elements? - @_close_on_action_elements = [element] - else - @_close_on_action_elements.push(element) - else if @_close_on_action_elements? - for e in @_close_on_action_elements - e.remove() - @_close_on_action_elements = [] - -class ExecutionQueue - constructor: (@_exec, @worksheet) -> - if not @_exec - throw Error("BUG: execution function must be provided") - @_queue = [] - @_state = 'ready' - - close: () => - @dbg("close")() - @_state = 'closed' - delete @_queue - - dbg: (f) => - return () -> # disabled logging - #return (m...) -> console.log("ExecutionQueue.#{f}(): #{misc.to_json(m)}") - - push: (opts) => - opts = defaults opts, - cell : required - cb : undefined - @dbg("push")() - if @_state == 'closed' - return - uuid = opts.cell.start_uuid() - if not uuid? # removed - return - for x in @_queue - if x.cell.start_uuid() == uuid - return # cell already queued up to run - if uuid == @_running_uuid - # currently running - return - @_queue.push(opts) - opts.cell.set_cell_flag(FLAGS.waiting) - @_process() - - clear: () => - @dbg("clear")() - if @_state == 'closed' - return - # TODO/NOTE: this of course doesn't fully account for multiple users! - # E.g., two people start, one cancels, the other will still be - # queued up... But we have to start somewhere, to even know what - # state needs to be sync'd around. - for x in @_queue - x.cell.remove_cell_flag(FLAGS.waiting) - @_queue = [] - @_state = 'ready' - - _process: () => - if @_state == 'closed' - return - dbg = @dbg('process') - dbg() - if @worksheet._restarting - dbg("waiting for restart to finish") - @worksheet.once 'restarted', => - @_process() - return - if @_state == 'running' - dbg("running...") - return - dbg("length ", @_queue.length) - if @_queue.length == 0 - return - x = @_queue.shift() - uuid = x.cell.start_uuid() - if not uuid? - # cell no longer exists - @_process() - return - @_running_uuid = uuid - orig_cb = x.cb - x.cb = (args...) => - if @_state == 'closed' - # ignore further output - return - orig_cb?(args...) - @_state = 'ready' - delete @_running_uuid - @_process() - @_state = 'running' - x.cell.remove_cell_flag(FLAGS.waiting) - @_exec(x) - -class SynchronizedWorksheetCell - constructor: (@doc, start, end) -> - # Determine input and end lines of the cell that contains the given line, and - # the corresponding uuid's. - @cm = @doc.focused_codemirror() - - # Input - x = @cm.getLine(start) - if x?[0] == MARKERS.cell - if misc.is_valid_uuid_string(x.slice(1,37)) and x[x.length-1] == MARKERS.cell - # valid input line - @_start_uuid = x.slice(1,37) - else - # replace input line by valid one - @_start_uuid = misc.uuid() - @cm.replaceRange(MARKERS.cell + @_start_uuid + MARKERS.cell, {line:start, ch:0}, {line:start,ch:x.length}) - else - @_start_uuid = misc.uuid() - @cm.replaceRange(MARKERS.cell + @_start_uuid + MARKERS.cell + '\n', {line:start, ch:0}) - end += 1 - @_start_line = start - - # Output - x = @cm.getLine(end) - if x?[0] == MARKERS.output - if misc.is_valid_uuid_string(x.slice(1,37)) and x[37] == MARKERS.output - # valid output line - @_output_uuid = x.slice(1,37) - else - # replace output line by valid one - @_output_uuid = misc.uuid() - @cm.replaceRange(MARKERS.output + @_output_uuid + MARKERS.output, {line:end, ch:0}, {line:end, ch:x.length}) - @_end_line = end - else - @_output_uuid = misc.uuid() - s = MARKERS.output + @_output_uuid + MARKERS.output + '\n' - if @cm.lineCount() <= end+1 - # last line of document, so insert new empty line - s = '\n' + s - end += 1 - @cm.replaceRange(s, {line:end+1, ch:0}) - @_end_line = end+1 - - start_line: => - if (@cm.getLine(@_start_line)?.indexOf(@_start_uuid) ? -1) != -1 - return @_start_line - return @_start_line = @cm.find_in_line(@_start_uuid)?.line - - end_line: => - if (@cm.getLine(@_end_line)?.indexOf(@_start_uuid) ? -1) != -1 - return @_end_line - return @_end_line = @cm.find_in_line(@_output_uuid)?.line - - start_uuid: => - return @_start_uuid - - output_uuid: => - return @_output_uuid - - # generate a new random output uuid and replace the existing one - new_output_uuid: => - line = @end_line() - if not line? - return - output_uuid = misc.uuid() - @cm.replaceRange(output_uuid, {line:line, ch:1}, {line:line, ch:37}) - @_output_uuid = output_uuid - return output_uuid - - # return current content of the input of this cell, including uuid marker line - raw_input: (offset=0) => - start = @start_line() - if not start? - return - end = @end_line() - if not end? - return - return @cm.getRange({line:start+offset,ch:0}, {line:end, ch:0}) - - input: => - return @raw_input(1) - - is_auto: => - input = @input() - if input? - for line in input.split('\n') - if line.length > 0 and line[0] != '#' - return line.slice(0,5) == '%auto' - return false - - # return current content of the output line of this cell as a string (or undefined) - raw_output: => - x = @_get_output() - if not x? - return - return @cm.getLine(x.loc.from.line) - - output: => - v = [] - raw = @raw_output() - if not raw? # might return undefined, see above - return v - for x in raw.slice(38).split(MARKERS.output) - if x?.length > 0 # empty strings cause json deserialization problems (i.e. that warning below) - try - v.push(misc.from_json(x)) - catch - console.warn("unable to read json message in worksheet: #{x}") - return v - - _get_output: () => - n = @end_line() - if not n? - console.warn("_get_output: unable to append output message since cell no longer exists") - return - loc = {from:{line:n,ch:0}, to:{line:n,ch:@cm.getLine(n).length}} - s = @cm.getLine(n) - return {loc: loc, s: s, n: n} - - output_element: () => - end = @end_line() - if not end? - return - return @cm.findMarksAt({line:end, ch:0})?[0]?.element - - get_output_height: () => - return @output_element()?.height() - - set_output_min_height: (min_height='') => - @output_element()?.css('min-height', min_height) - - mesg_to_json: (mesg) => - return stringify(misc.copy_without(mesg, ['id', 'event'])) - - # append an output message to this cell - append_output_message: (mesg) => - x = @_get_output() - if not x? - return - s = @mesg_to_json(mesg) - if x.s[x.s.length-1] != MARKERS.output - s = MARKERS.output + s - @cm.replaceRange(s, x.loc.to, x.loc.to) - - # Delete the last num output messages in this cell - delete_last_output: (num) => - x = @_get_output() - if not x? - return - {loc, s, n} = x - for _ in misc.range(num) - i = s.lastIndexOf(MARKERS.output) - if i == -1 - @set_output() # delete it all - return - s = s.slice(0,i) - s = s.slice(37) - @cm.replaceRange(s, {line:loc.from.line, ch:37}, loc.to) - - # For a given list output of messages, set the output of that cell to them. - set_output: (output=[]) => - line = @end_line() - if not line? - console.warn("set_output: unable to append output message since cell no longer exists") - return - ch = @cm.getLine(line).length - if output.length == 0 and loc?.to.ch == 38 - # nothing to do -- already empty - return - s = MARKERS.output + (@mesg_to_json(mesg) for mesg in output).join(MARKERS.output) - @cm.replaceRange(s, {line:line, ch:37}, {line:line, ch:ch}) - - remove_cell_flag: (flag) => - n = @start_line() - if n? - s = @doc.get_input_line_flagstring(n) - if s? and flag in s - s = s.replace(new RegExp(flag, "g"), "") - @doc.set_input_line_flagstring(n, s) - - set_cell_flag: (flag) => - n = @start_line() - if n? - s = @doc.get_input_line_flagstring(n) - if flag not in s - @doc.set_input_line_flagstring(n, s + flag) - - # returns a string with the flags in it - get_cell_flags: => - return @doc.get_input_line_flagstring(@start_line()) - - action: (opts={}) => - opts = defaults opts, - execute : false # if false, do whatever else we would do, but don't actually execute code; if true, execute - toggle_input : false # if true; toggle whether input is displayed; ranges all toggle same as first - toggle_output : false # if true; toggle whether output is displayed; ranges all toggle same as first - delete_output : false # if true; delete all the the output in the range - cm : undefined - if opts.toggle_input - n = @start_line() - if not n? - return - if FLAGS.hide_input in @get_cell_flags() - # input is currently hidden - @remove_cell_flag(FLAGS.hide_input) - else - # input is currently visible - @set_cell_flag(FLAGS.hide_input) - if opts.toggle_output - flags = @get_cell_flags() - n = @start_line() - if not n? - return - if FLAGS.hide_output in @get_cell_flags() - # output is currently hidden - @remove_cell_flag(FLAGS.hide_output) - else - # output is currently visible - @set_cell_flag(FLAGS.hide_output) - if opts.delete_output - if FLAGS.hide_input in @get_cell_flags() - # input is currently hidden -- so we do NOT delete output (this confuses people too much) - return - @set_output([]) - # also show it if hidden (since nothing there) - if FLAGS.hide_output in @get_cell_flags() - # output is currently hidden - @remove_cell_flag(FLAGS.hide_output) - if opts.execute - flags = @get_cell_flags() - if not flags? - # broken/gone - return - if FLAGS.hide_output in flags - # output is currently hidden - @remove_cell_flag(FLAGS.hide_output) - if FLAGS.execute in flags or FLAGS.running in flags - # already running or queued up for execution. - return - x = @client_side() - if x - x.cell = @ - @doc.execute_cell_client_side(x) - else - @doc.execute_cell_server_side(cell : @) - - # Determine if this cell can be evaluated client side, and if so return - # {mode:?, hide:?, once:?, code:?}, where code is everything after the mode line. - # Otherwise, returns undefined. - client_side: => - s = @input() - if not s? - return # no longer defined - s = s.trim() - i = s.indexOf('\n') - if i != -1 - line0 = s.slice(0,i) - rest = s.slice(i+1) - s = line0.replace(/\s/g,'').toLowerCase() # remove whitespace - x = CLIENT_SIDE_MODE_LINES[s] - if x? - x.code = rest - return x - return false - - -exports.SynchronizedWorksheet = SynchronizedWorksheet diff --git a/src/packages/frontend/sagews/worksheet.tsx b/src/packages/frontend/sagews/worksheet.tsx deleted file mode 100644 index 7e20ffb621..0000000000 --- a/src/packages/frontend/sagews/worksheet.tsx +++ /dev/null @@ -1,43 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -React component to render a Sage worksheet statically. This is -mainly for use by server-side share server, so needs to run fine -under node.js and in the frotend. -*/ - -import { field_cmp } from "@cocalc/util/misc"; -import Cell from "./cell"; -import type { Cell as CellType } from "./parse-sagews"; - -interface Props { - sagews: CellType[]; - style?: React.CSSProperties; -} - -export default function Worksheet({ sagews, style }: Props) { - const cells: CellType[] = []; - for (const cell of sagews) { - if (cell.type === "cell") { - cells.push(cell); - } - } - cells.sort(field_cmp("pos")); - const v: React.JSX.Element[] = []; - for (const cell of cells) { - const { id, input, output, flags } = cell; - v.push( - - ); - } - - return
{v}
; -} diff --git a/src/packages/frontend/syncdoc.coffee b/src/packages/frontend/syncdoc.coffee deleted file mode 100644 index 535a38c5c6..0000000000 --- a/src/packages/frontend/syncdoc.coffee +++ /dev/null @@ -1,577 +0,0 @@ -######################################################################### -# This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. -# License: MS-RSL – see LICENSE.md for details -######################################################################### - -{SAVE_DEBOUNCE_MS} = require("@cocalc/frontend/frame-editors/code-editor/const") - -$ = window.$ -misc = require('@cocalc/util/misc') -{defaults, required} = misc - -message = require('@cocalc/util/message') -markdown = require('./markdown') - -{webapp_client} = require('./webapp-client') -{alert_message} = require('./alerts') - -async = require('async') - -templates = $("#webapp-editor-templates") - -account = require('./account') - -{redux} = require('./app-framework') - -{EventEmitter} = require('events') - -{IS_MOBILE} = require('./feature') - - -class AbstractSynchronizedDoc extends EventEmitter - file_path: () => - if not @_file_path? - @_file_path = misc.path_split(@filename).head - return @_file_path - -synchronized_string = (opts) -> - new SynchronizedString(opts) - -exports.synchronized_string = synchronized_string - -class SynchronizedDocument extends AbstractSynchronizedDoc - codemirrors: () => - if @_closed - return [] - v = [@codemirror] - if @editor._layout > 0 - v.push(@codemirror1) - return v - - focused_codemirror: () => - @editor.focused_codemirror() - -underscore = require('underscore') - -class SynchronizedString extends AbstractSynchronizedDoc - constructor: (opts) -> - super() - @opts = defaults opts, - project_id : required - filename : required - sync_interval : 1000 # TODO: ignored right now -- no matter what, we won't send sync messages back to the server more frequently than this (in ms) - cursors : false - cb : required # cb(err) once doc has connected to hub first time and got session info; will in fact keep trying - # window.w = @ - @project_id = @opts.project_id - @filename = @opts.filename - @connect = @_connect - @_syncstring = webapp_client.conat_client.conat().sync.string - project_id : @project_id - path : @filename - cursors : opts.cursors - - @_syncstring.once 'ready', => - @emit('connect') # successful connection - # first time open a file, have to look on disk to - # load it -- this ensures that is done - try - await @_syncstring.wait_until_read_only_known() - catch err - opts.cb(err) - return - @_fully_loaded = true - opts.cb(undefined, @) - - @_syncstring.on 'change', => # only when change is external - @emit('sync') - - @_syncstring.on 'before-change', => - @emit('before-change') - - @_syncstring.on 'deleted', => - redux.getProjectActions(@project_id).close_tab(@filename) - - live: (s) => - cur = @_syncstring.to_str() - if s? and s != cur - @_syncstring.exit_undo_mode() - @_syncstring.from_str(s) - @emit('sync') - else - return cur - - sync: (cb) => - @_syncstring.commit() - await @_syncstring.save() - cb?() - - _connect: (cb) => - # no op - cb?() - - _save: (cb) => - if not @_fully_loaded or not @_syncstring? - cb?() - return - try - @_syncstring.commit() - await @_syncstring.save() - await @_syncstring.save_to_disk() - cb?() - catch err - cb?(err) - - save: (cb) => - misc.retry_until_success - f : @_save - start_delay : 3000 - max_tries : 4 - max_delay : 10000 - cb : cb - - #TODO: replace disconnect_from_session by close in our API - disconnect_from_session: => - @close() - - close: => - if @_closed - return - @_syncstring.close() - @removeAllListeners() - @_closed = true - - has_uncommitted_changes: => - return @_syncstring.has_uncommitted_changes() - - has_unsaved_changes: => - return @_syncstring.has_unsaved_changes() - - # per-session sync-aware undo - undo: () => - @_syncstring.set_doc(@_syncstring.undo()) - @emit('sync') - - # per-session sync-aware redo - redo: () => - @_syncstring.set_doc(@_syncstring.redo()) - @emit('sync') - - in_undo_mode: () => - return @_syncstring.in_undo_mode() - - exit_undo_mode: () => - return @_syncstring.exit_undo_mode() - -class SynchronizedDocument2 extends SynchronizedDocument - constructor: (editor, opts) -> - super() - @editor = editor - @opts = defaults opts, - cursor_interval : 1000 # ignored below right now - sync_interval : 2000 # never send sync messages upstream more often than this - cm_foldOptions : undefined - static_viewer : undefined # must be considered now due to es6 classes - allow_javascript_eval : true # used only by sage worksheets, which derive from this -- but we have to put this here due to super being called. - persistent : false - - if @opts.static_viewer? - return - - @project_id = @editor.project_id - @filename = @editor.filename - @connect = @_connect - @editor.save = @save - @codemirror = @editor.codemirror - @codemirror1 = @editor.codemirror1 - @element = @editor.element - - if @opts.cm_foldOptions? - for cm in @codemirrors() - cm.setOption('foldOptions', @opts.foldOptions) - - # replace undo/redo by sync-aware versions - for cm in [@codemirror, @codemirror1] - cm.undo = @undo - cm.redo = @redo - - @_users = redux.getStore('users') # TODO -- obviously not like this... - - @_other_cursor_timeout_s = 30 # only show active other cursors for this long - - @editor.show_startup_message("Loading...", 'info') - @codemirror.setOption('readOnly', true) - @codemirror1.setOption('readOnly', true) - @codemirror.setValue('Loading...') - - if @filename[0] == '/' - # uses symlink to '/', which is created by start_smc - @filename = '.smc/root' + @filename - - id = require('@cocalc/util/schema').client_db.sha1(@project_id, @filename) - @_syncstring = webapp_client.conat_client.conat().sync.string - id : id - project_id : @project_id - path : @filename - cursors : true - persistent : @opts.persistent - - @_syncstring.on 'before-change', @_set_syncstring_to_codemirror - - @_syncstring.on 'after-change', @_set_codemirror_to_syncstring - - @_syncstring.once 'load-time-estimate', (est) -> - # TODO: do something with this. - #console.log 'load time estimate', est - - # This is important to debounce since above hash/getValue - # grows linearly in size of document; also, we debounce - # instead of throttle, since we don't want to have this - # slow down the user while they are typing. - f = () => - if @_update_unsaved_uncommitted_changes() - # Check again in 5s no matter what if there are - # uncommitted changes, since otherwise - # there could be a stuck notification saying - # there are uncommitted changes. - setTimeout(f, 5000) - update_unsaved_uncommitted_changes = underscore.debounce(f, 1500) - @editor.has_unsaved_changes(false) # start by assuming no unsaved changes... - #dbg = webapp_client.dbg("SynchronizedDocument2(path='#{@filename}')") - #dbg("waiting for first change") - - @_syncstring.once "error", (err) => - if @_closed - return - if err.code == 'EACCES' - err = "You do not have permission to read '#{@filename}'." - @editor.show_startup_message(err, 'danger') - return - - @_syncstring.once 'ready', => - if @_closed - return - # Now wait until read_only is *defined*, so backend file has been opened. - await @_syncstring.wait_until_read_only_known() - if @_closed - return - @editor.show_content() - @editor._set(@_syncstring.to_str()) - @_fully_loaded = true - @codemirror.setOption('readOnly', false) - @codemirror1.setOption('readOnly', false) - @codemirror.clearHistory() # ensure that the undo history doesn't start with "empty document" - @codemirror1.clearHistory() - - update_unsaved_uncommitted_changes() - @_update_read_only() - - @_init_cursor_activity() - - redux.getProjectActions(@project_id)?.log_opened_time(@filename) - - @_syncstring.on 'change', => - if @_closed - return - #dbg("got upstream syncstring change: '#{misc.trunc_middle(@_syncstring.to_str(),400)}'") - #@_set_codemirror_to_syncstring() - @emit('sync') - - @_syncstring.on 'metadata-change', => - if @_closed - return - update_unsaved_uncommitted_changes() - @_update_read_only() - - @_syncstring.on 'deleted', => - if @_closed - return - redux.getProjectActions(@editor.project_id).close_tab(@filename) - - @save_state_debounce = underscore.debounce(@sync, SAVE_DEBOUNCE_MS) - - @codemirror.on 'change', (instance, changeObj) => - if @_closed - return - if not @_setting_from_syncstring - # console.log 'user_action = true' - @_user_action = true - # console.log("change event - origin=", changeObj.origin) - if changeObj.origin? - if changeObj.origin == 'undo' - @on_undo?(instance, changeObj) - if changeObj.origin == 'redo' - @on_redo?(instance, changeObj) - if changeObj.origin != 'setValue' - @_last_change_time = new Date() - @save_state_debounce?() - update_unsaved_uncommitted_changes() - - @emit('connect') # successful connection - @_init_cb?() # done initializing document (this is used, e.g., in the SynchronizedWorksheet derived class). - - _debug_sync_state: (info) => - console.log "--- #{info}" - console.log "codemirror='#{@codemirror?.getValue()}'" - console.log "syncstring='#{@_syncstring?.to_str()}'" - if info == 'after' and @codemirror?.getValue() != @_syncstring?.to_str() - console.warn("BUG -- values are different!") - - # Set value of the syncstring to equal current value of the codemirror editor - _set_syncstring_to_codemirror: => - if not @codemirror? - return - #console.log '_set_syncstring_to_codemirror' - #@_debug_sync_state('before') - if not @_user_action - # console.log "not setting due to no user action" - # user has not explicitly done anything, so there should be no changes. - return - #console.log 'user action so setting' - @_user_action = false - @_last_val = val = @codemirror.getValue() - if val != @_syncstring.to_str() - @_syncstring.exit_undo_mode() - @_syncstring.from_str(val) - #@_debug_sync_state('after') - - # Set value of the codemirror editor to equal current value of the syncstring - _set_codemirror_to_syncstring: => - if not @codemirror? - return - #console.log '_set_codemirror_to_syncstring' - #@_debug_sync_state('before') - @_setting_from_syncstring = true - @_last_set = val = @_syncstring.to_str() - @codemirror.setValueNoJump(val) - @_setting_from_syncstring = false - #@_debug_sync_state('after') - - has_unsaved_changes: => - if not @codemirror? - return false - # This is potentially VERY expensive!!! - return @_syncstring.hash_of_saved_version() != misc.hash_string(@codemirror.getValue()) - - has_uncommitted_changes: => - # WARNING: potentially expensive to do @codemirror.getValue(). - return @_syncstring.has_uncommitted_changes() or @codemirror.getValue() != @_syncstring.to_str() - - _update_unsaved_uncommitted_changes: => - if not @_fully_loaded or not @codemirror? or @_closed - return - if new Date() - (@_last_change_time ? 0) <= 1000 - # wait at least a second from when the user last changed the document, in case it's just a burst of typing. - return - x = @codemirror.getValue() - @editor.has_unsaved_changes(@_syncstring.hash_of_saved_version() != misc.hash_string(x)) - uncommitted_changes = @_syncstring.has_uncommitted_changes() or x != @_syncstring.to_str() - @editor.has_uncommitted_changes(uncommitted_changes) - return uncommitted_changes - - _update_read_only: => - @editor.set_readonly_ui(@_syncstring.is_read_only()) - - sync: (cb) => - if not @codemirror? or @_syncstring?.get_state() != 'ready' - # codemirror need not be defined or @_syncstring might be not ready to use, e.g., - # right when user closes the editor instance - cb?() - return - @_set_syncstring_to_codemirror() - @_syncstring.commit() - try - await @_syncstring.save() - cb?() - catch err - cb?(err) - - - # per-session sync-aware undo - undo: () => - if not @codemirror? - return - cm = @focused_codemirror() # see https://github.com/sagemathinc/cocalc/issues/1161 - if not @_syncstring.in_undo_mode() - @_set_syncstring_to_codemirror() - value = @_syncstring.undo().to_str() - cm.setValueNoJump(value, true) - @save_state_debounce?() - @_last_change_time = new Date() - - # per-session sync-aware redo - redo: () => - if not @codemirror? - return - if not @_syncstring.in_undo_mode() - return - doc = @_syncstring.redo() - if not doc? - # can't redo if version not defined/not available. - return - if not doc.to_str? - # BUG -- see https://github.com/sagemathinc/cocalc/issues/1831 - throw Error("doc must have a to_str method, but is doc='#{doc}', typeof(doc)='#{typeof(doc)}'") - value = doc.to_str() - @focused_codemirror().setValueNoJump(value, true) - @save_state_debounce?() - @_last_change_time = new Date() - - _connect: (cb) => - # no op - cb?() - - _save: (cb) => - if not @codemirror? or not @_fully_loaded - cb() # nothing to do -- not initialized/loaded yet... - return - @_set_syncstring_to_codemirror() - # Do save_to_disk immediately, then -- if any unsaved - # to backend changes, save those. Finally, save to disk again. - # We do this so we succeed at saving to disk, in case - # file is being **immediately** closed right when saving to disk, - # which happens on tab close. - try - await @_syncstring.save_to_disk() - @_syncstring.commit() - await @_syncstring.save() - await @_syncstring.save_to_disk() - catch err - cb(err) - return - @_update_unsaved_uncommitted_changes() - @_post_save_success?() - # hook so that derived classes can do things, e.g., make blobs permanent - cb() - - delete_trailing_whitespace: => - cm = @focused_codemirror() - omit_lines = {} - @_syncstring.get_cursors(excludeSelf:'never')?.map (x, _) => - x.get('locs')?.map (loc) => - y = loc.get('y') - if y? - omit_lines[y] = true - cm.delete_trailing_whitespace(omit_lines:omit_lines) - - save: (cb) => - if @_closed - cb?() - return - # This first call immediately sets saved button to disabled to make it feel like instant save. - @editor.has_unsaved_changes(false) - # We then simply ensure the save state is valid 5s later (in case save fails, say). - setTimeout(@_update_unsaved_uncommitted_changes, 5000) - - misc.retry_until_success - f : @_save - start_delay : 3000 - max_tries : 4 - max_delay : 10000 - cb : cb - - _init_cursor_activity: () => - for i, cm of [@codemirror, @codemirror1] - cm.on 'cursorActivity', (cm) => - if cm.name != @focused_codemirror().name - # ignore non-focused editor - return - if cm._setValueNoJump # if true, this is being caused by external setValueNoJump - return - # broadcast cursor positions - locs = ({x:c.anchor.ch, y:c.anchor.line} for c in cm.listSelections()) - @_syncstring.set_cursor_locs(locs) - # save primary cursor position to local storage for next time - #console.log("setting cursor#{cm.name} to #{misc.to_json(cm.getCursor())}") - @editor.local_storage("cursor#{cm.name}", cm.getCursor()) - - @_syncstring.on 'cursor_activity', (account_id) => - @_render_other_cursor(account_id) - - get_users_cursors: (account_id) => - if not @_syncstring? - return - x = @_syncstring.get_cursors()?.get(account_id) - #console.log("_render_other_cursor", x?.get('time'), misc.seconds_ago(@_other_cursor_timeout_s)) - # important: must use server time to compare, not local time. - if webapp_client.server_time() - x?.get('time') <= @_other_cursor_timeout_s*1000 - locs = x.get('locs')?.toJS() - return locs - - _render_other_cursor: (account_id) => - #if account_id == webapp_client.account_id - # nothing to do -- we don't draw our own cursor via this - # return - x = @_syncstring.get_cursors()?.get(account_id) - #console.log("_render_other_cursor", x?.get('time'), misc.seconds_ago(@_other_cursor_timeout_s)) - # important: must use server time to compare, not local time. - if webapp_client.server_time() - x?.get('time') <= @_other_cursor_timeout_s*1000 - locs = x.get('locs')?.toJS() - if locs? - #console.log("draw cursors for #{account_id} at #{misc.to_json(locs)} expiring after #{@_other_cursor_timeout_s}s") - @draw_other_cursors(account_id, locs) - - # Move the cursor with given color to the given pos. - draw_other_cursors: (account_id, locs) => - if not @codemirror? # can happen right when closing. - return - # ensure @_cursors is defined; this is map from key to ...? - #console.log("draw_other_cursors(#{account_id}, #{misc.to_json(locs)})") - @_cursors ?= {} - x = @_cursors[account_id] - if not x? - x = @_cursors[account_id] = [] - # First draw/update all current cursors - for [i, loc] in misc.enumerate(locs) - pos = {line:loc.y, ch:loc.x} - data = x[i] - name = misc.trunc(@_users.get_first_name(account_id), 10) - color = @_users.get_color_sync(account_id) - if not data? - data = x[i] = {cursor: templates.find(".smc-editor-codemirror-cursor").clone().show()} - if name != data.name - data.cursor.find(".smc-editor-codemirror-cursor-label").text(name) - data.name = name - if color != data.color - data.cursor.find(".smc-editor-codemirror-cursor-inside").css('border-left': "1px solid #{color}") - data.cursor.find(".smc-editor-codemirror-cursor-label" ).css(background: color) - data.color = color - - # Place cursor in the editor in the right spot - @codemirror.addWidget(pos, data.cursor[0], false) - - # Update cursor fade-out - # LABEL: first fade the label out over 15s - data.cursor.find(".smc-editor-codemirror-cursor-label").stop().animate(opacity:1).show().fadeOut(duration:15000) - # CURSOR: then fade the cursor out (a non-active cursor is a waste of space) over 25s. - data.cursor.find(".smc-editor-codemirror-cursor-inside").stop().animate(opacity:1).show().fadeOut(duration:25000) - - if x.length > locs.length - # Next remove any cursors that are no longer there (e.g., user went from 5 cursors to 1) - for i in [locs.length...x.length] - #console.log('removing cursor ', i) - x[i].cursor.remove() - @_cursors[account_id] = x.slice(0, locs.length) - - #TODO: replace disconnect_from_session by close in our API - disconnect_from_session: => - @close() - - close: => - if @_closed - return - @_syncstring?.close() - # TODO -- this doesn't work... - for cm in [@codemirror, @codemirror1] - continue if not cm? - cm.setOption("mode", "text/x-csrc") - cmElem = cm.getWrapperElement() - cmElem.parentNode.removeChild(cmElem) - delete @codemirror - delete @codemirror1 - delete @editor.codemirror - delete @editor.codemirror1 - @_closed = true - - -exports.SynchronizedDocument2 = SynchronizedDocument2 diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 1fe9cfbb56..e2f5603bbf 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -516,9 +516,6 @@ importers: codemirror: specifier: ^5.65.18 version: 5.65.20 - coffeescript: - specifier: ^2.5.1 - version: 2.7.0 color-map: specifier: ^2.0.6 version: 2.0.6 @@ -1624,21 +1621,9 @@ importers: buffer: specifier: ^6.0.3 version: 6.0.3 - cjsx-loader: - specifier: ^3.0.0 - version: 3.0.0 clean-webpack-plugin: specifier: ^4.0.0 version: 4.0.0(webpack@5.101.3) - coffee-cache: - specifier: ^1.0.2 - version: 1.0.2 - coffee-loader: - specifier: ^3.0.0 - version: 3.0.0(coffeescript@2.7.0)(webpack@5.101.3) - coffeescript: - specifier: ^2.5.1 - version: 2.7.0 css-loader: specifier: ^7.1.2 version: 7.1.2(@rspack/core@1.5.8(@swc/helpers@0.5.15))(webpack@5.101.3) @@ -5894,9 +5879,6 @@ packages: cjs-module-lexer@2.1.0: resolution: {integrity: sha512-UX0OwmYRYQQetfrLEZeewIFFI+wSTofC+pMBLNuH3RUuu/xzG1oz84UCEDOSoQlN3fZ4+AzmV50ZYvGqkMh9yA==} - cjsx-loader@3.0.0: - resolution: {integrity: sha512-bek+Ojam3A0QmqrT8GXvRUdo3ynHyPAjBHNZz2A2lSDccMU9GGZ1eNsTc7+cQ/n2H+UhDL8XW/H1wvG8cNLdZA==} - clamp@1.0.1: resolution: {integrity: sha512-kgMuFyE78OC6Dyu3Dy7vcx4uy97EIbVxJB/B0eJ3bUNAkwdNcxYzgKltnyADiYwsR7SEqkkUPsEUT//OVS6XMA==} @@ -5965,20 +5947,6 @@ packages: codemirror@5.65.20: resolution: {integrity: sha512-i5dLDDxwkFCbhjvL2pNjShsojoL3XHyDwsGv1jqETUoW+lzpBKKqNTUWgQwVAOa0tUm4BwekT455ujafi8payA==} - coffee-cache@1.0.2: - resolution: {integrity: sha512-sIqhtqg5AOfgVWH8uQ0b0i0rkawDD1icZNOgyYNJOlJBm8RrEzstLeOwgjZPcyVeMq0jNryx2TDL5xcPUo/27A==} - - coffee-loader@3.0.0: - resolution: {integrity: sha512-2UPQNXfMAt4RmI/K9VxnLyrXdYdHPHQuEFiGcb70pTsVPmrV9M6Xg3p9ub7t1ettZZqvXUujjHozp22uTfLkzg==} - engines: {node: '>= 12.13.0'} - peerDependencies: - coffeescript: '>= 2.0.0' - webpack: ^5.0.0 - - coffee-react-transform@4.0.0: - resolution: {integrity: sha512-TTedgCCnIR978F1hwiv/aufBMz0e2pMrb8wEwf20X9VDqDvCWokTQx0ioMY/9c9eq7DnC5nb9wKGspF2IzcoFQ==} - hasBin: true - coffeescript@2.7.0: resolution: {integrity: sha512-hzWp6TUE2d/jCcN67LrW1eh5b/rSDKQK6oD6VMLlggYVUUFexgTH9z3dNYihzX4RMhze5FTUsUmOXViJKFQR/A==} engines: {node: '>=6'} @@ -9327,9 +9295,6 @@ packages: engines: {node: '>=10'} hasBin: true - mkpath@0.1.0: - resolution: {integrity: sha512-bauHShmaxVQiEvlrAPWxSPn8spSL8gDVRl11r8vLT4r/KdnknLqtqwQbToZ2Oa8sJkExYY1z6/d+X7pNiqo4yg==} - mlly@1.7.4: resolution: {integrity: sha512-qmdSIPC4bDJXgZTCR7XosJiNKySV7O215tsPtDN9iEO/7q/76b/ijtgRu/+epFXSJhijtTCCGp3DWS549P3xKw==} @@ -17690,11 +17655,6 @@ snapshots: cjs-module-lexer@2.1.0: {} - cjsx-loader@3.0.0: - dependencies: - coffee-react-transform: 4.0.0 - loader-utils: 0.2.17 - clamp@1.0.1: {} classnames@2.5.1: {} @@ -17759,17 +17719,6 @@ snapshots: codemirror@5.65.20: {} - coffee-cache@1.0.2: - dependencies: - mkpath: 0.1.0 - - coffee-loader@3.0.0(coffeescript@2.7.0)(webpack@5.101.3): - dependencies: - coffeescript: 2.7.0 - webpack: 5.101.3 - - coffee-react-transform@4.0.0: {} - coffeescript@2.7.0: {} collect-v8-coverage@1.0.2: {} @@ -21878,8 +21827,6 @@ snapshots: mkdirp@3.0.1: {} - mkpath@0.1.0: {} - mlly@1.7.4: dependencies: acorn: 8.15.0 diff --git a/src/packages/static/package.json b/src/packages/static/package.json index d3ffce8c4b..6d9943d64d 100644 --- a/src/packages/static/package.json +++ b/src/packages/static/package.json @@ -66,11 +66,7 @@ "bootstrap": "=3.4.1", "bootstrap-colorpicker": "^2.5.3", "buffer": "^6.0.3", - "cjsx-loader": "^3.0.0", "clean-webpack-plugin": "^4.0.0", - "coffee-cache": "^1.0.2", - "coffee-loader": "^3.0.0", - "coffeescript": "^2.5.1", "css-loader": "^7.1.2", "entities": "^2.2.0", "eslint": "^8.56.0", diff --git a/src/packages/static/src/module-rules.ts b/src/packages/static/src/module-rules.ts index 2231b3c7c7..f235b6c470 100644 --- a/src/packages/static/src/module-rules.ts +++ b/src/packages/static/src/module-rules.ts @@ -45,11 +45,6 @@ export default function moduleRules( }, ], }, - { test: /\.coffee$/, loader: "coffee-loader" }, - { - test: /\.cjsx$/, - use: [{ loader: "coffee-loader" }, { loader: "cjsx-loader" }], - }, { test: /\.less$/, use: [ diff --git a/src/packages/static/src/rspack.config.ts b/src/packages/static/src/rspack.config.ts index d18d44b964..70c8993f32 100644 --- a/src/packages/static/src/rspack.config.ts +++ b/src/packages/static/src/rspack.config.ts @@ -224,8 +224,6 @@ export default function getConfig({ middleware }: Options = {}): Configuration { ".ts", ".tsx", ".json", - ".coffee", - ".cjsx", ".scss", ".sass", ], From b8931b0e24f9c5063bef728e604fcf711ea95195 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 8 Oct 2025 11:00:52 -0700 Subject: [PATCH 783/798] remove the backend sagews support completely --- src/cocalc.code-workspace | 1 - src/dev/laptop/README.md | 1 - src/dev/minimal/README.md | 1 - src/dev/project/smc-project-sagews-dev.sh | 73 - src/packages/conat/project/api/editor.ts | 9 - src/packages/frontend/components/html.tsx | 2 +- src/packages/project/client.ts | 11 - src/packages/project/conat/api/editor.ts | 28 - src/packages/project/configuration.ts | 2 - src/packages/project/data.ts | 5 - src/packages/project/port_manager.ts | 38 - src/packages/project/sage_restart.ts | 59 - src/packages/project/sage_session.ts | 298 -- src/packages/project/sage_socket.ts | 99 - src/packages/project/sagews/README.md | 12 - src/packages/project/sagews/control.ts | 10 - src/requirements.txt | 1 - src/smc_pyutil/smc_pyutil/bin/smc-sage-server | 5 - src/smc_sagews/README.md | 29 - src/smc_sagews/setup.py | 36 - src/smc_sagews/smc_sagews/__init__.py | 11 - src/smc_sagews/smc_sagews/daemon.py | 122 - src/smc_sagews/smc_sagews/graphics.py | 645 --- src/smc_sagews/smc_sagews/julia.py | 490 -- src/smc_sagews/smc_sagews/markdown2Mathjax.py | 221 - src/smc_sagews/smc_sagews/sage_jupyter.py | 435 -- src/smc_sagews/smc_sagews/sage_parsing.py | 659 --- src/smc_sagews/smc_sagews/sage_salvus.py | 4503 ----------------- src/smc_sagews/smc_sagews/sage_server.py | 2414 --------- .../smc_sagews/sage_server_command_line.py | 83 - .../smc_sagews/test_direct/README.md | 51 - .../smc_sagews/test_direct/conftest.py | 100 - .../smc_sagews/test_direct/test_jupyter.py | 15 - src/smc_sagews/smc_sagews/tests/README.md | 85 - src/smc_sagews/smc_sagews/tests/a.html | 8 - src/smc_sagews/smc_sagews/tests/a.py | 2 - src/smc_sagews/smc_sagews/tests/a.sage | 7 - src/smc_sagews/smc_sagews/tests/conftest.py | 882 ---- src/smc_sagews/smc_sagews/tests/runtests.sh | 5 - .../tests/sage_init_files/define_var.sage | 2 - .../tests/sage_init_files/runtime_err.sage | 1 - .../tests/sage_init_files/syntax_err.sage | 1 - .../smc_sagews/tests/test_00_timing.py | 151 - .../smc_sagews/tests/test_doctests.py | 89 - src/smc_sagews/smc_sagews/tests/test_env.py | 78 - .../smc_sagews/tests/test_graphics.py | 136 - .../smc_sagews/tests/test_sagews.py | 479 -- .../smc_sagews/tests/test_sagews_modes.py | 344 -- .../smc_sagews/tests/test_unicode.py | 131 - 49 files changed, 1 insertion(+), 12869 deletions(-) delete mode 100755 src/dev/project/smc-project-sagews-dev.sh delete mode 100644 src/packages/project/port_manager.ts delete mode 100644 src/packages/project/sage_restart.ts delete mode 100644 src/packages/project/sage_session.ts delete mode 100644 src/packages/project/sage_socket.ts delete mode 100644 src/packages/project/sagews/README.md delete mode 100644 src/packages/project/sagews/control.ts delete mode 100755 src/smc_pyutil/smc_pyutil/bin/smc-sage-server delete mode 100644 src/smc_sagews/README.md delete mode 100644 src/smc_sagews/setup.py delete mode 100644 src/smc_sagews/smc_sagews/__init__.py delete mode 100755 src/smc_sagews/smc_sagews/daemon.py delete mode 100644 src/smc_sagews/smc_sagews/graphics.py delete mode 100644 src/smc_sagews/smc_sagews/julia.py delete mode 100644 src/smc_sagews/smc_sagews/markdown2Mathjax.py delete mode 100644 src/smc_sagews/smc_sagews/sage_jupyter.py delete mode 100644 src/smc_sagews/smc_sagews/sage_parsing.py delete mode 100644 src/smc_sagews/smc_sagews/sage_salvus.py delete mode 100755 src/smc_sagews/smc_sagews/sage_server.py delete mode 100644 src/smc_sagews/smc_sagews/sage_server_command_line.py delete mode 100644 src/smc_sagews/smc_sagews/test_direct/README.md delete mode 100644 src/smc_sagews/smc_sagews/test_direct/conftest.py delete mode 100644 src/smc_sagews/smc_sagews/test_direct/test_jupyter.py delete mode 100644 src/smc_sagews/smc_sagews/tests/README.md delete mode 100644 src/smc_sagews/smc_sagews/tests/a.html delete mode 100644 src/smc_sagews/smc_sagews/tests/a.py delete mode 100644 src/smc_sagews/smc_sagews/tests/a.sage delete mode 100644 src/smc_sagews/smc_sagews/tests/conftest.py delete mode 100755 src/smc_sagews/smc_sagews/tests/runtests.sh delete mode 100755 src/smc_sagews/smc_sagews/tests/sage_init_files/define_var.sage delete mode 100644 src/smc_sagews/smc_sagews/tests/sage_init_files/runtime_err.sage delete mode 100644 src/smc_sagews/smc_sagews/tests/sage_init_files/syntax_err.sage delete mode 100644 src/smc_sagews/smc_sagews/tests/test_00_timing.py delete mode 100644 src/smc_sagews/smc_sagews/tests/test_doctests.py delete mode 100644 src/smc_sagews/smc_sagews/tests/test_env.py delete mode 100644 src/smc_sagews/smc_sagews/tests/test_graphics.py delete mode 100644 src/smc_sagews/smc_sagews/tests/test_sagews.py delete mode 100644 src/smc_sagews/smc_sagews/tests/test_sagews_modes.py delete mode 100644 src/smc_sagews/smc_sagews/tests/test_unicode.py diff --git a/src/cocalc.code-workspace b/src/cocalc.code-workspace index 5b0008c220..0de8979d9d 100644 --- a/src/cocalc.code-workspace +++ b/src/cocalc.code-workspace @@ -26,7 +26,6 @@ "requirements.txt": true, "scripts": true, "smc_pyutil": true, - "smc_sagews": true, ".mypy_cache": true, "pnpm-lock.yaml": true }, diff --git a/src/dev/laptop/README.md b/src/dev/laptop/README.md index f8517f7980..0db985bbd9 100644 --- a/src/dev/laptop/README.md +++ b/src/dev/laptop/README.md @@ -36,7 +36,6 @@ From the `src/` directory: From the `src/` directory, run - pip install --user --upgrade smc_sagews/ pip install --user --upgrade smc_pyutil/ ## The servers diff --git a/src/dev/minimal/README.md b/src/dev/minimal/README.md index be91cc5cbb..c4ee94ad27 100644 --- a/src/dev/minimal/README.md +++ b/src/dev/minimal/README.md @@ -14,7 +14,6 @@ Install all the software preqreqs (postgresql, node, etc.), then cd ~/cocalc/src . smc-env npm run install -pip install --user smc_sagews pip install --user smc_pyutil npm install forever diff --git a/src/dev/project/smc-project-sagews-dev.sh b/src/dev/project/smc-project-sagews-dev.sh deleted file mode 100755 index d4f97b2651..0000000000 --- a/src/dev/project/smc-project-sagews-dev.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash - -# script "smc-project-sagews-dev" -# use when ready to start new branch of development for SMC sagews -# requirements: -# git upstream remote repo is sagemathinc.smc, used read-only -# git origin remote repo is developer's fork of smc in developer's github -# local smc git repo at ~/smc -# be ready with new branch name, for example "isssue456utf" -# -# usage: -# new-issue [new-branch-name] -# script will ask for new-branch-name if not provided on command line - -trap "echo '**script error exit**';exit" ERR - -echo "NOTE: restart your SMC dev project before running this script" - -if [[ $# -ge 1 ]];then - BNAME=$1 -else - echo -n "name of new SMC branch (issueXXXdescword): " - read BNAME -fi -echo -n "ok to start smc dev on new branch $BNAME? (y|n) " -read ans -case $ans in -y|Y) - ;; -*) - echo canceled - ;; -esac - -cd ~/smc -# echo "checking that 'upstream' remote is main SMC github repo" -git config remote.upstream.url | grep -q sagemathinc/smc.git || { - echo "git remote upstream must point to sagemathinc/smc.git" - exit 1 -} - -# echo "checking that 'origin' remote is non-upstream SMC github repo" -git remote show origin|grep "Push.*smc.git"|grep -vq sagemathinc || { - echo "git remote origin must point to non-sagemathinc smc github repo" - exit 1 -} - -if [[ $EUID -eq 0 ]]; then - echo "This script must NOT be run as root" - exit 1 -fi - - -echo "killing old sage worksheet processes" -pkill -f sage_server_command_line || test $? -eq 1 -rm -f ~/.smc/sage_server/sage_server.pid - -echo "updating git" -git checkout master -git pull upstream master -git checkout -b $BNAME - -echo "updating smc_sagews and tests" -cd ~/smc/src/smc_sagews -pip install --user --upgrade ./ - -echo "running tests" -cd ~/smc/src/smc_sagews/smc_sagews/tests - -# baseline run of all tests before we start making changes -python -m pytest - -echo "setup for branch $BNAME done" diff --git a/src/packages/conat/project/api/editor.ts b/src/packages/conat/project/api/editor.ts index cf9984b9da..cb970b56ed 100644 --- a/src/packages/conat/project/api/editor.ts +++ b/src/packages/conat/project/api/editor.ts @@ -2,11 +2,6 @@ import type { Options as FormatterOptions } from "@cocalc/util/code-formatter"; export const editor = { formatString: true, - - printSageWS: true, - sagewsStart: true, - sagewsStop: true, - createTerminalService: true, }; @@ -29,10 +24,6 @@ export interface Editor { path?: string; // only used for CLANG }) => Promise; - printSageWS: (opts) => Promise; - sagewsStart: (path_sagews: string) => Promise; - sagewsStop: (path_sagews: string) => Promise; - createTerminalService: ( termPath: string, opts: CreateTerminalOptions, diff --git a/src/packages/frontend/components/html.tsx b/src/packages/frontend/components/html.tsx index dba6e9f8e1..c76b351b3b 100644 --- a/src/packages/frontend/components/html.tsx +++ b/src/packages/frontend/components/html.tsx @@ -49,7 +49,7 @@ export interface Props { reload_images?: boolean; /* If true, after rendering run the smc_image_scaling pluging to handle - smc-image-scaling= attributes, which are used in smc_sagews to rescale certain + smc-image-scaling= attributes, which might be used to rescale certain png images produced by other kernels (e.g., the R kernel). See https://github.com/sagemathinc/cocalc/issues/4421. This functionality is NOT actually used at all right now, since it doesn't work on the share server diff --git a/src/packages/project/client.ts b/src/packages/project/client.ts index 9b1fa744e7..b2e4c79b4f 100644 --- a/src/packages/project/client.ts +++ b/src/packages/project/client.ts @@ -42,7 +42,6 @@ import * as data from "./data"; import initJupyter from "./jupyter/init"; import * as kucalc from "./kucalc"; import { getLogger } from "./logger"; -import * as sage_session from "./sage_session"; import synctable_conat from "@cocalc/project/conat/synctable"; import pubsub from "@cocalc/project/conat/pubsub"; import type { ConatSyncTableFunction } from "@cocalc/conat/sync/synctable"; @@ -555,16 +554,6 @@ export class Client extends EventEmitter implements ProjectClientInterface { execute_code(opts); } - // return new sage session -- the code that actually calls this is in the @cocalc/sync package - // in "packages/sync/editor/generic/evaluator.ts" - public sage_session({ - path, - }: { - path: string; // the path to the *worksheet* file - }): sage_session.SageSessionType { - return sage_session.sage_session({ path, client: this }); - } - // Save a blob to the central db blobstore. // The sha1 is optional. public save_blob({ diff --git a/src/packages/project/conat/api/editor.ts b/src/packages/project/conat/api/editor.ts index 7ddc203753..9edc482c2f 100644 --- a/src/packages/project/conat/api/editor.ts +++ b/src/packages/project/conat/api/editor.ts @@ -1,30 +1,2 @@ export { formatString } from "../../formatters"; - -import { printSageWS as printSageWS0 } from "@cocalc/project/print_to_pdf"; -export { sagewsStart, sagewsStop } from "@cocalc/project/sagews/control"; - -import { filename_extension } from "@cocalc/util/misc"; -export async function printSageWS(opts): Promise { - let pdf; - const ext = filename_extension(opts.path); - if (ext) { - pdf = `${opts.path.slice(0, opts.path.length - ext.length)}pdf`; - } else { - pdf = opts.path + ".pdf"; - } - - await printSageWS0({ - path: opts.path, - outfile: pdf, - title: opts.options?.title, - author: opts.options?.author, - date: opts.options?.date, - contents: opts.options?.contents, - subdir: opts.options?.subdir, - extra_data: opts.options?.extra_data, - timeout: opts.options?.timeout, - }); - return pdf; -} - export { createTerminalService } from "@cocalc/project/conat/terminal"; diff --git a/src/packages/project/configuration.ts b/src/packages/project/configuration.ts index 2a9f7e1b75..e2c354203b 100644 --- a/src/packages/project/configuration.ts +++ b/src/packages/project/configuration.ts @@ -85,8 +85,6 @@ async function get_sage_info(): Promise<{ exists: boolean; version: number[] | undefined; }> { - // TODO probably also check if smc_sagews is working? or the sage server? - // without sage, sagews files are disabled const exists = await have("sage"); let version: number[] | undefined = undefined; if (exists) { diff --git a/src/packages/project/data.ts b/src/packages/project/data.ts index 28b1a18eb3..0c344c881d 100644 --- a/src/packages/project/data.ts +++ b/src/packages/project/data.ts @@ -23,11 +23,6 @@ export const rootSymlink = join(data, "root"); export const SSH_LOG = join(data, "sshd.log"); export const SSH_ERR = join(data, "sshd.err"); export const compute_server_id = parseInt(process.env.COMPUTE_SERVER_ID ?? "0"); -export const sageServerPaths = { - log: join(data, "sage_server", "sage_server.log"), - pid: join(data, "sage_server", "sage_server.pid"), - port: join(data, "sage_server", "sage_server.port"), -} as const; // secret token must be after compute_server_id is set, since it uses it. export { secretToken } from "./secret-token"; diff --git a/src/packages/project/port_manager.ts b/src/packages/project/port_manager.ts deleted file mode 100644 index 256a032c1b..0000000000 --- a/src/packages/project/port_manager.ts +++ /dev/null @@ -1,38 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2023 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { readFile } from "node:fs/promises"; -import { sageServerPaths } from "@cocalc/project/data"; - -type Type = "sage"; - -/* -The port_manager manages the ports for the sage worksheet server. -*/ - -// a local cache -const ports: { [type in Type]?: number } = {}; - -export async function get_port(type: Type = "sage"): Promise { - const val = ports[type]; - if (val != null) { - return val; - } else { - const content = await readFile(sageServerPaths.port); - try { - const val = parseInt(content.toString()); - ports[type] = val; - return val; - } catch (err) { - throw new Error(`${type}_server port file corrupted -- ${err}`); - } - } -} - -export function forget_port(type: Type = "sage") { - if (ports[type] != null) { - delete ports[type]; - } -} diff --git a/src/packages/project/sage_restart.ts b/src/packages/project/sage_restart.ts deleted file mode 100644 index d7dfb40fa6..0000000000 --- a/src/packages/project/sage_restart.ts +++ /dev/null @@ -1,59 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { executeCode } from "@cocalc/backend/execute-code"; -import { getLogger } from "@cocalc/backend/logger"; -import { to_json } from "@cocalc/util/misc"; - -const winston = getLogger("sage-restart"); - -// Wait up to this long for the Sage server to start responding -// connection requests, after we restart it. It can -// take a while, since it pre-imports the sage library -// at startup, before forking. -export const SAGE_SERVER_MAX_STARTUP_TIME_S = 60; - -let restarting = false; -let restarted = 0; // time when we last restarted it - -export async function restartSageServer() { - const dbg = (m) => winston.debug(`restartSageServer: ${to_json(m)}`); - if (restarting) { - dbg("hit lock"); - throw new Error("already restarting sage server"); - } - - const t = Date.now() - restarted; - - if (t <= SAGE_SERVER_MAX_STARTUP_TIME_S * 1000) { - const err = `restarted sage server ${t}ms ago: not allowing too many restarts too quickly...`; - dbg(err); - throw new Error(err); - } - - restarting = true; - - dbg("restarting the daemon"); - - try { - const output = await executeCode({ - command: "smc-sage-server restart", - timeout: 45, - ulimit_timeout: false, // very important -- so doesn't kill after 30 seconds of cpu! - err_on_exit: true, - bash: true, - }); - dbg( - `successfully restarted sage server daemon -- '${JSON.stringify(output)}'` - ); - } catch (err) { - const msg = `failed to restart sage server daemon -- ${err}`; - dbg(msg); - throw new Error(msg); - } finally { - restarting = false; - restarted = Date.now(); - } -} diff --git a/src/packages/project/sage_session.ts b/src/packages/project/sage_session.ts deleted file mode 100644 index 8a9d0e55bc..0000000000 --- a/src/packages/project/sage_session.ts +++ /dev/null @@ -1,298 +0,0 @@ -//######################################################################## -// This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. -// License: MS-RSL – see LICENSE.md for details -//######################################################################## - -/* -Start the Sage server and also get a new socket connection to it. -*/ - -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { getLogger } from "@cocalc/backend/logger"; -import processKill from "@cocalc/backend/misc/process-kill"; -import { abspath } from "@cocalc/backend/misc_node"; -import type { - Type as TCPMesgType, - Message as TCPMessage, -} from "@cocalc/backend/tcp/enable-messaging-protocol"; -import { CoCalcSocket } from "@cocalc/backend/tcp/enable-messaging-protocol"; -import * as message from "@cocalc/util/message"; -import { - path_split, - to_json, - trunc, - trunc_middle, - uuid, -} from "@cocalc/util/misc"; -import { CB } from "@cocalc/util/types/callback"; -import { ISageSession, SageCallOpts } from "@cocalc/util/types/sage"; -import { Client } from "./client"; -import { getSageSocket } from "./sage_socket"; - -const logger = getLogger("sage-session"); - -//############################################## -// Direct Sage socket session -- used internally in local hub, e.g., to assist CodeMirror editors... -//############################################## - -// we have to make sure to only export the type to avoid error TS4094 -export type SageSessionType = InstanceType; - -interface SageSessionOpts { - client: Client; - path: string; // the path to the *worksheet* file -} - -const cache: { [path: string]: SageSessionType } = {}; - -export function sage_session(opts: Readonly): SageSessionType { - const { path } = opts; - // compute and cache if not cached; otherwise, get from cache: - return (cache[path] = cache[path] ?? new SageSession(opts)); -} - -/* -Sage Session object - -Until you actually try to call it no socket need -*/ -class SageSession implements ISageSession { - private _path: string; - private _client: Client; - private _output_cb: { - [key: string]: CB<{ done: boolean; error: string }, any>; - } = {}; - private _socket: CoCalcSocket | undefined; - - constructor(opts: Readonly) { - this.dbg = this.dbg.bind(this); - this.dbg("constructor")(); - this._path = opts.path; - this._client = opts.client; - this._output_cb = {}; - } - - private dbg = (f: string) => { - return (m?: string) => - logger.debug(`SageSession(path='${this._path}').${f}: ${m}`); - }; - - close = (): void => { - if (this._socket != null) { - const pid = this._socket.pid; - if (pid != null) { - processKill(pid, 9); - } - this._socket.end(); - delete this._socket; - } - for (let id in this._output_cb) { - const cb = this._output_cb[id]; - cb({ done: true, error: "killed" }); - } - this._output_cb = {}; - delete cache[this._path]; - }; - - // return true if there is a socket connection to a sage server process - is_running = (): boolean => { - return this._socket != null; - }; - - // NOTE: There can be many simultaneous init_socket calls at the same time, - // if e.g., the socket doesn't exist and there are a bunch of calls to @call - // at the same time. - // See https://github.com/sagemathinc/cocalc/issues/3506 - // wrapped in reuseInFlight ! - init_socket = reuseInFlight(async (): Promise => { - const dbg = this.dbg("init_socket()"); - dbg(); - try { - const socket: CoCalcSocket = await getSageSocket(); - - dbg("successfully opened a sage session"); - this._socket = socket; - - socket.on("end", () => { - delete this._socket; - return dbg("codemirror session terminated"); - }); - - // CRITICAL: we must define this handler before @_init_path below, - // or @_init_path can't possibly work... since it would wait for - // this handler to get the response message! - socket.on("mesg", (type: TCPMesgType, mesg: TCPMessage) => { - dbg(`sage session: received message ${type}`); - switch (type) { - case "json": - this._handle_mesg_json(mesg); - break; - case "blob": - this._handle_mesg_blob(mesg); - break; - } - }); - - await this._init_path(); - } catch (err) { - if (err) { - dbg(`fail -- ${err}.`); - throw err; - } - } - }); - - private _init_path = async (): Promise => { - const dbg = this.dbg("_init_path()"); - dbg(); - return new Promise((resolve, reject) => { - this.call({ - input: { - event: "execute_code", - code: "os.chdir(salvus.data['path']);__file__=salvus.data['file']", - data: { - path: abspath(path_split(this._path).head), - file: abspath(this._path), - }, - preparse: false, - }, - cb: (resp) => { - let err: string | undefined = undefined; - if (resp.stderr) { - err = resp.stderr; - dbg(`error '${err}'`); - } - if (resp.done) { - if (err) { - reject(err); - } else { - resolve(); - } - } - }, - }); - }); - }; - - public call = async ({ - input, - cb, - }: Readonly): Promise => { - const dbg = this.dbg("call"); - dbg(`input='${trunc(to_json(input), 300)}'`); - switch (input.event) { - case "ping": - cb({ pong: true }); - return; - - case "status": - cb({ running: this.is_running() }); - return; - - case "signal": - if (this._socket != null) { - dbg(`sending signal ${input.signal} to process ${this._socket.pid}`); - const pid = this._socket.pid; - if (pid != null) processKill(pid, input.signal); - } - cb({}); - return; - - case "restart": - dbg("restarting sage session"); - if (this._socket != null) { - this.close(); - } - try { - await this.init_socket(); - cb({}); - } catch (err) { - cb({ error: err }); - } - return; - - case "raw_input": - dbg("sending sage_raw_input event"); - this._socket?.write_mesg("json", { - event: "sage_raw_input", - value: input.value, - }); - return; - - default: - // send message over socket and get responses - try { - if (this._socket == null) { - await this.init_socket(); - } - - if (input.id == null) { - input.id = uuid(); - dbg(`generated new random uuid for input: '${input.id}' `); - } - - if (this._socket == null) { - throw new Error("no socket"); - } - - this._socket.write_mesg("json", input); - - this._output_cb[input.id] = cb; // this is when opts.cb will get called... - } catch (err) { - cb({ done: true, error: err }); - } - } - }; - private _handle_mesg_blob = (mesg: TCPMessage) => { - const { uuid } = mesg; - let { blob } = mesg; - const dbg = this.dbg(`_handle_mesg_blob(uuid='${uuid}')`); - dbg(); - - if (blob == null) { - dbg("no blob -- dropping message"); - return; - } - - // This should never happen, typing enforces this to be a Buffer - if (typeof blob === "string") { - dbg("blob is string -- converting to buffer"); - blob = Buffer.from(blob, "utf8"); - } - - this._client.save_blob({ - blob, - uuid, - cb: (err, resp) => { - if (err) { - resp = message.save_blob({ - error: err, - sha1: uuid, // dumb - that sha1 should be called uuid... - }); - } - this._socket?.write_mesg("json", resp); - }, - }); - }; - - private _handle_mesg_json = (mesg: TCPMessage) => { - const dbg = this.dbg("_handle_mesg_json"); - dbg(`mesg='${trunc_middle(to_json(mesg), 400)}'`); - if (mesg == null) return; // should not happen - const { id } = mesg; - if (id == null) return; // should not happen - const cb = this._output_cb[id]; - if (cb != null) { - // Must do this check first since it uses done:false. - if (mesg.done || mesg.done == null) { - delete this._output_cb[id]; - mesg.done = true; - } - if (mesg.done != null && !mesg.done) { - // waste of space to include done part of mesg if just false for everything else... - delete mesg.done; - } - cb(mesg); - } - }; -} diff --git a/src/packages/project/sage_socket.ts b/src/packages/project/sage_socket.ts deleted file mode 100644 index c476dd52fc..0000000000 --- a/src/packages/project/sage_socket.ts +++ /dev/null @@ -1,99 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { getLogger } from "@cocalc/backend/logger"; -import { secretToken } from "@cocalc/project/data"; -import { enable_mesg } from "@cocalc/backend/misc_node"; -import { CoCalcSocket } from "@cocalc/backend/tcp/enable-messaging-protocol"; -import { connectToLockedSocket } from "@cocalc/backend/tcp/locked-socket"; -import * as message from "@cocalc/util/message"; -import * as common from "./common"; -import { forget_port, get_port } from "./port_manager"; -import { - SAGE_SERVER_MAX_STARTUP_TIME_S, - restartSageServer, -} from "./sage_restart"; -import { until, once } from "@cocalc/util/async-utils"; - -const logger = getLogger("get-sage-socket"); - -// Get a new connection to the Sage server. If the server -// isn't running, e.g., it was killed due to running out of memory, -// attempt to restart it and try to connect. -export async function getSageSocket(): Promise { - let socket: CoCalcSocket | undefined = undefined; - await until( - async () => { - try { - socket = await _getSageSocket(); - return true; - } catch (err) { - logger.debug( - `unable to get sage socket, so restarting sage server - ${err}`, - ); - // Failed for some reason: try to restart one time, then try again. - // We do this because the Sage server can easily get killed due to out of memory conditions. - // But we don't constantly try to restart the server, since it can easily fail to start if - // there is something wrong with a local Sage install. - // Note that restarting the sage server doesn't impact currently running worksheets (they - // have their own process that isn't killed). - try { - // starting the sage server can also easily fail, so must be in the try - await restartSageServer(); - // get socket immediately -- don't want to wait up to ~5s! - socket = await _getSageSocket(); - return true; - } catch (err) { - logger.debug( - `error restarting sage server or getting socket -- ${err}`, - ); - } - return false; - } - }, - { - start: 1000, - max: 5000, - decay: 1.5, - timeout: SAGE_SERVER_MAX_STARTUP_TIME_S * 1000, - }, - ); - if (socket === undefined) { - throw Error("bug"); - } - return socket; -} - -async function _getSageSocket(): Promise { - logger.debug("get sage server port"); - const port = await get_port("sage"); - logger.debug(`get and unlock socket on port ${port}`); - if (!port) { - throw new Error("port is not set"); - } - try { - const sage_socket: CoCalcSocket = await connectToLockedSocket({ - port, - token: secretToken, - }); - logger.debug("Successfully unlocked a sage session connection."); - - logger.debug("request sage session from server."); - enable_mesg(sage_socket); - sage_socket.write_mesg("json", message.start_session({ type: "sage" })); - logger.debug( - "Waiting to read one JSON message back, which will describe the session....", - ); - const [_type, desc] = await once(sage_socket, "mesg", 30000); - logger.debug(`Got message back from Sage server: ${common.json(desc)}`); - sage_socket.pid = desc.pid; - return sage_socket; - } catch (err) { - forget_port("sage"); - const msg = `_new_session: sage session denied connection: ${err}`; - logger.debug(`Failed to connect -- ${msg}`); - throw Error(msg); - } -} diff --git a/src/packages/project/sagews/README.md b/src/packages/project/sagews/README.md deleted file mode 100644 index 1c42c34bc7..0000000000 --- a/src/packages/project/sagews/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Sage Server - -Functionality: - -- keep track of list of running sage processes, with some info about them (total memory, time) -- write that list periodically to somewhere in the database (about this project), so can be shown to users -- start a sage process -- stop/kill a sage process -- interrupt a sage process -- evaluate a block of code (with data) and push out results as they appear -- maintain queue of evaluation requests -- if sage server likely to be started (based on previous _usage_ of this project), start it right when project starts up diff --git a/src/packages/project/sagews/control.ts b/src/packages/project/sagews/control.ts deleted file mode 100644 index fa31dd09f8..0000000000 --- a/src/packages/project/sagews/control.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { getLogger } from "@cocalc/backend/logger"; -const logger = getLogger("project:sagews:control"); - -export async function sagewsStart(path_ipynb: string) { - logger.debug("sagewsStart: ", path_ipynb); -} - -export async function sagewsStop(path_ipynb: string) { - logger.debug("sagewsStop: ", path_ipynb); -} diff --git a/src/requirements.txt b/src/requirements.txt index 09fd734ba7..1490c53f33 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,4 +1,3 @@ pyyaml==6.0.1 ./smc_pyutil -./smc_sagews diff --git a/src/smc_pyutil/smc_pyutil/bin/smc-sage-server b/src/smc_pyutil/smc_pyutil/bin/smc-sage-server deleted file mode 100755 index 9db51039c1..0000000000 --- a/src/smc_pyutil/smc_pyutil/bin/smc-sage-server +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -if which sage >/dev/null; then - sage -python -c "from smc_sagews.sage_server_command_line import main; main('$1')" -fi diff --git a/src/smc_sagews/README.md b/src/smc_sagews/README.md deleted file mode 100644 index d588499755..0000000000 --- a/src/smc_sagews/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# CoCalc Worksheet Server - -## Install - -To install into a copy of Sage, do this: -```sh -sage -pip install --upgrade ./ -``` -You will also probably want to install smc_pyutil systemwide. It provides a script `smc-sage-server` for starting/stopping the server. - -## Development - -In any project, just do this: - -```sh -cd cocalc/src/smc_sagews -smc-sage-server restart -``` - -Then on a sage worksheet that you are using, click the restart button. -It should be using your own custom copy of the smc_sagews server. -To confirm, type `sage_server?` and look at the path of the file. - -**UPDATE:** This might not work in 2022, but see [the comment here](https://github.com/sagemathinc/cocalc/issues/6219#issuecomment-1327918057) for something that does - -## Testing - -See smc_sagews/tests/README.md - diff --git a/src/smc_sagews/setup.py b/src/smc_sagews/setup.py deleted file mode 100644 index ccf2dc5834..0000000000 --- a/src/smc_sagews/setup.py +++ /dev/null @@ -1,36 +0,0 @@ -def readme(): - with open('README.md') as f: - return f.read() - - -from setuptools import setup - -setup( - name='smc_sagews', - version='1.2', - description='CoCalc Worksheets', - long_description=readme(), - url='https://github.com/sagemathinc/cocalc', - author='SageMath, Inc.', - author_email='office@sagemath.com', - license='GPLv3+', - packages=['smc_sagews'], - install_requires=[ - 'markdown2', - 'ansi2html', - 'ushlex', - 'six', - 'jupyter_client<7', # not compatible, see https://github.com/sagemathinc/cocalc/issues/5715 - ], - zip_safe=False, - classifiers=[ - 'License :: OSI Approved :: GPLv3', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.12', - 'Topic :: Mathematics :: Server', - ], - keywords='server mathematics cloud', - test_suite='nose.collector', - tests_require=['nose']) diff --git a/src/smc_sagews/smc_sagews/__init__.py b/src/smc_sagews/smc_sagews/__init__.py deleted file mode 100644 index 706c3a78cc..0000000000 --- a/src/smc_sagews/smc_sagews/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -# All our code in this modules assumes that an environment variable SMC exists -# and that the directory is points to also exists. We make it ~/.smc by default. - -from __future__ import absolute_import - -import os -if not 'SMC' in os.environ: - os.environ['SMC'] = os.path.join(os.environ['HOME'], '.smc') - -if not os.path.exists(os.environ['SMC']): - os.makedirs(os.environ['SMC']) \ No newline at end of file diff --git a/src/smc_sagews/smc_sagews/daemon.py b/src/smc_sagews/smc_sagews/daemon.py deleted file mode 100755 index b68d7a3393..0000000000 --- a/src/smc_sagews/smc_sagews/daemon.py +++ /dev/null @@ -1,122 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2009-2010 Sauce Labs Inc -# -# Portions taken from twistd: -# -# Copyright (c) 2001-2009 -# Allen Short -# Andrew Bennetts -# Apple Computer, Inc. -# Benjamin Bruheim -# Bob Ippolito -# Canonical Limited -# Christopher Armstrong -# David Reid -# Donovan Preston -# Eric Mangold -# Itamar Shtull-Trauring -# James Knight -# Jason A. Mobarak -# Jean-Paul Calderone -# Jonathan Lange -# Jonathan D. Simms -# Jürgen Hermann -# Kevin Turner -# Mary Gardiner -# Matthew Lefkowitz -# Massachusetts Institute of Technology -# Moshe Zadka -# Paul Swartz -# Pavel Pergamenshchik -# Ralph Meijer -# Sean Riley -# Software Freedom Conservancy -# Travis B. Hartwell -# Thomas Herve -# Eyal Lotem -# Antoine Pitrou -# Andy Gayton -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -from __future__ import absolute_import - -import os -import sys -import errno - - -def basic_daemonize(silence=True): - # See http://www.erlenstar.demon.co.uk/unix/faq_toc.html#TOC16 - if os.fork(): # launch child and... - os._exit(0) # kill off parent - os.setsid() - if os.fork(): # launch child and... - os._exit(0) # kill off parent again. - os.umask(0o022) # Don't allow others to write - - if not silence: - print("daemonize.basic_daemonize: process NOT silenced, for debugging") - else: - null = os.open('/dev/null', os.O_RDWR) - for i in range(3): - try: - os.dup2(null, i) - except OSError as e: - if e.errno != errno.EBADF: - raise - os.close(null) - - -def writePID(pidfile): - open(pidfile, 'w').write(str(os.getpid())) - if not os.path.exists(pidfile): - raise Exception("pidfile %s does not exist" % pidfile) - - -def checkPID(pidfile): - if not pidfile: - return - if os.path.exists(pidfile): - try: - pid = int(open(pidfile).read()) - except ValueError: - sys.exit('Pidfile %s contains non-numeric value' % pidfile) - try: - os.kill(pid, 0) - except OSError as why: - if why.args[0] == errno.ESRCH: - # The pid doesnt exists. - print(('Removing stale pidfile %s' % pidfile)) - os.remove(pidfile) - else: - sys.exit("Can't check status of PID %s from pidfile %s: %s" % - (pid, pidfile, why.args[1])) - else: - sys.exit("Another server is running, PID %s\n" % pid) - - -def daemonize(pidfile): - checkPID(pidfile) - # CRITICAL: DO NOT set silence=False in production. It hangs starting the sage server - # properly, which breaks things badly for users (e.g., their first worksheet - # never works). - basic_daemonize(silence=True) - writePID(pidfile) diff --git a/src/smc_sagews/smc_sagews/graphics.py b/src/smc_sagews/smc_sagews/graphics.py deleted file mode 100644 index c4de7051c1..0000000000 --- a/src/smc_sagews/smc_sagews/graphics.py +++ /dev/null @@ -1,645 +0,0 @@ -############################################################################### -# -# CoCalc: Collaborative Calculation -# -# Copyright (C) 2016, Sagemath Inc. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -############################################################################### - -from __future__ import absolute_import - -import json, math -from . import sage_salvus - -from uuid import uuid4 - - -def uuid(): - return str(uuid4()) - - -def json_float(t): - if t is None: - return t - t = float(t) - # Neither of nan or inf get JSON'd in a way that works properly, for some reason. I don't understand why. - if math.isnan(t) or math.isinf(t): - return None - else: - return t - - -####################################################### -# Three.js based plotting -####################################################### - -import sage.plot.plot3d.index_face_set -import sage.plot.plot3d.shapes -import sage.plot.plot3d.base -import sage.plot.plot3d.shapes2 -from sage.structure.element import Element - - -def jsonable(x): - if isinstance(x, Element): - return json_float(x) - elif isinstance(x, (list, tuple)): - return [jsonable(y) for y in x] - return x - - -def graphics3d_to_jsonable(p): - obj_list = [] - - def parse_obj(obj): - material_name = '' - faces = [] - for item in obj.split("\n"): - tmp = str(item.strip()) - if not tmp: - continue - k = tmp.split() - if k[0] == "usemtl": # material name - material_name = k[1] - elif k[0] == 'f': # face - v = [int(a) for a in k[1:]] - faces.append(v) - # other types are parse elsewhere in a different pass. - - return [{"material_name": material_name, "faces": faces}] - - def parse_texture(p): - texture_dict = [] - textures = p.texture_set() - for item in range(0, len(textures)): - texture_pop = textures.pop() - string = str(texture_pop) - item = string.split("(")[1] - name = item.split(",")[0] - color = texture_pop.color - tmp_dict = {"name": name, "color": color} - texture_dict.append(tmp_dict) - return texture_dict - - def get_color(name, texture_set): - for item in range(0, len(texture_set)): - if (texture_set[item]["name"] == name): - color = texture_set[item]["color"] - color_list = [color[0], color[1], color[2]] - break - else: - color_list = [] - return color_list - - def parse_mtl(p): - mtl = p.mtl_str() - all_material = [] - for item in mtl.split("\n"): - if "newmtl" in item: - tmp = str(item.strip()) - tmp_list = [] - try: - texture_set = parse_texture(p) - color = get_color(name, texture_set) - except (ValueError, UnboundLocalError): - pass - try: - tmp_list = { - "name": name, - "ambient": ambient, - "specular": specular, - "diffuse": diffuse, - "illum": illum_list[0], - "shininess": shininess_list[0], - "opacity": opacity_diffuse[3], - "color": color - } - all_material.append(tmp_list) - except (ValueError, UnboundLocalError): - pass - - ambient = [] - specular = [] - diffuse = [] - illum_list = [] - shininess_list = [] - opacity_diffuse = [] - tmp_list = [] - name = tmp.split()[1] - - if "Ka" in item: - tmp = str(item.strip()) - for t in tmp.split(): - try: - ambient.append(json_float(t)) - except ValueError: - pass - - if "Ks" in item: - tmp = str(item.strip()) - for t in tmp.split(): - try: - specular.append(json_float(t)) - except ValueError: - pass - - if "Kd" in item: - tmp = str(item.strip()) - for t in tmp.split(): - try: - diffuse.append(json_float(t)) - except ValueError: - pass - - if "illum" in item: - tmp = str(item.strip()) - for t in tmp.split(): - try: - illum_list.append(json_float(t)) - except ValueError: - pass - - if "Ns" in item: - tmp = str(item.strip()) - for t in tmp.split(): - try: - shininess_list.append(json_float(t)) - except ValueError: - pass - - if "d" in item: - tmp = str(item.strip()) - for t in tmp.split(): - try: - opacity_diffuse.append(json_float(t)) - except ValueError: - pass - - try: - color = list(p.all[0].texture.color.rgb()) - except (ValueError, AttributeError): - pass - - try: - texture_set = parse_texture(p) - color = get_color(name, texture_set) - except (ValueError, AttributeError): - color = [] - #pass - - tmp_list = { - "name": name, - "ambient": ambient, - "specular": specular, - "diffuse": diffuse, - "illum": illum_list[0], - "shininess": shininess_list[0], - "opacity": opacity_diffuse[3], - "color": color - } - all_material.append(tmp_list) - - return all_material - - ##################################### - # Conversion functions - ##################################### - - def convert_index_face_set(p, T, extra_kwds): - if T is not None: - p = p.transform(T=T) - face_geometry = parse_obj(p.obj()) - if hasattr(p, 'has_local_colors') and p.has_local_colors(): - convert_index_face_set_with_colors(p, T, extra_kwds) - return - material = parse_mtl(p) - vertex_geometry = [] - obj = p.obj() - for item in obj.split("\n"): - if "v" in item: - tmp = str(item.strip()) - for t in tmp.split(): - try: - vertex_geometry.append(json_float(t)) - except ValueError: - pass - myobj = { - "face_geometry": face_geometry, - "type": 'index_face_set', - "vertex_geometry": vertex_geometry, - "material": material, - "has_local_colors": 0 - } - for e in ['wireframe', 'mesh']: - if p._extra_kwds is not None: - v = p._extra_kwds.get(e, None) - if v is not None: - myobj[e] = jsonable(v) - obj_list.append(myobj) - - def convert_index_face_set_with_colors(p, T, extra_kwds): - face_geometry = [{ - "material_name": - p.texture.id, - "faces": [[int(v) + 1 for v in f[0]] + [f[1]] - for f in p.index_faces_with_colors()] - }] - material = parse_mtl(p) - vertex_geometry = [json_float(t) for v in p.vertices() for t in v] - myobj = { - "face_geometry": face_geometry, - "type": 'index_face_set', - "vertex_geometry": vertex_geometry, - "material": material, - "has_local_colors": 1 - } - for e in ['wireframe', 'mesh']: - if p._extra_kwds is not None: - v = p._extra_kwds.get(e, None) - if v is not None: - myobj[e] = jsonable(v) - obj_list.append(myobj) - - def convert_text3d(p, T, extra_kwds): - obj_list.append({ - "type": - "text", - "text": - p.string, - "pos": [0, 0, 0] if T is None else T([0, 0, 0]), - "color": - "#" + p.get_texture().hex_rgb(), - 'fontface': - str(extra_kwds.get('fontface', 'Arial')), - 'constant_size': - bool(extra_kwds.get('constant_size', True)), - 'fontsize': - int(extra_kwds.get('fontsize', 12)) - }) - - def convert_line(p, T, extra_kwds): - obj_list.append({ - "type": - "line", - "points": - jsonable(p.points if T is None else - [T.transform_point(point) for point in p.points]), - "thickness": - jsonable(p.thickness), - "color": - "#" + p.get_texture().hex_rgb(), - "arrow_head": - bool(p.arrow_head) - }) - - def convert_point(p, T, extra_kwds): - obj_list.append({ - "type": "point", - "loc": p.loc if T is None else T(p.loc), - "size": json_float(p.size), - "color": "#" + p.get_texture().hex_rgb() - }) - - def convert_combination(p, T, extra_kwds): - for x in p.all: - handler(x)(x, T, p._extra_kwds) - - def convert_transform_group(p, T, extra_kwds): - if T is not None: - T = T * p.get_transformation() - else: - T = p.get_transformation() - for x in p.all: - handler(x)(x, T, p._extra_kwds) - - def nothing(p, T, extra_kwds): - pass - - def handler(p): - if isinstance(p, sage.plot.plot3d.index_face_set.IndexFaceSet): - return convert_index_face_set - elif isinstance(p, sage.plot.plot3d.shapes.Text): - return convert_text3d - elif isinstance(p, sage.plot.plot3d.base.TransformGroup): - return convert_transform_group - elif isinstance(p, sage.plot.plot3d.base.Graphics3dGroup): - return convert_combination - elif isinstance(p, sage.plot.plot3d.shapes2.Line): - return convert_line - elif isinstance(p, sage.plot.plot3d.shapes2.Point): - return convert_point - elif isinstance(p, sage.plot.plot3d.base.PrimitiveObject): - return convert_index_face_set - elif isinstance(p, sage.plot.plot3d.base.Graphics3d): - # this is an empty scene - return nothing - else: - raise NotImplementedError("unhandled type ", type(p)) - - # start it going -- this modifies obj_list - handler(p)(p, None, None) - - # now obj_list is full of the objects - return obj_list - - -### -# Interactive 2d Graphics -### - -import os, matplotlib.figure - - -class InteractiveGraphics(object): - def __init__(self, g, **events): - self._g = g - self._events = events - - def figure(self, **kwds): - if isinstance(self._g, matplotlib.figure.Figure): - return self._g - - options = dict() - options.update(self._g.SHOW_OPTIONS) - options.update(self._g._extra_kwds) - options.update(kwds) - options.pop('dpi') - options.pop('transparent') - options.pop('fig_tight') - fig = self._g.matplotlib(**options) - - from matplotlib.backends.backend_agg import FigureCanvasAgg - canvas = FigureCanvasAgg(fig) - fig.set_canvas(canvas) - fig.tight_layout( - ) # critical, since sage does this -- if not, coords all wrong - return fig - - def save(self, filename, **kwds): - if isinstance(self._g, matplotlib.figure.Figure): - self._g.savefig(filename) - else: - # When fig_tight=True (the default), the margins are very slightly different. - # I don't know how to properly account for this yet (or even if it is possible), - # since it only happens at figsize time -- do "a=plot(sin); a.save??". - # So for interactive graphics, we just set this to false no matter what. - kwds['fig_tight'] = False - self._g.save(filename, **kwds) - - def show(self, **kwds): - fig = self.figure(**kwds) - ax = fig.axes[0] - # upper left data coordinates - xmin, ymax = ax.transData.inverted().transform( - fig.transFigure.transform((0, 1))) - # lower right data coordinates - xmax, ymin = ax.transData.inverted().transform( - fig.transFigure.transform((1, 0))) - - id = '_a' + uuid().replace('-', '') - - def to_data_coords(p): - # 0<=x,y<=1 - return ((xmax - xmin) * p[0] + xmin, - (ymax - ymin) * (1 - p[1]) + ymin) - - if kwds.get('svg', False): - filename = '%s.svg' % id - del kwds['svg'] - else: - filename = '%s.png' % id - - fig.savefig(filename) - - def f(event, p): - self._events[event](to_data_coords(p)) - - sage_salvus.salvus.namespace[id] = f - x = {} - for ev in list(self._events.keys()): - x[ev] = id - - sage_salvus.salvus.file(filename, show=True, events=x) - os.unlink(filename) - - def __del__(self): - for ev in self._events: - u = self._id + ev - if u in sage_salvus.salvus.namespace: - del sage_salvus.salvus.namespace[u] - - -### -# D3-based interactive 2d Graphics -### - - -### -# The following is a modified version of graph_plot_js.py from the Sage library, which was -# written by Nathann Cohen in 2013. -### -def graph_to_d3_jsonable(G, - vertex_labels=True, - edge_labels=False, - vertex_partition=[], - edge_partition=[], - force_spring_layout=False, - charge=-120, - link_distance=50, - link_strength=1, - gravity=.04, - vertex_size=7, - edge_thickness=2, - width=None, - height=None, - **ignored): - r""" - Display a graph in CoCalc using the D3 visualization library. - - INPUT: - - - ``G`` -- the graph - - - ``vertex_labels`` (boolean) -- Whether to display vertex labels (set to - ``True`` by default). - - - ``edge_labels`` (boolean) -- Whether to display edge labels (set to - ``False`` by default). - - - ``vertex_partition`` -- a list of lists representing a partition of the - vertex set. Vertices are then colored in the graph according to the - partition. Set to ``[]`` by default. - - - ``edge_partition`` -- same as ``vertex_partition``, with edges - instead. Set to ``[]`` by default. - - - ``force_spring_layout`` -- whether to take sage's position into account if - there is one (see :meth:`~sage.graphs.generic_graph.GenericGraph.` and - :meth:`~sage.graphs.generic_graph.GenericGraph.`), or to compute a spring - layout. Set to ``False`` by default. - - - ``vertex_size`` -- The size of a vertex' circle. Set to `7` by default. - - - ``edge_thickness`` -- Thickness of an edge. Set to ``2`` by default. - - - ``charge`` -- the vertices' charge. Defines how they repulse each - other. See ``_ for more - information. Set to ``-120`` by default. - - - ``link_distance`` -- See - ``_ for more - information. Set to ``30`` by default. - - - ``link_strength`` -- See - ``_ for more - information. Set to ``1.5`` by default. - - - ``gravity`` -- See - ``_ for more - information. Set to ``0.04`` by default. - - - EXAMPLES:: - - show(graphs.RandomTree(50), d3=True) - - show(graphs.PetersenGraph(), d3=True, vertex_partition=g.coloring()) - - show(graphs.DodecahedralGraph(), d3=True, force_spring_layout=True) - - show(graphs.DodecahedralGraph(), d3=True) - - g = digraphs.DeBruijn(2,2) - g.allow_multiple_edges(True) - g.add_edge("10","10","a") - g.add_edge("10","10","b") - g.add_edge("10","10","c") - g.add_edge("10","10","d") - g.add_edge("01","11","1") - show(g, d3=True, vertex_labels=True,edge_labels=True, - link_distance=200,gravity=.05,charge=-500, - edge_partition=[[("11","12","2"),("21","21","a")]], - edge_thickness=4) - - """ - directed = G.is_directed() - multiple_edges = G.has_multiple_edges() - - # Associated an integer to each vertex - v_to_id = {v: i for i, v in enumerate(G.vertices())} - - # Vertex colors - color = {i: len(vertex_partition) for i in range(G.order())} - for i, l in enumerate(vertex_partition): - for v in l: - color[v_to_id[v]] = i - - # Vertex list - nodes = [] - for v in G.vertices(): - nodes.append({"name": str(v), "group": str(color[v_to_id[v]])}) - - # Edge colors. - edge_color_default = "#aaa" - from sage.plot.colors import rainbow - color_list = rainbow(len(edge_partition)) - edge_color = {} - for i, l in enumerate(edge_partition): - for e in l: - u, v, label = e if len(e) == 3 else e + (None, ) - edge_color[u, v, label] = color_list[i] - if not directed: - edge_color[v, u, label] = color_list[i] - - # Edge list - edges = [] - seen = {} # How many times has this edge been seen ? - - for u, v, l in G.edges(): - - # Edge color - color = edge_color.get((u, v, l), edge_color_default) - - # Computes the curve of the edge - curve = 0 - - # Loop ? - if u == v: - seen[u, v] = seen.get((u, v), 0) + 1 - curve = seen[u, v] * 10 + 10 - - # For directed graphs, one also has to take into accounts - # edges in the opposite direction - elif directed: - if G.has_edge(v, u): - seen[u, v] = seen.get((u, v), 0) + 1 - curve = seen[u, v] * 15 - else: - if multiple_edges and len(G.edge_label(u, v)) != 1: - # Multiple edges. The first one has curve 15, then - # -15, then 30, then -30, ... - seen[u, v] = seen.get((u, v), 0) + 1 - curve = (1 if seen[u, v] % 2 else -1) * (seen[u, v] // - 2) * 15 - - elif not directed and multiple_edges: - # Same formula as above for multiple edges - if len(G.edge_label(u, v)) != 1: - seen[u, v] = seen.get((u, v), 0) + 1 - curve = (1 if seen[u, v] % 2 else -1) * (seen[u, v] // 2) * 15 - - # Adding the edge to the list - edges.append({ - "source": v_to_id[u], - "target": v_to_id[v], - "strength": 0, - "color": color, - "curve": curve, - "name": str(l) if edge_labels else "" - }) - - loops = [e for e in edges if e["source"] == e["target"]] - edges = [e for e in edges if e["source"] != e["target"]] - - # Defines the vertices' layout if possible - Gpos = G.get_pos() - pos = [] - if Gpos is not None and force_spring_layout is False: - charge = 0 - link_strength = 0 - gravity = 0 - - for v in G.vertices(): - x, y = Gpos[v] - pos.append([json_float(x), json_float(-y)]) - - return { - "nodes": nodes, - "links": edges, - "loops": loops, - "pos": pos, - "directed": G.is_directed(), - "charge": int(charge), - "link_distance": int(link_distance), - "link_strength": int(link_strength), - "gravity": float(gravity), - "vertex_labels": bool(vertex_labels), - "edge_labels": bool(edge_labels), - "vertex_size": int(vertex_size), - "edge_thickness": int(edge_thickness), - "width": json_float(width), - "height": json_float(height) - } diff --git a/src/smc_sagews/smc_sagews/julia.py b/src/smc_sagews/smc_sagews/julia.py deleted file mode 100644 index 5e5e3e5493..0000000000 --- a/src/smc_sagews/smc_sagews/julia.py +++ /dev/null @@ -1,490 +0,0 @@ -r""" -Pexpect-based interface to Julia - -EXAMPLES:: - - TODO - -AUTHORS: - -- William Stein (2014-10-26) -""" - -########################################################################## -# -# Copyright (C) 2016, Sagemath Inc. -# -# Distributed under the terms of the GNU General Public License (GPL) -# -# http://www.gnu.org/licenses/ -# -########################################################################## - -from __future__ import absolute_import - -import os, pexpect - -import six -def is_string(s): - return isinstance(s, six.string_types) - -from uuid import uuid4 - - -def uuid(): - return str(uuid4()) - - -from sage.interfaces.expect import Expect, ExpectElement, ExpectFunction, FunctionElement, gc_disabled -from sage.structure.element import RingElement - -PROMPT_LENGTH = 16 - - -class Julia(Expect): - def __init__(self, - maxread=100000, - script_subdirectory=None, - logfile=None, - server=None, - server_tmpdir=None): - """ - Pexpect-based interface to Julia - """ - self._prompt = 'julia>' - Expect.__init__(self, - name='Julia', - prompt=self._prompt, - command="julia", - maxread=maxread, - server=server, - server_tmpdir=server_tmpdir, - script_subdirectory=script_subdirectory, - restart_on_ctrlc=False, - verbose_start=False, - logfile=logfile) - - self.__seq = 0 - self.__in_seq = 1 - - def _start(self): - """ - """ - pexpect_env = dict(os.environ) - pexpect_env[ - 'TERM'] = 'vt100' # we *use* the codes. DUH. I should have thought of this 10 years ago... - self._expect = pexpect.spawn(self._Expect__command, - logfile=self._Expect__logfile, - env=pexpect_env) - self._expect.delaybeforesend = 0 # not a good idea for a CAS. - self._expect.expect("\x1b\[0Kjulia>") - - def eval(self, code, **ignored): - """ - """ - if is_string(code): - code = code.encode('utf8') - - START = "\x1b[?2004l\x1b[0m" - END = "\x1b[0G\x1b[0K\x1b[0G\x1b[0Kjulia> " - if not self._expect: - self._start() - with gc_disabled(): - s = self._expect - u = uuid() - line = code + '\n\n\n\n\n__ans__=ans;println("%s");ans=__ans__;\n' % u - s.send(line) - s.expect(u) - result = s.before - self._last_result = result - s.expect(u) - self._last_result += s.before - s.expect(u) - self._last_result += s.before - i = result.rfind(START) - if i == -1: - return result - result = result[len(START) + i:] - i = result.find(END) - if i == -1: - return result - result = result[:i].rstrip() - if result.startswith("ERROR:"): - julia_error = result.replace("in anonymous at no file", '') - raise RuntimeError(julia_error) - return result - - def _an_element_impl(self): - """ - EXAMPLES:: - - sage: julia._an_element_impl() - 0 - """ - return self(0) - - def set(self, var, value): - """ - Set the variable var to the given value. - - EXAMPLES:: - - sage: julia.set('x', '2') - sage: julia.get('x') - '2' - - TEST: - - It must also be possible to eval the variable by name:: - - sage: julia.eval('x') - '2' - """ - cmd = '%s=%s;' % (var, value) - out = self.eval(cmd) - if '***' in out: #TODO - raise TypeError( - "Error executing code in Sage\nCODE:\n\t%s\nSAGE ERROR:\n\t%s" - % (cmd, out)) - - def get(self, var): - """ - EXAMPLES:: - - sage: julia.set('x', '2') - sage: julia.get('x') - '2' - """ - out = self.eval(var) - return out - - def _repr_(self): - return 'Julia Interpreter' - - def __reduce__(self): - """ - EXAMPLES:: - - sage: julia.__reduce__() - """ - return reduce_load_Julia, tuple([]) - - def _quit_string(self): - """ - EXAMPLES:: - - sage: julia._quit_string() - 'quit()' - - sage: l = Julia() - sage: l._start() - sage: l.quit() - sage: l.is_running() - False - """ - return 'quit()' - - def _read_in_file_command(self, filename): - """ - EXAMPLES:: - - sage: julia._read_in_file_command(tmp_filename()) # TODO - """ - def trait_names(self): - """ - EXAMPLES:: - - sage: julia.trait_names() - ['ANY', ..., 'zip'] - """ - s = julia.eval('\t\t') - v = [] - for x in s.split('\x1b[')[:-1]: - i = x.find("G") - if i != -1: - c = x[i + 1:].strip() - if c and c.isalnum(): - v.append(c) - v.sort() - return v - - def kill(self, var): - """ - EXAMPLES:: - - sage: julia.kill('x') - Traceback (most recent call last): - ... - NotImplementedError - """ - raise NotImplementedError - - def console(self): - """ - Spawn a new Julia command-line session. - - EXAMPLES:: - - sage: julia.console() #not tested - ... - """ - julia_console() - - def version(self): - """ - Returns the version of Julia being used. - - EXAMPLES:: - - sage: julia.version() - 'Version information is given by julia.console().' - """ - return self.eval("versioninfo()") - - def _object_class(self): - """ - EXAMPLES:: - - sage: julia._object_class() - - """ - return JuliaElement - - def _function_class(self): - """ - EXAMPLES:: - - sage: julia._function_class() - - """ - return JuliaFunction - - def _function_element_class(self): - """ - EXAMPLES:: - - sage: julia._function_element_class() - - """ - return JuliaFunctionElement - - def _true_symbol(self): - """ - EXAMPLES:: - - sage: julia._true_symbol() - 'true' - """ - return 'true' - - def _false_symbol(self): - """ - EXAMPLES:: - - sage: julia._false_symbol() - 'false' - """ - return 'false' - - def _equality_symbol(self): - """ - """ - return "==" - - def help(self, command): - """ - EXAMPLES:: - - - """ - if '"' in command: - raise ValueError('quote in command name') - return self.eval('help("%s")' % command) - - def function_call(self, function, args=None, kwds=None): - """ - EXAMPLES:: - - sage: julia.function_call('sin', ['2']) - 0.9092974 - sage: julia.sin(2) - 0.9092974 - """ - args, kwds = self._convert_args_kwds(args, kwds) - self._check_valid_function_name(function) - return self.new("%s(%s)" % - (function, ",".join([s.name() for s in args]))) - - -class JuliaElement(ExpectElement): - def trait_names(self): - # for now... (until I understand types) - return self._check_valid().trait_names() - - def __richcmp__(self, other): - """ - EXAMPLES:: - - sage: one = julia(1); two = julia(2) - sage: one == one - True - sage: one != two - True - sage: one < two - True - sage: two > one - True - sage: one < 1 - False - sage: two == 2 - True - - """ - P = self._check_valid() - if not hasattr(other, 'parent') or P is not other.parent(): - other = P(other) - - if P.eval('%s == %s' % - (self.name(), other.name())) == P._true_symbol(): - return 0 - elif P.eval('%s < %s' % - (self.name(), other.name())) == P._true_symbol(): - return -1 - else: - return 1 - - def bool(self): - """ - EXAMPLES:: - - sage: julia(2).bool() - True - sage: julia(0).bool() - False - sage: bool(julia(2)) - True - """ - P = self._check_valid() - return P.eval("bool(%s)" % self.name()) == P._true_symbol() - - def _add_(self, right): - """ - EXAMPLES:: - - sage: a = julia(1); b = julia(2) - sage: a + b - 3 - """ - P = self._check_valid() - return P.new('%s + %s' % (self._name, right._name)) - - def _sub_(self, right): - """ - EXAMPLES:: - - sage: a = julia(1); b = julia(2) - sage: a - b - -1 - """ - P = self._check_valid() - return P.new('%s - %s' % (self._name, right._name)) - - def _mul_(self, right): - """ - EXAMPLES:: - - sage: a = julia(1); b = julia(2) - sage: a * b - 2 - """ - P = self._check_valid() - return P.new('%s * %s' % (self._name, right._name)) - - def _div_(self, right): - """ - EXAMPLES:: - - sage: a = julia(1); b = julia(2) - sage: a / b - 1/2 - """ - P = self._check_valid() - return P.new('%s / %s' % (self._name, right._name)) - - def __pow__(self, n): - """ - EXAMPLES:: - - sage: a = julia(3) - sage: a^3 - 27 - """ - P = self._check_valid() - right = P(n) - return P.new('%s ^ %s' % (self._name, right._name)) - - -class JuliaFunctionElement(FunctionElement): - def _sage_doc_(self): - """ - EXAMPLES:: - - sage: two = julia(2) - sage: two.sin._sage_doc_() - 'Base.sin(x)\r\n\r\n Compute sine of "x", where "x" is in radians' - """ - M = self._obj.parent() - return M.help(self._name) - - -class JuliaFunction(ExpectFunction): - def _sage_doc_(self): - """ - EXAMPLES:: - - sage: julia.sin._sage_doc_() - Traceback (most recent call last): - ... - NotImplementedError - """ - M = self._parent - return M.help(self._name) - - -def is_JuliaElement(x): - """ - EXAMPLES:: - - sage: from sage.interfaces.julia import is_JuliaElement - sage: is_JuliaElement(julia(2)) - True - sage: is_JuliaElement(2) - False - """ - return isinstance(x, JuliaElement) - - -# An instance -julia = Julia() - - -def reduce_load_Julia(): - """ - EXAMPLES:: - - sage: from sage.interfaces.julia import reduce_load_Julia - sage: reduce_load_Julia() - Julia Interpreter - """ - return julia - - -def julia_console(): - """ - Spawn a new Julia command-line session. - - EXAMPLES:: - - sage: julia.console() #not tested - ... - """ - os.system('julia') diff --git a/src/smc_sagews/smc_sagews/markdown2Mathjax.py b/src/smc_sagews/smc_sagews/markdown2Mathjax.py deleted file mode 100644 index 09d91df4dc..0000000000 --- a/src/smc_sagews/smc_sagews/markdown2Mathjax.py +++ /dev/null @@ -1,221 +0,0 @@ -from __future__ import absolute_import - -__version_info__ = (0, 3, 9) -__version__ = '.'.join(map(str, __version_info__)) -__author__ = "Matthew Young" - -import re -from markdown2 import markdown - - -def break_tie(inline, equation): - """If one of the delimiters is a substring of the other (e.g., $ and $$) it is possible that the two will begin at the same location. In this case we need some criteria to break the tie and decide which operation takes precedence. I've gone with the longer of the two delimiters takes priority (for example, $$ over $). This function should return a 2 for the equation block taking precedence, a 1 for the inline block. The magic looking return statement is to map 0->2 and 1->1.""" - tmp = (inline.end() - inline.start() > equation.end() - equation.start()) - return (tmp * 3 + 2) % 4 - - -def markdown_safe(placeholder): - """Is the placeholder changed by markdown? If it is, this isn't a valid placeholder.""" - mdstrip = re.compile("

(.*)

\n") - md = markdown(placeholder) - mdp = mdstrip.match(md) - if mdp and mdp.group(1) == placeholder: - return True - return False - - -def mathdown(text): - """Convenience function which runs the basic markdown and mathjax processing sequentially.""" - tmp = sanitizeInput(text) - return reconstructMath(markdown(tmp[0]), tmp[1]) - - -def sanitizeInput(string, - inline_delims=["$", "$"], - equation_delims=["$$", "$$"], - placeholder="$0$"): - """Given a string that will be passed to markdown, the content of the different math blocks is stripped out and replaced by a placeholder which MUST be ignored by markdown. A list is returned containing the text with placeholders and a list of the stripped out equations. Note that any pre-existing instances of the placeholder are "replaced" with themselves and a corresponding dummy entry is placed in the returned codeblock. The sanitized string can then be passed safetly through markdown and then reconstructed with reconstructMath. - - There are potential four delimiters that can be specified. The left and right delimiters for inline and equation mode math. These can potentially be anything that isn't already used by markdown and is compatible with mathjax (see documentation for both). - """ - #Check placeholder is valid. - if not markdown_safe(placeholder): - raise ValueError("Placeholder %s altered by markdown processing." % - placeholder) - #really what we want is a reverse markdown function, but as that's too much work, this will do - inline_left = re.compile("(?= 0):tmp] - #Set the new post - post = tmp - #Back to start! - continue - elif startmatches[1] is None and startmatches[2] is None: - #No more blocks, add in the rest of string and be done with it... - sanitizedString = sanitizedString + string[post * (post >= 0):] - return (sanitizedString, codeblocks) - elif startmatches[1] is None: - inBlock = 2 - elif startmatches[2] is None: - inBlock = 1 - else: - inBlock = (startpoints[1] < - startpoints[2]) + (startpoints[1] > startpoints[2]) * 2 - if not inBlock: - inBlock = break_tie(startmatches[1], startmatches[2]) - #Magic to ensure minimum index is 0 - sanitizedString = sanitizedString + string[ - (post * (post >= 0)):startpoints[inBlock]] - post = startmatches[inBlock].end() - #Now find the matching end... - while terminator < post: - endpoint = scanners[inBlock][1].search() - #If we run out of terminators before ending this loop, we're done - if endpoint is None: - #Add the unterminated codeblock to the sanitized string - sanitizedString = sanitizedString + string[ - startpoints[inBlock]:] - return (sanitizedString, codeblocks) - terminator = endpoint.start() - #We fonud a matching endpoint, add the bit to the appropriate codeblock... - codeblocks.append(str(inBlock) + string[post:endpoint.start()]) - #Now add in the appropriate placeholder - sanitizedString = sanitizedString + placeholder - #Fabulous. Now we can start again once we update post... - post = endpoint.end() - - -def reconstructMath(processedString, - codeblocks, - inline_delims=["$", "$"], - equation_delims=["$$", "$$"], - placeholder="$0$", - htmlSafe=False): - """This is usually the output of sanitizeInput, after having passed the output string through markdown. The delimiters given to this function should match those used to construct the string to begin with. - - This will output a string containing html suitable to use with mathjax. - - "<" and ">" "&" symbols in math can confuse the html interpreter because they mark the begining and end of definition blocks. To avoid issues, if htmlSafe is set to True these symbols will be replaced by ascii codes in the math blocks. The downside to this is that if anyone is already doing this, there already niced text might be mangled (I think I've taken steps to make sure it won't but not extensively tested...)""" - delims = [['', ''], inline_delims, equation_delims] - placeholder_re = re.compile("(?", ">") - #Step through the codeblocks one at a time and replace the next occurance of the placeholder. Extra placeholders are invalid math blocks and ignored... - outString = '' - scan = placeholder_re.scanner(processedString) - post = 0 - for i in range(len(codeblocks)): - inBlock = int(codeblocks[i][0]) - match = scan.search() - if not match: - raise ValueError( - "More codeblocks given than valid placeholders in text.") - outString = outString + processedString[post:match.start( - )] + delims[inBlock][0] + codeblocks[i][1:] + delims[inBlock][1] - post = match.end() - #Add the rest of the string (if we need to) - if post < len(processedString): - outString = outString + processedString[post:] - return outString - - -def findBoundaries(string): - """A depricated function. Finds the location of string boundaries in a stupid way.""" - last = '' - twod = [] - oned = [] - boundary = False - inoned = False - intwod = False - for count, char in enumerate(string): - if char == "$" and last != '\\': - #We just hit a valid $ character! - if inoned: - oned.append(count) - inoned = False - elif intwod: - if boundary: - twod.append(count) - intwod = False - boundary = False - else: - boundary = True - elif boundary: - #This means the last character was also a valid $ - twod.append(count) - intwod = True - boundary = False - else: - #This means the last character was NOT a useable $ - boundary = True - elif boundary: - #The last character was a valid $, but this one isn't... - #This means the last character was a valid $, but this isn't - if inoned: - print("THIS SHOULD NEVER HAPPEN!") - elif intwod: - #ignore it... - pass - else: - oned.append(count - 1) - inoned = True - boundary = False - last = char - #What if we finished on a boundary character? Actually doesn't matter, but let's include it for completeness - if boundary: - if not (inoned or intwod): - oned.append(count) - inoned = True - return (oned, twod) diff --git a/src/smc_sagews/smc_sagews/sage_jupyter.py b/src/smc_sagews/smc_sagews/sage_jupyter.py deleted file mode 100644 index 521c5c73cf..0000000000 --- a/src/smc_sagews/smc_sagews/sage_jupyter.py +++ /dev/null @@ -1,435 +0,0 @@ -""" -sage_jupyter.py - -Spawn and send commands to jupyter kernels. - -AUTHORS: - - Hal Snyder (main author) - - William Stein - - Harald Schilly -""" - -######################################################################################### -# Copyright (C) 2016, SageMath, Inc. # -# # -# Distributed under the terms of the GNU General Public License (GPL), version 2+ # -# # -# http://www.gnu.org/licenses/ # -######################################################################################### - -from __future__ import absolute_import -import os -import string -import textwrap -import six - -salvus = None # set externally - -# jupyter kernel - - -class JUPYTER(object): - def __call__(self, kernel_name, **kwargs): - if kernel_name.startswith('sage'): - raise ValueError( - "You may not run Sage kernels from a Sage worksheet.\nInstead use the sage_select command in a Terminal to\nswitch to a different version of Sage, then restart your project." - ) - return _jkmagic(kernel_name, **kwargs) - - def available_kernels(self): - ''' - Returns the list of available Jupyter kernels. - ''' - v = os.popen("jupyter kernelspec list").readlines() - return ''.join(x for x in v if not x.strip().startswith('sage')) - - def _get_doc(self): - ds0 = textwrap.dedent(r"""\ - Use the jupyter command to use any Jupyter kernel that you have installed using from your CoCalc worksheet - - | py3 = jupyter("python3") - - After that, begin a sagews cell with %py3 to send statements to the Python3 - kernel that you just created: - - | %py3 - | print(42) - - You can even draw graphics. - - | %py3 - | import numpy as np; import pylab as plt - | x = np.linspace(0, 3*np.pi, 500) - | plt.plot(x, np.sin(x**2)) - | plt.show() - - You can set the default mode for all cells in the worksheet. After putting the following - in a cell, click the "restart" button, and you have an anaconda worksheet. - - | %auto - | anaconda5 = jupyter('anaconda5') - | %default_mode anaconda5 - - Each call to jupyter creates its own Jupyter kernel. So you can have more than - one instance of the same kernel type in the same worksheet session. - - | p1 = jupyter('python3') - | p2 = jupyter('python3') - | p1('a = 5') - | p2('a = 10') - | p1('print(a)') # prints 5 - | p2('print(a)') # prints 10 - - For details on supported features and known issues, see the SMC Wiki page: - https://github.com/sagemathinc/cocalc/wiki/sagejupyter - """) - # print("calling JUPYTER._get_doc()") - kspec = self.available_kernels() - ks2 = kspec.replace("kernels:\n ", "kernels:\n\n|") - return ds0 + ks2 - - __doc__ = property(_get_doc) - - -jupyter = JUPYTER() - - -def _jkmagic(kernel_name, **kwargs): - r""" - Called when user issues `my_kernel = jupyter("kernel_name")` from a cell. - These are not intended to be called directly by user. - - Start a jupyter kernel and create a sagews function for it. See docstring for class JUPYTER above. - Based on http://jupyter-client.readthedocs.io/en/latest/api/index.html - - INPUT: - - - ``kernel_name`` -- name of kernel as it appears in output of `jupyter kernelspec list` - - """ - # CRITICAL: We import these here rather than at module scope, since they can take nearly a second - # of CPU time to import. - import jupyter_client # TIMING: takes a bit of time - from ansi2html import Ansi2HTMLConverter # TIMING: this is surprisingly bad. - from six.moves.queue import Empty # TIMING: cheap - import base64, tempfile, sys, re # TIMING: cheap - - import warnings - import sage.misc.latex - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - km, kc = jupyter_client.manager.start_new_kernel( - kernel_name=kernel_name) - import atexit - atexit.register(km.shutdown_kernel) - atexit.register(kc.hb_channel.close) - - # inline: no header or style tags, useful for full == False - # linkify: little gimmik, translates URLs to anchor tags - conv = Ansi2HTMLConverter(inline=True, linkify=True) - - def hout(s, block=True, scroll=False, error=False): - r""" - wrapper for ansi conversion before displaying output - - INPUT: - - - ``s`` - string to display in output of sagews cell - - - ``block`` - set false to prevent newlines between output segments - - - ``scroll`` - set true to put output into scrolling div - - - ``error`` - set true to send text output to stderr - """ - # `full = False` or else cell output is huge - if "\x1b[" in s: - # use html output if ansi control code found in string - h = conv.convert(s, full=False) - if block: - h2 = '
' + h + '
' - else: - h2 = '
' + h + '
' - if scroll: - h2 = '
' + h2 + '
' - salvus.html(h2) - else: - if error: - sys.stderr.write(s) - sys.stderr.flush() - else: - sys.stdout.write(s) - sys.stdout.flush() - - def run_code(code=None, **kwargs): - def p(*args): - from smc_sagews.sage_server import log - if run_code.debug: - log("kernel {}: {}".format(kernel_name, - ' '.join(str(a) for a in args))) - - if kwargs.get('get_kernel_client', False): - return kc - - if kwargs.get('get_kernel_manager', False): - return km - - if kwargs.get('get_kernel_name', False): - return kernel_name - - if code is None: - return - - # execute the code - msg_id = kc.execute(code) - - # get responses - shell = kc.shell_channel - iopub = kc.iopub_channel - stdinj = kc.stdin_channel - - # buffering for %capture because we don't know whether output is stdout or stderr - # until shell execute_reply message is received with status 'ok' or 'error' - capture_mode = not hasattr(sys.stdout._f, 'im_func') - - # handle iopub messages - while True: - try: - msg = iopub.get_msg() - msg_type = msg['msg_type'] - content = msg['content'] - - except Empty: - # shouldn't happen - p("iopub channel empty") - break - - p('iopub', msg_type, str(content)[:300]) - - if msg['parent_header'].get('msg_id') != msg_id: - p('*** non-matching parent header') - continue - - if msg_type == 'status' and content['execution_state'] == 'idle': - break - - def display_mime(msg_data): - ''' - jupyter server does send data dictionaries, that do contain mime-type:data mappings - depending on the type, handle them in the salvus API - ''' - # sometimes output is sent in several formats - # 1. if there is an image format, prefer that - # 2. elif default text or image mode is available, prefer that - # 3. else choose first matching format in modes list - from smc_sagews.sage_salvus import show - - def show_plot(data, suffix): - r""" - If an html style is defined for this kernel, use it. - Otherwise use salvus.file(). - """ - suffix = '.' + suffix - fname = tempfile.mkstemp(suffix=suffix)[1] - fmode = 'wb' if six.PY3 else 'w' - with open(fname, fmode) as fo: - fo.write(data) - - if run_code.smc_image_scaling is None: - salvus.file(fname) - else: - img_src = salvus.file(fname, show=False) - # The max-width is because this smc-image-scaling is very difficult - # to deal with when using React to render this on the share server, - # and not scaling down is really ugly. When the width gets set - # as normal in a notebook, this won't impact anything, but when - # it is displayed on share server (where width is not set) at least - # it won't look like total crap. See https://github.com/sagemathinc/cocalc/issues/4421 - htms = ''.format( - img_src, run_code.smc_image_scaling) - salvus.html(htms) - os.unlink(fname) - - mkeys = list(msg_data.keys()) - imgmodes = ['image/svg+xml', 'image/png', 'image/jpeg'] - txtmodes = [ - 'text/html', 'text/plain', 'text/latex', 'text/markdown' - ] - if any('image' in k for k in mkeys): - dfim = run_code.default_image_fmt - #print('default_image_fmt %s'%dfim) - dispmode = next((m for m in mkeys if dfim in m), None) - if dispmode is None: - dispmode = next(m for m in imgmodes if m in mkeys) - #print('dispmode is %s'%dispmode) - # https://en.wikipedia.org/wiki/Data_scheme#Examples - # latex") - show(msg_data['text/latex']) - else: - txt = re.sub(r"^\[\d+\] ", "", msg_data[dispmode]) - hout(txt) - elif dispmode == 'text/html': - salvus.html(msg_data[dispmode]) - elif dispmode == 'text/latex': - p('text/latex', msg_data[dispmode]) - sage.misc.latex.latex.eval(msg_data[dispmode]) - elif dispmode == 'text/markdown': - salvus.md(msg_data[dispmode]) - return - - # reminder of iopub loop is switch on value of msg_type - - if msg_type == 'execute_input': - # the following is a cheat to avoid forking a separate thread to listen on stdin channel - # most of the time, ignore "execute_input" message type - # but if code calls python3 input(), wait for message on stdin channel - if 'code' in content: - ccode = content['code'] - if kernel_name.startswith( - ('python', 'anaconda', 'octave')) and re.match( - r'^[^#]*\W?input\(', ccode): - # FIXME input() will be ignored if it's aliased to another name - p('iopub input call: ', ccode) - try: - # do nothing if no messsage on stdin channel within 0.5 sec - imsg = stdinj.get_msg(timeout=0.5) - imsg_type = imsg['msg_type'] - icontent = imsg['content'] - p('stdin', imsg_type, str(icontent)[:300]) - # kernel is now blocked waiting for input - if imsg_type == 'input_request': - prompt = '' if icontent[ - 'password'] else icontent['prompt'] - value = salvus.raw_input(prompt=prompt) - xcontent = dict(value=value) - xmsg = kc.session.msg('input_reply', xcontent) - p('sending input_reply', xcontent) - stdinj.send(xmsg) - except: - pass - elif kernel_name == 'octave' and re.search( - r"\s*pause\s*([#;\n].*)?$", ccode, re.M): - # FIXME "1+2\npause\n3+4" pauses before executing any code - # would need block parser here - p('iopub octave pause: ', ccode) - try: - # do nothing if no messsage on stdin channel within 0.5 sec - imsg = stdinj.get_msg(timeout=0.5) - imsg_type = imsg['msg_type'] - icontent = imsg['content'] - p('stdin', imsg_type, str(icontent)[:300]) - # kernel is now blocked waiting for input - if imsg_type == 'input_request': - prompt = "Paused, enter any value to continue" - value = salvus.raw_input(prompt=prompt) - xcontent = dict(value=value) - xmsg = kc.session.msg('input_reply', xcontent) - p('sending input_reply', xcontent) - stdinj.send(xmsg) - except: - pass - elif msg_type == 'execute_result': - if not 'data' in content: - continue - p('execute_result data keys: ', list(content['data'].keys())) - display_mime(content['data']) - - elif msg_type == 'display_data': - if 'data' in content: - display_mime(content['data']) - - elif msg_type == 'status': - if content['execution_state'] == 'idle': - # when idle, kernel has executed all input - break - - elif msg_type == 'clear_output': - salvus.clear() - - elif msg_type == 'stream': - if 'text' in content: - # bash kernel uses stream messages with output in 'text' field - # might be ANSI color-coded - if 'name' in content and content['name'] == 'stderr': - hout(content['text'], error=True) - else: - hout(content['text'], block=False) - - elif msg_type == 'error': - # XXX look for ename and evalue too? - if 'traceback' in content: - tr = content['traceback'] - if isinstance(tr, list): - for tr in content['traceback']: - hout(tr + '\n', error=True) - else: - hout(tr, error=True) - - # handle shell messages - while True: - try: - msg = shell.get_msg(timeout=0.2) - msg_type = msg['msg_type'] - content = msg['content'] - except Empty: - # shouldn't happen - p("shell channel empty") - break - if msg['parent_header'].get('msg_id') == msg_id: - p('shell', msg_type, len(str(content)), str(content)[:300]) - if msg_type == 'execute_reply': - if content['status'] == 'ok': - if 'payload' in content: - payload = content['payload'] - if len(payload) > 0: - if 'data' in payload[0]: - data = payload[0]['data'] - if 'text/plain' in data: - text = data['text/plain'] - hout(text, scroll=True) - break - else: - # not our reply - continue - return - - # 'html', 'plain', 'latex', 'markdown' - support depends on jupyter kernel - run_code.default_text_fmt = 'html' - - # 'svg', 'png', 'jpeg' - support depends on jupyter kernel - run_code.default_image_fmt = 'png' - - # set to floating point fraction e.g. 0.5 - run_code.smc_image_scaling = None - - # set True to record jupyter messages to sage_server log - run_code.debug = False - - # allow `anaconda.jupyter_kernel.kernel_name` etc. - run_code.kernel_name = kernel_name - - return run_code diff --git a/src/smc_sagews/smc_sagews/sage_parsing.py b/src/smc_sagews/smc_sagews/sage_parsing.py deleted file mode 100644 index 2b7a61dd2f..0000000000 --- a/src/smc_sagews/smc_sagews/sage_parsing.py +++ /dev/null @@ -1,659 +0,0 @@ -""" -sage_parser.py - -Code for parsing Sage code blocks sensibly. -""" - -######################################################################################### -# Copyright (C) 2016, Sagemath Inc. -# # -# Distributed under the terms of the GNU General Public License (GPL), version 2+ # -# # -# http://www.gnu.org/licenses/ # -######################################################################################### - -from __future__ import absolute_import -import string -import traceback -import __future__ as future -import ast - -# for the "input()" call -import six - - -def get_future_features(code, mode): - if '__future__' not in code: - return {} - features = {} - node = ast.parse(code, mode=mode) - #Make it work for all outer-container node types (module, interactive, expression) - body = getattr(node, 'body', ()) - if isinstance(body, ast.AST): - body = [body] - #The first non-future statement ends processing for future statements - for stmt in body: - #Future statements must be "from __future__ import ..." - if isinstance(stmt, ast.ImportFrom): - if getattr(stmt, 'module', None) == '__future__': - for alias in stmt.names: - assert isinstance(alias, ast.alias) - name = alias.name - if (name not in future.all_feature_names): - raise SyntaxError( - "future feature %.50r is not defined: %.150r" % - (name, code)) - attr = getattr(future, alias.name, None) - if (attr is not None) and isinstance( - attr, future._Feature): - features[alias.name] = attr - else: - #If the module is not '__future__', we're done processing future statements - break - else: - #If the statement is not an "ImportFrom", we're done processing future statements - break - return features - - -def get_input(prompt): - try: - r = six.input(prompt) - z = r - if z.rstrip().endswith(':'): - while True: - try: - z = six.input('... ') - except EOFError: - quit = True - break - if z != '': - r += '\n ' + z - else: - break - return r - except EOFError: - return None - - -#def strip_leading_prompts(code, prompts=['sage:', '....:', '...:', '>>>', '...']): -# code, literals, state = strip_string_literals(code) -# code2 = [] -# for line in code.splitlines(): -# line2 = line.lstrip() -# for p in prompts: -# if line2.startswith(p): -# line2 = line2[len(p):] -# if p[0] != '.': -# line2 = line2.lstrip() -# break -# code2.append(line2) -# code = ('\n'.join(code2))%literals -# return code - - -def preparse_code(code): - import sage.all_cmdline - return sage.all_cmdline.preparse(code, ignore_prompts=True) - - -def strip_string_literals(code, state=None): - new_code = [] - literals = {} - counter = 0 - start = q = 0 - if state is None: - in_quote = False - raw = False - else: - in_quote, raw = state - while True: - sig_q = code.find("'", q) - dbl_q = code.find('"', q) - hash_q = code.find('#', q) - q = min(sig_q, dbl_q) - if q == -1: q = max(sig_q, dbl_q) - if not in_quote and hash_q != -1 and (q == -1 or hash_q < q): - # it's a comment - newline = code.find('\n', hash_q) - if newline == -1: newline = len(code) - counter += 1 - label = "L%s" % counter - literals[label] = code[hash_q:newline] - new_code.append(code[start:hash_q].replace('%', '%%')) - new_code.append("%%(%s)s" % label) - start = q = newline - elif q == -1: - if in_quote: - counter += 1 - label = "L%s" % counter - literals[label] = code[start:] - new_code.append("%%(%s)s" % label) - else: - new_code.append(code[start:].replace('%', '%%')) - break - elif in_quote: - if code[q - 1] == '\\': - k = 2 - while code[q - k] == '\\': - k += 1 - if k % 2 == 0: - q += 1 - if code[q:q + len(in_quote)] == in_quote: - counter += 1 - label = "L%s" % counter - literals[label] = code[start:q + len(in_quote)] - new_code.append("%%(%s)s" % label) - q += len(in_quote) - start = q - in_quote = False - else: - q += 1 - else: - raw = q > 0 and code[q - 1] in 'rR' - if len(code) >= q + 3 and (code[q + 1] == code[q] == code[q + 2]): - in_quote = code[q] * 3 - else: - in_quote = code[q] - new_code.append(code[start:q].replace('%', '%%')) - start = q - q += len(in_quote) - - return "".join(new_code), literals, (in_quote, raw) - - -def end_of_expr(s): - """ - The input string s is a code expression that contains no strings (they have been stripped). - Find the end of the expression that starts at the beginning of s by finding the first whitespace - at which the parenthesis and brackets are matched. - - The returned index is the position *after* the expression. - """ - i = 0 - parens = 0 - brackets = 0 - while i < len(s): - c = s[i] - if c == '(': - parens += 1 - elif c == '[': - brackets += 1 - elif c == ')': - parens -= 1 - elif c == ']': - brackets -= 1 - elif parens == 0 and brackets == 0 and (c == ' ' or c == '\t'): - return i - i += 1 - return i - - -# NOTE/TODO: The dec_args dict will leak memory over time. However, it only -# contains code that was entered, so it should never get big. It -# seems impossible to know for sure whether a bit of code will be -# eventually needed later, so this leakiness seems necessary. -dec_counter = 0 -dec_args = {} - - -# Divide the input code (a string) into blocks of code. -def divide_into_blocks(code): - global dec_counter - - # strip string literals from the input, so that we can parse it without having to worry about strings - code, literals, state = strip_string_literals(code) - - # divide the code up into line lines. - code = code.splitlines() - - # Compute the line-level code decorators. - c = list(code) - try: - v = [] - for line in code: - done = False - - # Transform shell escape into sh decorator. - if line.lstrip().startswith('!'): - line = line.replace('!', "%%sh ", 1) - - # Check for cell decorator - # NOTE: strip_string_literals maps % to %%, because %foo is used for python string templating. - if line.lstrip().startswith('%%'): - i = line.find("%") - j = end_of_expr( - line[i + - 2:]) + i + 2 + 1 # +1 for the space or tab delimiter - expr = line[j:] % literals - # Special case -- if % starts line *and* expr is empty (or a comment), - # then code decorators impacts the rest of the code. - sexpr = expr.strip() - if i == 0 and (len(sexpr) == 0 or sexpr.startswith('#')): - new_line = '%ssalvus.execute_with_code_decorators(*_salvus_parsing.dec_args[%s])' % ( - line[:i], dec_counter) - expr = ('\n'.join(code[len(v) + 1:])) % literals - done = True - else: - # Expr is nonempty -- code decorator only impacts this line - new_line = '%ssalvus.execute_with_code_decorators(*_salvus_parsing.dec_args[%s])' % ( - line[:i], dec_counter) - - dec_args[dec_counter] = ([line[i + 2:j] % literals], expr) - dec_counter += 1 - else: - new_line = line - v.append(new_line) - if done: - break - code = v - except Exception as mesg: - code = c - - ## Tested this: Completely disable block parsing: - ## but it requires the caller to do "exec compile(block+'\n', '', 'exec') in namespace, locals", which means no display hook, - ## so "2+2" breaks. - ## return [[0,len(code)-1,('\n'.join(code))%literals]] - - # Remove comment lines -- otherwise could get empty blocks that can't be exec'd. - # For example, exec compile('#', '', 'single') is a syntax error. - # Also, comments will confuse the code to break into blocks before. - comment_lines = {} - for label, v in literals.items(): - if v.startswith('#'): - comment_lines["%%(%s)s" % label] = True - code = [x for x in code if not comment_lines.get(x.strip(), False)] - - # take only non-whitespace lines now for Python code (string literals have already been removed). - code = [x for x in code if x.strip()] - - # Compute the blocks - i = len(code) - 1 - blocks = [] - while i >= 0: - stop = i - paren_depth = code[i].count('(') - code[i].count(')') - brack_depth = code[i].count('[') - code[i].count(']') - curly_depth = code[i].count('{') - code[i].count('}') - while i >= 0 and ( - (len(code[i]) > 0 and (code[i][0] in string.whitespace)) - or paren_depth < 0 or brack_depth < 0 or curly_depth < 0): - i -= 1 - if i >= 0: - paren_depth += code[i].count('(') - code[i].count(')') - brack_depth += code[i].count('[') - code[i].count(']') - curly_depth += code[i].count('{') - code[i].count('}') - block = ('\n'.join(code[i:])) % literals - bs = block.strip() - if bs: # has to not be only whitespace - blocks.insert(0, [i, stop, bs]) - code = code[:i] - i = len(code) - 1 - - # merge try/except/finally/decorator/else/elif blocks - i = 1 - - def merge(): - "Merge block i-1 with block i." - blocks[i - 1][-1] += '\n' + blocks[i][-1] - blocks[i - 1][1] = blocks[i][1] - del blocks[i] - - while i < len(blocks): - s = blocks[i][-1].lstrip() - - # finally/except lines after a try - if (s.startswith('finally') or s.startswith('except') - ) and blocks[i - 1][-1].lstrip().startswith('try'): - merge() - - # function definitions - elif (s.startswith('def') or s.startswith('@')) and blocks[ - i - 1][-1].splitlines()[-1].lstrip().startswith('@'): - merge() - - # lines starting with else conditions (if *and* for *and* while!) - elif s.startswith('else') and ( - blocks[i - 1][-1].lstrip().startswith('if') - or blocks[i - 1][-1].lstrip().startswith('while') - or blocks[i - 1][-1].lstrip().startswith('for') - or blocks[i - 1][-1].lstrip().startswith('try') - or blocks[i - 1][-1].lstrip().startswith('elif')): - merge() - - # lines starting with elif - elif s.startswith('elif') and blocks[i - - 1][-1].lstrip().startswith('if'): - merge() - - # do not merge blocks -- move on to next one - else: - i += 1 - - return blocks - - -############################################ - -CHARS0 = string.ascii_letters + string.digits + '_' -CHARS = CHARS0 + '.' - - -def guess_last_expression( - obj): # TODO: bad guess -- need to use a parser to go any further. - i = len(obj) - 1 - while i >= 0 and obj[i] in CHARS: - i -= 1 - return obj[i + 1:] - - -def is_valid_identifier(target): - if len(target) == 0: return False - for x in target: - if x not in CHARS0: - return False - if target[0] not in string.ascii_letters + '_': - return False - return True - - -# Keywords from http://docs.python.org/release/2.7.2/reference/lexical_analysis.html -_builtin_completions = list(__builtins__.keys()) + [ - 'and', 'del', 'from', 'not', 'while', 'as', 'elif', 'global', 'or', 'with', - 'assert', 'else', 'if', 'pass', 'yield', 'break', 'except', 'import', - 'print', 'class', 'exec', 'in', 'raise', 'continue', 'finally', 'is', - 'return', 'def', 'for', 'lambda', 'try' -] - - -def introspect(code, namespace, preparse=True): - """ - INPUT: - - - code -- a string containing Sage (if preparse=True) or Python code. - - - namespace -- a dictionary to complete in (we also complete using - builtins such as 'def', 'for', etc. - - - preparse -- a boolean - - OUTPUT: - - An object: {'result':, 'target':, 'expr':, 'status':, 'get_help':, 'get_completions':, 'get_source':} - """ - import re - # result: the docstring, source code, or list of completions (at - # return, it might thus be either a list or a string) - result = [] - - # expr: the part of code that is used to do the completion, e.g., - # for 'a = n.m.foo', expr would be 'n.m.foo'. It can be more complicated, - # e.g., for '(2+3).foo.bar' it would be '(2+3).foo'. - expr = '' - - # target: for completions, target is the part of the code that we - # complete on in the namespace defined by the object right before - # it, e.g., for n.m.foo, the target is "foo". target is the empty - # string for source code and docstrings. - target = '' - - # When returning, exactly one of the following will be true: - get_help = False # getting docstring of something - get_source = False # getting source code of a function - get_completions = True # getting completions of an identifier in some namespace - - try: - # Strip all strings from the code, replacing them by template - # symbols; this makes parsing much easier. - # we strip, since trailing space could cause confusion below - code0, literals, state = strip_string_literals(code.strip()) - - # Move i so that it points to the start of the last expression in the code. - # (TODO: this should probably be replaced by using ast on preparsed version. Not easy.) - i = max([code0.rfind(t) for t in '\n;=']) + 1 - while i < len(code0) and code0[i] in string.whitespace: - i += 1 - - # Break the line in two pieces: before_expr | expr; we may - # need before_expr in order to evaluate and make sense of - # expr. We also put the string literals back in, so that - # evaluation works. - expr = code0[i:] % literals - before_expr = code0[:i] % literals - - chrs = set('.()[]? ') - if not any(c in expr for c in chrs): - # Easy case: this is just completion on a simple identifier in the namespace. - get_help = False - get_completions = True - get_source = False - target = expr - else: - # Now for all of the other harder cases. - i = max([expr.rfind(s) for s in '?(']) - # expr ends in two ?? -- source code - if i >= 1 and i == len(expr) - 1 and expr[i - 1] == '?': - get_source = True - get_completions = False - get_help = False - target = "" - obj = expr[:i - 1] - # ends in ( or ? (but not ??) -- docstring - elif i == len(expr) - 1: - get_help = True - get_completions = False - get_source = False - target = "" - obj = expr[:i] - # completions (not docstrings or source) - else: - get_help = False - get_completions = True - get_source = False - i = expr.rfind('.') - target = expr[i + 1:] - if target == '' or is_valid_identifier( - target) or '*' in expr and '* ' not in expr: - # this case includes list.*end[tab] - obj = expr[:i] - else: - # this case includes aaa=...;3 * aa[tab] - expr = guess_last_expression(target) - i = expr.rfind('.') - if i != -1: - target = expr[i + 1:] - obj = expr[:i] - else: - target = expr - - if get_completions and target == expr: - j = len(expr) - if '*' in expr: - # this case includes *_factors and abc =...;3 * ab[tab] - try: - pattern = expr.replace("*", ".*").replace("?", ".") - reg = re.compile(pattern + "$") - v = list( - filter(reg.match, - list(namespace.keys()) + _builtin_completions)) - # for 2*sq[tab] - if len(v) == 0: - gle = guess_last_expression(expr) - j = len(gle) - if j > 0: - target = gle - v = [ - x[j:] for x in (list(namespace.keys()) + - _builtin_completions) - if x.startswith(gle) - ] - except: - pass - else: - v = [ - x[j:] - for x in (list(namespace.keys()) + _builtin_completions) - if x.startswith(expr) - ] - # for 2+sqr[tab] - if len(v) == 0: - gle = guess_last_expression(expr) - j = len(gle) - if j > 0 and j < len(expr): - target = gle - v = [ - x[j:] for x in (list(namespace.keys()) + - _builtin_completions) - if x.startswith(gle) - ] - else: - - # We will try to evaluate - # obj. This is danerous and a priori could take - # forever, so we spend at most 1 second doing this -- - # if it takes longer a signal kills the evaluation. - # Obviously, this could in fact lock if - # non-interruptable code is called, which should be rare. - - O = None - try: - import signal - - def mysig(*args): - raise KeyboardInterrupt - - signal.signal(signal.SIGALRM, mysig) - signal.alarm(1) - import sage.all_cmdline - if before_expr.strip(): - try: - exec((before_expr if not preparse else - preparse_code(before_expr)), namespace) - except Exception as msg: - pass - # uncomment for debugging only - # traceback.print_exc() - # We first try to evaluate the part of the expression before the name - try: - O = eval(obj if not preparse else preparse_code(obj), - namespace) - except (SyntaxError, TypeError, AttributeError): - # If that fails, we try on a subexpression. - # TODO: This will not be needed when - # this code is re-written to parse using an - # AST, instead of using this lame hack. - obj = guess_last_expression(obj) - try: - O = eval(obj if not preparse else preparse_code(obj), - namespace) - except: - pass - finally: - signal.signal(signal.SIGALRM, signal.SIG_IGN) - - def get_file(): - try: - import sage.misc.sageinspect - eval_getdoc = eval('getdoc(O)', { - 'getdoc': sage.misc.sageinspect.sage_getfile, - 'O': O - }) - return " File: " + eval_getdoc + "\n" - except Exception as err: - return "Unable to read source filename (%s)" % err - - if get_help: - import sage.misc.sageinspect - result = get_file() - try: - - def our_getdoc(s): - try: - x = sage.misc.sageinspect.sage_getargspec(s) - defaults = list(x.defaults) if x.defaults else [] - args = list(x.args) if x.args else [] - v = [] - if x.keywords: - v.insert(0, '**kwds') - if x.varargs: - v.insert(0, '*args') - while defaults: - d = defaults.pop() - k = args.pop() - v.insert(0, '%s=%r' % (k, d)) - v = args + v - t = " Signature : %s(%s)\n" % (obj, ', '.join(v)) - except: - t = "" - try: - ds_raw = sage.misc.sageinspect.sage_getdoc(s) - if (six.PY3 and type(s) == bytes) or six.PY2: - ds = ds_raw.decode('utf-8') - else: - ds = ds_raw - ds = ds.strip() - t += " Docstring :\n%s" % ds - except Exception as ex: - t += " Problem retrieving Docstring :\n%s" % ex - # print ex # issue 1780: 'ascii' codec can't decode byte 0xc3 in position 3719: ordinal not in range(128) - pass - return t - - result += eval('getdoc(O)', {'getdoc': our_getdoc, 'O': O}) - except Exception as err: - result += "Unable to read docstring (%s)" % err - # Get rid of the 3 spaces in front of everything. - result = result.lstrip().replace('\n ', '\n') - - elif get_source: - import sage.misc.sageinspect - result = get_file() - try: - result += " Source:\n " + eval( - 'getsource(O)', { - 'getsource': sage.misc.sageinspect.sage_getsource, - 'O': O - }) - except Exception as err: - result += "Unable to read source code (%s)" % err - - elif get_completions: - if O is not None: - v = dir(O) - if hasattr(O, 'trait_names'): - v += O.trait_names() - if not target.startswith('_'): - v = [x for x in v if x and not x.startswith('_')] - # this case excludes abc = ...;for a in ab[tab] - if '*' in expr and '* ' not in expr: - try: - pattern = target.replace("*", ".*") - pattern = pattern.replace("?", ".") - reg = re.compile(pattern + "$") - v = list(filter(reg.match, v)) - except: - pass - else: - j = len(target) - v = [x[j:] for x in v if x.startswith(target)] - else: - v = [] - - if get_completions: - result = list(sorted(set(v), key=lambda x: x.lower())) - - except Exception as msg: - traceback.print_exc() - result = [] - status = 'ok' - else: - status = 'ok' - return { - 'result': result, - 'target': target, - 'expr': expr, - 'status': status, - 'get_help': get_help, - 'get_completions': get_completions, - 'get_source': get_source - } diff --git a/src/smc_sagews/smc_sagews/sage_salvus.py b/src/smc_sagews/smc_sagews/sage_salvus.py deleted file mode 100644 index d89eeb052b..0000000000 --- a/src/smc_sagews/smc_sagews/sage_salvus.py +++ /dev/null @@ -1,4503 +0,0 @@ -################################################################################## -# # -# Extra code that the Salvus server makes available in the running Sage session. # -# # -################################################################################## - -######################################################################################### -# Copyright (C) 2016, Sagemath Inc. -# # -# Distributed under the terms of the GNU General Public License (GPL), version 2+ # -# # -# http://www.gnu.org/licenses/ # -######################################################################################### - -from __future__ import absolute_import, division -import six - -# set backend of matplot lib before any other module is loaded -import matplotlib -matplotlib.use('Agg') - -import copy, os, sys, types, re - -import sage.all - - -def is_string(s): - return isinstance(s, six.string_types) - - -def is_dataframe(obj): - if 'pandas' not in str(type(obj)): - # avoid having to import pandas unless it's really likely to be necessary. - return - # CRITICAL: do not import pandas at the top level since it can take up to 3s -- it's **HORRIBLE**. - try: - from pandas import DataFrame - except ImportError: - return False - return isinstance(obj, DataFrame) - - -# This reduces a lot of confusion for Sage worksheets -- people expect -# to be able to import from the current working directory. -sys.path.append('.') - -salvus = None - - -def set_salvus(salvus_obj): - global salvus - salvus = salvus_obj - from . import sage_jupyter - sage_jupyter.salvus = salvus_obj - - -import json -from uuid import uuid4 - - -def uuid(): - return str(uuid4()) - - -########################################################################## -# New function interact implementation -########################################################################## -import inspect - -interacts = {} - - -def jsonable(x): - """ - Given any object x, make a JSON-able version of x, doing as best we can. - For some objects, sage as Sage integers, this works well. For other - objects which make no sense in Javascript, we get a string. - """ - import sage.all - try: - json.dumps(x) - return x - except: - if isinstance(x, (sage.all.Integer)): - return int(x) - else: - return str(x) - - -class InteractCell(object): - def __init__(self, - f, - layout=None, - width=None, - style=None, - update_args=None, - auto_update=True, - flicker=False, - output=True): - """ - Given a function f, create an object that describes an interact - for working with f interactively. - - INPUT: - - - `f` -- Python function - - ``width`` -- (default: None) overall width of the interact canvas - - ``style`` -- (default: None) extra CSS style to apply to canvas - - ``update_args`` -- (default: None) only call f if one of the args in - this list of strings changes. - - ``auto_update`` -- (default: True) call f every time an input changes - (or one of the arguments in update_args). - - ``flicker`` -- (default: False) if False, the output part of the cell - never shrinks; it can only grow, which aleviates flicker. - - ``output`` -- (default: True) if False, do not automatically - provide any area to display output. - """ - self._flicker = flicker - self._output = output - self._uuid = uuid() - # Prevent garbage collection until client specifically requests it, - # since we want to be able to store state. - interacts[self._uuid] = self - self._f = f - self._width = jsonable(width) - self._style = str(style) - - if six.PY3: - _fas = inspect.getfullargspec(f) - args, varargs, varkw, defaults = _fas.args, _fas.varargs, _fas.varkw, _fas.defaults - elif six.PY2: - (args, varargs, varkw, defaults) = inspect.getargspec(f) - - if defaults is None: - defaults = [] - - n = len(args) - len(defaults) - self._controls = dict([ - (arg, interact_control(arg, defaults[i - n] if i >= n else None)) - for i, arg in enumerate(args) - ]) - - self._last_vals = {} - for arg in args: - self._last_vals[arg] = self._controls[arg].default() - - self._ordered_args = args - self._args = set(args) - - if isinstance(layout, dict): - # Implement the layout = {'top':, 'bottom':, 'left':, - # 'right':} dictionary option that is in the Sage - # notebook. I personally think it is really awkward and - # unsuable, but there may be many interacts out there that - # use it. - # Example layout={'top': [['a', 'b'], ['x', 'y']], 'left': [['c']], 'bottom': [['d']]} - top = layout.get('top', []) - bottom = layout.get('bottom', []) - left = layout.get('left', []) - right = layout.get('right', []) - new_layout = [] - for row in top: - new_layout.append(row) - if len(left) > 0 and len(right) > 0: - new_layout.append(left[0] + [''] + right[0]) - del left[0] - del right[0] - elif len(left) > 0 and len(right) == 0: - new_layout.append(left[0] + ['']) - del left[0] - elif len(left) == 0 and len(right) > 0: - new_layout.append([''] + right[0]) - del right[0] - while len(left) > 0 and len(right) > 0: - new_layout.append(left[0] + ['_salvus_'] + right[0]) - del left[0] - del right[0] - while len(left) > 0: - new_layout.append(left[0]) - del left[0] - while len(right) > 0: - new_layout.append(right[0]) - del right[0] - for row in bottom: - new_layout.append(row) - layout = new_layout - - if layout is None: - layout = [[(str(arg), 12, None)] for arg in self._ordered_args] - else: - try: - v = [] - for row in layout: - new_row = [] - for x in row: - if is_string(x): - x = (x, ) - if len(x) == 1: - new_row.append((str(x[0]), 12 // len(row), None)) - elif len(x) == 2: - new_row.append((str(x[0]), int(x[1]), None)) - elif len(x) == 3: - new_row.append((str(x[0]), int(x[1]), str(x[2]))) - v.append(new_row) - layout = v - except: - raise ValueError( - "layout must be None or a list of tuples (variable_name, width, [optional label]), where width is an integer between 1 and 12, variable_name is a string, and label is a string. The widths in each row must add up to at most 12. The empty string '' denotes the output area." - ) - - # Append a row for any remaining controls: - layout_vars = set(sum([[x[0] for x in row] for row in layout], [])) - for v in args: - if v not in layout_vars: - layout.append([(v, 12, None)]) - - if self._output: - if '' not in layout_vars: - layout.append([('', 12, None)]) - - self._layout = layout - - # TODO -- this is UGLY - if not auto_update: - c = button('Update') - c._opts['var'] = 'auto_update' - self._controls['auto_update'] = c - self._ordered_args.append("auto_update") - layout.append([('auto_update', 2)]) - update_args = ['auto_update'] - - self._update_args = update_args - - def jsonable(self): - """ - Return a JSON-able description of this interact, which the client - can use for laying out controls. - """ - X = { - 'controls': - [self._controls[arg].jsonable() for arg in self._ordered_args], - 'id': - self._uuid - } - if self._width is not None: - X['width'] = self._width - if self._layout is not None: - X['layout'] = self._layout - X['style'] = self._style - X['flicker'] = self._flicker - return X - - def __call__(self, vals): - """ - Call self._f with inputs specified by vals. Any input variables not - specified in vals will have the value they had last time. - """ - self.changed = [str(x) for x in list(vals.keys())] - for k, v in vals.items(): - x = self._controls[k](v) - self._last_vals[k] = x - - if self._update_args is not None: - do_it = False - for v in self._update_args: - if v in self.changed: - do_it = True - if not do_it: - return - - interact_exec_stack.append(self) - try: - self._f(**dict([(k, self._last_vals[k]) for k in self._args])) - finally: - interact_exec_stack.pop() - - -class InteractFunction(object): - def __init__(self, interact_cell): - self.__dict__['interact_cell'] = interact_cell - - def __call__(self, **kwds): - salvus.clear() - for arg, value in kwds.items(): - self.__setattr__(arg, value) - return self.interact_cell(kwds) - - def __setattr__(self, arg, value): - I = self.__dict__['interact_cell'] - if arg in I._controls and not isinstance(value, control): - # setting value of existing control - v = I._controls[arg].convert_to_client(value) - desc = {'var': arg, 'default': v} - I._last_vals[arg] = value - else: - # create a new control - new_control = interact_control(arg, value) - I._controls[arg] = new_control - desc = new_control.jsonable() - # set the id of the containing interact - desc['id'] = I._uuid - salvus.javascript("worksheet.set_interact_var(obj)", - obj=jsonable(desc)) - - def __getattr__(self, arg): - I = self.__dict__['interact_cell'] - try: - return I._last_vals[arg] - except Exception as err: - print(err) - raise AttributeError( - "no interact control corresponding to input variable '%s'" % - arg) - - def __delattr__(self, arg): - I = self.__dict__['interact_cell'] - try: - del I._controls[arg] - except KeyError: - pass - desc = {'id': I._uuid, 'name': arg} - salvus.javascript("worksheet.del_interact_var(obj)", - obj=jsonable(desc)) - - def changed(self): - """ - Return the variables that changed since last evaluation of the interact function - body. [SALVUS only] - - For example:: - - @interact - def f(n=True, m=False, xyz=[1,2,3]): - print(n, m, xyz, interact.changed()) - """ - return self.__dict__['interact_cell'].changed - - -class _interact_layout: - def __init__(self, *args): - self._args = args - - def __call__(self, f): - return interact(f, *self._args) - - -class Interact(object): - """ - Use interact to create interactive worksheet cells with sliders, - text boxes, radio buttons, check boxes, color selectors, and more. - - Put ``@interact`` on the line before a function definition in a - cell by itself, and choose appropriate defaults for the variable - names to determine the types of controls (see tables below). You - may also put ``@interact(layout=...)`` to control the layout of - controls. Within the function, you may explicitly set the value - of the control corresponding to a variable foo to bar by typing - interact.foo = bar. - - Type "interact.controls.[tab]" to get access to all of the controls. - - INPUT: - - - ``f`` -- function - - ``width`` -- number, or string such as '80%', '300px', '20em'. - - ``style`` -- CSS style string, which allows you to change the border, - background color, etc., of the interact. - - ``update_args`` -- (default: None); list of strings, so that - only changing the corresponding controls causes the function to - be re-evaluated; changing other controls will not cause an update. - - ``auto_update`` -- (default: True); if False, a button labeled - 'Update' will appear which you can click on to re-evalute. - - ``layout`` -- (default: one control per row) a list [row0, - row1, ...] of lists of tuples row0 = [(var_name, width, - label), ...], where the var_name's are strings, the widths - must add up to at most 12, and the label is optional. This - will layout all of the controls and output using Twitter - Bootstraps "Fluid layout", with spans corresponding - to the widths. Use var_name='' to specify where the output - goes, if you don't want it to last. You may specify entries for - controls that you will create later using interact.var_name = foo. - - - NOTES: The flicker and layout options above are only in SALVUS. - For backwards compatibility with the Sage notebook, if layout - is a dictionary (with keys 'top', 'bottom', 'left', 'right'), - then the appropriate layout will be rendered as it used to be - in the Sage notebook. - - OUTPUT: - - - creates an interactive control. - - - AUTOMATIC CONTROL RULES - ----------------------- - - There are also some defaults that allow you to make controls - automatically without having to explicitly specify them. E.g., - you can make ``x`` a continuous slider of values between ``u`` and - ``v`` by just writing ``x=(u,v)`` in the argument list. - - - ``u`` - blank input_box - - ``u=elt`` - input_box with ``default=element``, unless other rule below - - ``u=(umin,umax)`` - continuous slider (really `100` steps) - - ``u=(umin,umax,du)`` - slider with step size ``du`` - - ``u=list`` - buttons if ``len(list)`` at most `5`; otherwise, drop down - - ``u=generator`` - a slider (up to `10000` steps) - - ``u=bool`` - a checkbox - - ``u=Color('blue')`` - a color selector; returns ``Color`` object - - ``u=matrix`` - an ``input_grid`` with ``to_value`` set to - ``matrix.parent()`` and default values given by the matrix - - ``u=(default, v)`` - ``v`` anything as above, with given ``default`` value - - ``u=(label, v)`` - ``v`` anything as above, with given ``label`` (a string) - - EXAMPLES: - - - The layout option:: - - @interact(layout={'top': [['a', 'b']], 'left': [['c']], - 'bottom': [['d']], 'right':[['e']]}) - def _(a=x^2, b=(0..20), c=100, d=x+1, e=sin(2)): - print(a+b+c+d+e) - - We illustrate some features that are only in Salvus, not in the - Sage cell server or Sage notebook. - - You can set the value of a control called foo to 100 using - interact.foo=100. For example:: - - @interact - def f(n=20, twice=None): - interact.twice = int(n)*2 - - - In this example, we create and delete multiple controls depending - on properties of the input:: - - @interact - def f(n=20, **kwds): - print(kwds) - n = Integer(n) - if n % 2 == 1: - del interact.half - else: - interact.half = input_box(n/2, readonly=True) - if n.is_prime(): - interact.is_prime = input_box('True', readonly=True) - else: - del interact.is_prime - - We illustrate not automatically updating the function until a - button is pressed:: - - @interact(auto_update=False) - def f(a=True, b=False): - print(a, b) - - You can access the value of a control associated to a variable foo - that you create using interact.foo, and check whether there is a - control associated to a given variable name using hasattr:: - - @interact - def f(): - if not hasattr(interact, 'foo'): - interact.foo = 'hello' - else: - print(interact.foo) - - An indecisive interact:: - - @interact - def f(n=selector(['yes', 'no'])): - for i in range(5): - interact.n = i%2 - sleep(.2) - - We use the style option to make a holiday interact:: - - @interact(width=25, - style="background-color:lightgreen; border:5px dashed red;") - def f(x=button('Merry ...',width=20)): - pass - - We make a little box that can be dragged around, resized, and is - updated via a computation (in this case, counting primes):: - - @interact(width=30, - style="background-color:lightorange; position:absolute; z-index:1000; box-shadow : 8px 8px 4px #888;") - def f(prime=text_control(label="Counting primes: ")): - salvus.javascript("cell.element.closest('.salvus-cell-output-interact').draggable().resizable()") - p = 2 - c = 1 - while True: - interact.prime = '%s, %.2f'%(p, float(c)/p) - p = next_prime(p) - c += 1 - sleep(.25) - """ - def __call__(self, - f=None, - layout=None, - width=None, - style=None, - update_args=None, - auto_update=True, - flicker=False, - output=True): - if f is None: - return _interact_layout(layout, width, style, update_args, - auto_update, flicker) - else: - return salvus.interact(f, - layout=layout, - width=width, - style=style, - update_args=update_args, - auto_update=auto_update, - flicker=flicker, - output=output) - - def __setattr__(self, arg, value): - I = interact_exec_stack[-1] - if arg in I._controls and not isinstance(value, control): - # setting value of existing control - v = I._controls[arg].convert_to_client(value) - desc = {'var': arg, 'default': v} - I._last_vals[arg] = value - else: - # create a new control - new_control = interact_control(arg, value) - I._controls[arg] = new_control - desc = new_control.jsonable() - desc['id'] = I._uuid - salvus.javascript("worksheet.set_interact_var(obj)", obj=desc) - - def __delattr__(self, arg): - try: - del interact_exec_stack[-1]._controls[arg] - except KeyError: - pass - desc['id'] = I._uuid - salvus.javascript("worksheet.del_interact_var(obj)", obj=jsonable(arg)) - - def __getattr__(self, arg): - try: - return interact_exec_stack[-1]._last_vals[arg] - except Exception as err: - raise AttributeError( - "no interact control corresponding to input variable '%s'" % - arg) - - def changed(self): - """ - Return the variables that changed since last evaluation of the interact function - body. [SALVUS only] - - For example:: - - @interact - def f(n=True, m=False, xyz=[1,2,3]): - print(n, m, xyz, interact.changed()) - """ - return interact_exec_stack[-1].changed - - -interact = Interact() -interact_exec_stack = [] - - -class control: - def __init__(self, - control_type, - opts, - repr, - convert_from_client=None, - convert_to_client=jsonable): - # The type of the control -- a string, used for CSS selectors, switches, etc. - self._control_type = control_type - # The options that define the control -- passed to client - self._opts = dict(opts) - # Used to print the control to a string. - self._repr = repr - # Callable that the control may use in converting from JSON - self._convert_from_client = convert_from_client - self._convert_to_client = convert_to_client - self._last_value = self._opts['default'] - - def convert_to_client(self, value): - try: - return self._convert_to_client(value) - except Exception as err: - sys.stderr.write("convert_to_client: %s -- %s\n" % (err, self)) - sys.stderr.flush() - return jsonable(value) - - def __call__(self, obj): - """ - Convert JSON-able object returned from client to describe - value of this control. - """ - if self._convert_from_client is not None: - try: - x = self._convert_from_client(obj) - except Exception as err: - sys.stderr.write("%s -- %s\n" % (err, self)) - sys.stderr.flush() - x = self._last_value - else: - x = obj - self._last_value = x - return x - - def __repr__(self): - return self._repr - - def label(self): - """Return the label of this control.""" - return self._opts['label'] - - def default(self): - """Return default value of this control.""" - return self(self._opts['default']) - - def type(self): - """Return type that values of this control are coerced to.""" - return self._opts['type'] - - def jsonable(self): - """Return JSON-able object the client browser uses to render the control.""" - X = {'control_type': self._control_type} - for k, v in self._opts.items(): - X[k] = jsonable(v) - return X - - -def list_of_first_n(v, n): - """Given an iterator v, return first n elements it produces as a list.""" - if not hasattr(v, 'next'): - v = v.__iter__() - w = [] - while n > 0: - try: - w.append(next(v)) - except StopIteration: - return w - n -= 1 - return w - - -def automatic_control(default): - from sage.all import Color - from sage.structure.element import is_Matrix - label = None - default_value = None - - for _ in range(2): - if isinstance(default, tuple) and len(default) == 2 and is_string( - default[0]): - label, default = default - if isinstance(default, tuple) and len(default) == 2 and hasattr( - default[1], '__iter__'): - default_value, default = default - - if isinstance(default, control): - if label: - default._opts['label'] = label - return default - elif is_string(default): - return input_box(default, label=label, type=str) - elif is_string(default): - return input_box(default, label=label, type=str) - elif isinstance(default, bool): - return checkbox(default, label=label) - elif isinstance(default, list): - return selector(default, - default=default_value, - label=label, - buttons=len(default) <= 5) - elif isinstance(default, Color): - return color_selector(default=default, label=label) - elif isinstance(default, tuple): - if len(default) == 2: - return slider(default[0], - default[1], - default=default_value, - label=label) - elif len(default) == 3: - return slider(default[0], - default[1], - default[2], - default=default_value, - label=label) - else: - return slider(list(default), default=default_value, label=label) - elif is_Matrix(default): - return input_grid(default.nrows(), - default.ncols(), - default=default.list(), - to_value=default.parent(), - label=label) - elif hasattr(default, '__iter__'): - return slider(list_of_first_n(default, 10000), - default=default_value, - label=label) - else: - return input_box(default, label=label) - - -def interact_control(arg, value): - if isinstance(value, control): - if value._opts['label'] is None: - value._opts['label'] = arg - c = value - else: - c = automatic_control(value) - if c._opts['label'] is None: - c._opts['label'] = arg - c._opts['var'] = arg - return c - - -def sage_eval(x, locals=None, **kwds): - if is_string(x): - x = str(x).strip() - if x.isspace(): - return None - from sage.all import sage_eval - return sage_eval(x, locals=locals, **kwds) - - -class ParseValue: - def __init__(self, type): - self._type = type - - def _eval(self, value): - if is_string(value): - if not value: - return '' - return sage_eval( - value, locals=None if salvus is None else salvus.namespace) - else: - return value - - def __call__(self, value): - from sage.all import Color - if self._type is None: - return self._eval(value) - elif self._type is str: - return str(value) - elif self._type is str: - return str(value) - elif self._type is Color: - try: - return Color(value) - except ValueError: - try: - return Color("#" + value) - except ValueError: - raise TypeError("invalid color '%s'" % value) - else: - return self._type(self._eval(value)) - - -def input_box(default=None, - label=None, - type=None, - nrows=1, - width=None, - readonly=False, - submit_button=None): - """ - An input box interactive control for use with the :func:`interact` command. - - INPUT: - - - default -- default value - - label -- label test - - type -- the type that the input is coerced to (from string) - - nrows -- (default: 1) the number of rows of the box - - width -- width; how wide the box is - - readonly -- is it read-only? - - submit_button -- defaults to true if nrows > 1 and false otherwise. - """ - return control(control_type='input-box', - opts=locals(), - repr="Input box", - convert_from_client=ParseValue(type)) - - -def checkbox(default=True, label=None, readonly=False): - """ - A checkbox interactive control for use with the :func:`interact` command. - """ - return control(control_type='checkbox', opts=locals(), repr="Checkbox") - - -def color_selector(default='blue', - label=None, - readonly=False, - widget=None, - hide_box=False): - """ - A color selector. - - SALVUS only: the widget option is ignored -- SALVUS only provides - bootstrap-colorpicker. - - EXAMPLES:: - - @interact - def f(c=color_selector()): - print(c) - """ - from sage.all import Color - default = Color(default).html_color() - return control(control_type='color-selector', - opts=locals(), - repr="Color selector", - convert_from_client=lambda x: Color(str(x)), - convert_to_client=lambda x: Color(x).html_color()) - - -def text_control(default='', label=None, classes=None): - """ - A read-only control that displays arbitrary HTML amongst the other - interact controls. This is very powerful, since it can display - any HTML. - - INPUT:: - - - ``default`` -- actual HTML to display - - ``label`` -- string or None - - ``classes`` -- space separated string of CSS classes - - EXAMPLES:: - - We output the factorization of a number in a text_control:: - - @interact - def f(n=2013, fact=text_control("")): - interact.fact = factor(n) - - We use a CSS class to make the text_control look like a button: - - @interact - def f(n=text_control("foo bar", classes='btn')): - pass - - We animate a picture into view: - - @interact - def f(size=[10,15,..,30], speed=[1,2,3,4]): - for k in range(size): - interact.g = text_control(""%(20*k)) - sleep(speed/50.0) - """ - return control(control_type='text', - opts=locals(), - repr="Text %r" % (default)) - - -def button(default=None, label=None, classes=None, width=None, icon=None): - """ - Create a button. [SALVUS only] - - You can tell that pressing this button triggered the interact - evaluation because interact.changed() will include the variable - name tied to the button. - - INPUT: - - - ``default`` -- value variable is set to - - ``label`` -- string (default: None) - - ``classes`` -- string if None; if given, space separated - list of CSS classes. e.g., Bootstrap CSS classes such as: - btn-primary, btn-info, btn-success, btn-warning, btn-danger, - btn-link, btn-large, btn-small, btn-mini. - See http://twitter.github.com/bootstrap/base-css.html#buttons - If button_classes a single string, that class is applied to all buttons. - - ``width`` - an integer or string (default: None); if given, - all buttons are this width. If an integer, the default units - are 'ex'. A string that specifies any valid HTML units (e.g., '100px', '3em') - is also allowed [SALVUS only]. - - ``icon`` -- None or string name of any icon listed at the font - awesome website (http://fortawesome.github.com/Font-Awesome/), e.g., 'fa-repeat' - - EXAMPLES:: - - @interact - def f(hi=button('Hello', label='', classes="btn-primary btn-large"), - by=button("By")): - if 'hi' in interact.changed(): - print("Hello to you, good sir.") - if 'by' in interact.changed(): - print("See you.") - - Some buttons with icons:: - - @interact - def f(n=button('repeat', icon='fa-repeat'), - m=button('see?', icon="fa-eye", classes="btn-large")): - print(interact.changed()) - """ - return control(control_type="button", - opts=locals(), - repr="Button", - convert_from_client=lambda x: default, - convert_to_client=lambda x: str(x)) - - -class Slider: - def __init__(self, start, stop, step_size, max_steps): - if isinstance(start, (list, tuple)): - self.vals = start - else: - if step_size is None: - if stop is None: - step_size = start / float(max_steps) - else: - step_size = (stop - start) / float(max_steps) - from sage.all import srange # sage range is much better/more flexible. - self.vals = srange(start, stop, step_size, include_endpoint=True) - # Now check to see if any of thee above constructed a list of - # values that exceeds max_steps -- if so, linearly interpolate: - if len(self.vals) > max_steps: - n = len(self.vals) // max_steps - self.vals = [self.vals[n * i] for i in range(len(self.vals) // n)] - - def to_client(self, val): - if val is None: - return 0 - if isinstance(val, (list, tuple)): - return [self.to_client(v) for v in val] - else: - # Find index into self.vals of closest match. - try: - return self.vals.index(val) # exact match - except ValueError: - pass - z = [(abs(val - x), i) for i, x in enumerate(self.vals)] - z.sort() - return z[0][1] - - def from_client(self, val): - if val is None: - return self.vals[0] - # val can be a n-tuple or an integer - if isinstance(val, (list, tuple)): - return tuple([self.vals[v] for v in val]) - else: - return self.vals[int(val)] - - -class InputGrid: - def __init__(self, nrows, ncols, default, to_value): - self.nrows = nrows - self.ncols = ncols - self.to_value = to_value - self.value = copy.deepcopy(self.adapt(default)) - - def adapt(self, x): - if not isinstance(x, list): - return [[x for _ in range(self.ncols)] for _ in range(self.nrows)] - elif not all(isinstance(elt, list) for elt in x): - return [[x[i * self.ncols + j] for j in range(self.ncols)] - for i in range(self.nrows)] - else: - return x - - def from_client(self, x): - if len(x) == 0: - self.value = [] - elif isinstance(x[0], list): - self.value = [[sage_eval(t) for t in z] for z in x] - else: - # x is a list of (unicode) strings -- we sage eval them all at once (instead of individually). - s = '[' + ','.join([str(t) for t in x]) + ']' - v = sage_eval(s) - self.value = [ - v[n:n + self.ncols] - for n in range(0, self.nrows * self.ncols, self.ncols) - ] - - return self.to_value( - self.value) if self.to_value is not None else self.value - - def to_client(self, x=None): - if x is None: - v = self.value - else: - v = self.adapt(x) - self.value = v # save value in our local cache - return [[repr(x) for x in y] for y in v] - - -def input_grid(nrows, ncols, default=0, label=None, to_value=None, width=5): - r""" - A grid of input boxes, for use with the :func:`interact` command. - - INPUT: - - - ``nrows`` - an integer - - ``ncols`` - an integer - - ``default`` - an object; the default put in this input box - - ``label`` - a string; the label rendered to the left of the box. - - ``to_value`` - a list; the grid output (list of rows) is - sent through this function. This may reformat the data or - coerce the type. - - ``width`` - an integer; size of each input box in characters - - EXAMPLES: - - Solving a system:: - - @interact - def _(m = input_grid(2,2, default = [[1,7],[3,4]], - label=r'$M\qquad =$', to_value=matrix, width=8), - v = input_grid(2,1, default=[1,2], - label=r'$v\qquad =$', to_value=matrix)): - try: - x = m.solve_right(v) - html('$$%s %s = %s$$'%(latex(m), latex(x), latex(v))) - except: - html('There is no solution to $$%s x=%s$$'%(latex(m), latex(v))) - - Squaring an editable and randomizable matrix:: - - @interact - def f(reset = button('Randomize', classes="btn-primary", icon="fa-th"), - square = button("Square", icon="fa-external-link"), - m = input_grid(4,4,default=0, width=5, label="m =", to_value=matrix)): - if 'reset' in interact.changed(): - print("randomize") - interact.m = [[random() for _ in range(4)] for _ in range(4)] - if 'square' in interact.changed(): - salvus.tex(m^2) - - """ - ig = InputGrid(nrows, ncols, default, to_value) - - return control(control_type='input-grid', - opts={ - 'default': ig.to_client(), - 'label': label, - 'width': width, - 'nrows': nrows, - 'ncols': ncols - }, - repr="Input Grid", - convert_from_client=ig.from_client, - convert_to_client=ig.to_client) - - -def slider(start, - stop=None, - step=None, - default=None, - label=None, - display_value=True, - max_steps=500, - step_size=None, - range=False, - width=None, - animate=True): - """ - An interactive slider control for use with :func:`interact`. - - There are several ways to call the slider function, but they all - take several named arguments: - - - ``default`` - an object (default: None); default value is closest - value. If range=True, default can also be a 2-tuple (low, high). - - ``label`` -- string - - ``display_value`` -- bool (default: True); whether to display the - current value to the right of the slider. - - ``max_steps`` -- integer, default: 500; this is the maximum - number of values that the slider can take on. Do not make - it too large, since it could overwhelm the client. [SALVUS only] - - ``range`` -- bool (default: False); instead, you can select - a range of values (lower, higher), which are returned as a - 2-tuple. You may also set the value of the slider or - specify a default value using a 2-tuple. - - ``width`` -- how wide the slider appears to the user [SALVUS only] - - ``animate`` -- True (default), False,"fast", "slow", or the - duration of the animation in milliseconds. [SALVUS only] - - You may call the slider function as follows: - - - slider([list of objects], ...) -- slider taking values the objects in the list - - - slider([start,] stop[, step]) -- slider over numbers from start - to stop. When step is given it specifies the increment (or - decrement); if it is not given, then the number of steps equals - the width of the control in pixels. In all cases, the number of - values will be shrunk to be at most the pixel_width, since it is - not possible to select more than this many values using a slider. - - EXAMPLES:: - - - Use one slider to modify the animation speed of another:: - - @interact - def f(speed=(50,100,..,2000), x=slider([1..50], animate=1000)): - if 'speed' in interact.triggers(): - print("change x to have speed {}".format(speed)) - del interact.x - interact.x = slider([1..50], default=interact.x, animate=speed) - return - """ - if step_size is not None: # for compat with sage - step = step_size - slider = Slider(start, stop, step, max_steps) - vals = [str(x) for x in slider.vals] # for display by the client - if range and default is None: - default = [0, len(vals) - 1] - return control(control_type='range-slider' if range else 'slider', - opts={ - 'default': slider.to_client(default), - 'label': label, - 'animate': animate, - 'vals': vals, - 'display_value': display_value, - 'width': width - }, - repr="Slider", - convert_from_client=slider.from_client, - convert_to_client=slider.to_client) - - -def range_slider(*args, **kwds): - """ - range_slider is the same as :func:`slider`, except with range=True. - - EXAMPLES: - - A range slider with a constraint:: - - @interact - def _(t = range_slider([1..1000], default=(100,200), label=r'Choose a range for $\alpha$')): - print(t) - """ - kwds['range'] = True - return slider(*args, **kwds) - - -def selector(values, - label=None, - default=None, - nrows=None, - ncols=None, - width=None, - buttons=False, - button_classes=None): - """ - A drop down menu or a button bar for use in conjunction with - the :func:`interact` command. We use the same command to - create either a drop down menu or selector bar of buttons, - since conceptually the two controls do exactly the same thing - - they only look different. If either ``nrows`` or ``ncols`` - is given, then you get a buttons instead of a drop down menu. - - INPUT: - - - ``values`` - either (1) a list [val0, val1, val2, ...] or (2) - a list of pairs [(val0, lbl0), (val1,lbl1), ...] in which case - all labels must be given -- use None to auto-compute a given label. - - ``label`` - a string (default: None); if given, this label - is placed to the left of the entire button group - - ``default`` - an object (default: first); default value in values list - - ``nrows`` - an integer (default: None); if given determines - the number of rows of buttons; if given, buttons=True - - ``ncols`` - an integer (default: None); if given determines - the number of columns of buttons; if given, buttons=True - - ``width`` - an integer or string (default: None); if given, - all buttons are this width. If an integer, the default units - are 'ex'. A string that specifies any valid HTML units (e.g., '100px', '3em') - is also allowed [SALVUS only]. - - ``buttons`` - a bool (default: False, except as noted - above); if True, use buttons - - ``button_classes`` - [SALVUS only] None, a string, or list of strings - of the of same length as values, whose entries are a whitespace-separated - string of CSS classes, e.g., Bootstrap CSS classes such as: - btn-primary, btn-info, btn-success, btn-warning, btn-danger, - btn-link, btn-large, btn-small, btn-mini. - See http://twitter.github.com/bootstrap/base-css.html#buttons - If button_classes a single string, that class is applied to all buttons. - """ - if (len(values) > 0 and isinstance(values[0], tuple) - and len(values[0]) == 2): - vals = [z[0] for z in values] - lbls = [str(z[1]) if z[1] is not None else None for z in values] - else: - vals = values - lbls = [None] * len(vals) - - for i in range(len(vals)): - if lbls[i] is None: - v = vals[i] - lbls[i] = v if is_string(v) else str(v) - - if default is None: - default = 0 - else: - try: - default = vals.index(default) - except IndexError: - default = 0 - - opts = dict(locals()) - for k in ['vals', 'values', 'i', 'v', 'z']: - if k in opts: - del opts[k] # these could have a big jsonable repr - - opts['lbls'] = lbls - return control(control_type='selector', - opts=opts, - repr="Selector labeled %r with values %s" % (label, values), - convert_from_client=lambda n: vals[int(n)], - convert_to_client=lambda x: vals.index(x)) - - -interact_functions = {} -interact_controls = [ - 'button', 'checkbox', 'color_selector', 'input_box', 'range_slider', - 'selector', 'slider', 'text_control', 'input_grid' -] - -for f in ['interact'] + interact_controls: - interact_functions[f] = globals()[f] - - -# A little magic so that "interact.controls.[tab]" shows all the controls. -class Controls: - pass - - -Interact.controls = Controls() -for f in interact_controls: - interact.controls.__dict__[f] = interact_functions[f] - -########################################################################################## -# Cell object -- programatically control the current cell. -########################################################################################## - - -class Cell(object): - def id(self): - """ - Return the UUID of the cell in which this function is called. - """ - return salvus._id - - def hide(self, component='input'): - """ - Hide the 'input' or 'output' component of a cell. - """ - salvus.hide(component) - - def show(self, component='input'): - """ - Show the 'input' or 'output' component of a cell. - """ - salvus.show(component) - - def hideall(self): - """ - Hide the input and output fields of the cell in which this code executes. - """ - salvus.hide('input') - salvus.hide('output') - - #def input(self, val=None): - # """ - # Get or set the value of the input component of the cell in - # which this code executes. - # """ - # salvus.javascript("cell.set_input(obj)", obj=val) - # - #def output(self, val=None): - # """ - # Get or set the value of the output component of the cell in - # which this code executes. - # """ - # salvus.javascript("cell.set_output(obj)", obj=val) - # return salvus.output(val, self._id) - - -cell = Cell() - -########################################################################################## -# Cell decorators -- aka "percent modes" -########################################################################################## - -import sage.misc.html -try: - _html = sage.misc.html.HTML() -except: - _html = sage.misc.html.HTMLFragmentFactory - - -class HTML: - """ - Cell mode that renders everything after %html as HTML - - EXAMPLES:: - - --- - %html -

A Title

-

Subtitle

- - --- - %html(hide=True) -

A Title

-

Subtitle

- - --- - %html("

A title

", hide=False) - - --- - %html(hide=False)

Title

- - """ - def __init__(self, hide=False): - self._hide = hide - - def __call__(self, *args, **kwds): - if len(kwds) > 0 and len(args) == 0: - return HTML(**kwds) - if len(args) > 0: - self._render(args[0], **kwds) - - def _render(self, s, hide=None): - if hide is None: - hide = self._hide - if hide: - salvus.hide('input') - salvus.html(s) - - def table(self, rows=None, header=False): - """ - Renders a given matrix or nested list as an HTML table. - - Arguments:: - - * **rows**: the rows of the table as a list of lists - * **header**: if True, the first row is formatted as a header (default: False) - """ - # TODO: support columns as in http://doc.sagemath.org/html/en/reference/misc/sage/misc/table.html - assert rows is not None, '"rows" is a mandatory argument, should be a list of lists' - - from sage.matrix.matrix import is_Matrix - import numpy as np - - if is_Matrix(rows): - table = list(rows) # list of Sage Vectors - elif isinstance(rows, np.ndarray): - table = rows.tolist() - else: - table = rows - - assert isinstance(table, - (tuple, list)), '"rows" must be a list of lists' - - def as_unicode(s): - ''' - This not only deals with unicode strings, but also converts e.g. `Integer` objects to a str - ''' - try: - if six.PY3: - return str(s, 'utf8') - else: - return str(s).encode('utf-8') - except: - return "?".encode('utf-8') - - def mk_row(row, header=False): - is_vector = hasattr(row, 'is_vector') and row.is_vector() - assert isinstance( - row, (tuple, list) - ) or is_vector, '"rows" must contain lists or vectors for each row' - tag = 'th' if header else 'td' - row = [ - '<{tag}>{}'.format(as_unicode(_), tag=tag) for _ in row - ] - return '{}'.format(''.join(row)) - - thead = '{}'.format(mk_row( - table.pop(0), header=True)) if header else '' - h_rows = [mk_row(row) for row in table] - html_table = '{}{}
' - self(html_table.format(thead, ''.join(h_rows))) - - -html = HTML() -html.iframe = _html.iframe # written in a way that works fine - - -def coffeescript(s=None, once=False): - """ - Execute code using CoffeeScript. - - For example: - - %coffeescript console.log 'hi' - - or - - coffeescript("console.log 'hi'") - - You may either pass in a string or use this as a cell decorator, - i.e., put %coffeescript at the top of a cell. - - If you set once=False, the code will be executed every time the output of the cell is rendered, e.g., - on load, like with %auto:: - - coffeescript('console.log("hi")', once=False) - - or - - %coffeescript(once=False) - console.log("hi") - - - EXTRA FUNCTIONALITY: - - When executing code, a function called print is defined, and objects cell and worksheet.:: - - print(1,2,'foo','bar') -- displays the inputs in the output cell - - cell -- has attributes cell.output (the html output box) and cell.cell_id - - worksheet -- has attributes project_page and editor, and methods interrupt, kill, and - - execute_code: (opts) => - opts = defaults opts, - code : required - data : undefined - preparse : true - cb : undefined - - OPTIMIZATION: When used alone as a cell decorator in a Sage worksheet - with once=False (the default), rendering is done entirely client side, - which is much faster, not requiring a round-trip to the server. - """ - if s is None: - return lambda s: salvus.javascript(s, once=once, coffeescript=True) - else: - return salvus.javascript(s, coffeescript=True, once=once) - - -def javascript(s=None, once=False): - """ - Execute code using JavaScript. - - For example: - - %javascript console.log('hi') - - or - - javascript("console.log('hi')") - - - You may either pass in a string or use this as a cell decorator, - i.e., put %javascript at the top of a cell. - - If once=False (the default), the code will be executed every time the output of the - cell is rendered, e.g., on load, like with %auto:: - - javascript('.. some code ', once=False) - - or - - %javascript(once=False) - ... some code - - WARNING: If once=True, then this code is likely to get executed *before* the rest - of the output for this cell has been rendered by the client. - - javascript('console.log("HI")', once=False) - - EXTRA FUNCTIONALITY: - - When executing code, a function called print is defined, and objects cell and worksheet.:: - - print(1,2,'foo','bar') -- displays the inputs in the output cell - - cell -- has attributes cell.output (the html output box) and cell.cell_id - - worksheet -- has attributes project_page and editor, and methods interrupt, kill, and - - execute_code: (opts) => - opts = defaults opts, - code : required - data : undefined - preparse : true - cb : undefined - - This example illustrates using worksheet.execute_code:: - - %coffeescript - for i in [500..505] - worksheet.execute_code - code : "i=salvus.data['i']; i, factor(i)" - data : {i:i} - cb : (mesg) -> - if mesg.stdout then print(mesg.stdout) - if mesg.stderr then print(mesg.stderr) - - OPTIMIZATION: When used alone as a cell decorator in a Sage worksheet - with once=False (the default), rendering is done entirely client side, - which is much faster, not requiring a round-trip to the server. - """ - if s is None: - return lambda s: salvus.javascript(s, once=once) - else: - return salvus.javascript(s, once=once) - - -javascript_exec_doc = r""" - -To send code from Javascript back to the Python process to -be executed use the worksheet.execute_code function:: - - %javascript worksheet.execute_code(string_to_execute) - -You may also use a more general call format of the form:: - - %javascript - worksheet.execute_code({code:string_to_execute, data:jsonable_object, - preparse:true or false, cb:function}); - -The data object is available when the string_to_execute is being -evaluated as salvus.data. For example, if you execute this code -in a cell:: - - javascript(''' - worksheet.execute_code({code:"a = salvus.data['b']/2; print(a)", data:{b:5}, - preparse:false, cb:function(mesg) { console.log(mesg)} }); - ''') - -then the Python variable a is set to 2, and the Javascript console log will display:: - - Object {done: false, event: "output", id: "..."} - Object {stdout: "2\n", done: true, event: "output", id: "..."} - -You can also send an interrupt signal to the Python process from -Javascript by calling worksheet.interrupt(), and kill the process -with worksheet.kill(). For example, here the a=4 never -happens (but a=2 does):: - - %javascript - worksheet.execute_code({code:'a=2; sleep(100); a=4;', - cb:function(mesg) { worksheet.interrupt(); console.log(mesg)}}) - -or using CoffeeScript (a Javascript preparser):: - - %coffeescript - worksheet.execute_code - code : 'a=2; sleep(100); a=4;' - cb : (mesg) -> - worksheet.interrupt() - console.log(mesg) - -The Javascript code is evaluated with numerous standard Javascript libraries available, -including jQuery, Twitter Bootstrap, jQueryUI, etc. - -""" - -for s in [coffeescript, javascript]: - s.__doc__ += javascript_exec_doc - - -def latex0(s=None, **kwds): - """ - Create and display an arbitrary LaTeX document as a png image in the Salvus Notebook. - - In addition to directly calling latex.eval, you may put %latex (or %latex.eval(density=75, ...etc...)) - at the top of a cell, which will typeset everything else in the cell. - """ - if s is None: - return lambda t: latex0(t, **kwds) - if 'filename' not in kwds: - import tempfile - delete_file = True - kwds['filename'] = tempfile.mkstemp(suffix=".png")[1] - else: - delete_file = False - if 'locals' not in kwds: - kwds['locals'] = salvus.namespace - if 'globals' not in kwds: - kwds['globals'] = salvus.namespace - sage.misc.latex.latex.add_package_to_preamble_if_available('soul') - sage.misc.latex.Latex.eval(sage.misc.latex.latex, s, **kwds) - salvus.file(kwds['filename'], once=False) - if delete_file: - os.unlink(kwds['filename']) - return '' - - -latex0.__doc__ += sage.misc.latex.Latex.eval.__doc__ - - -class Time: - """ - Time execution of code exactly once in Salvus by: - - - putting %time at the top of a cell to time execution of the entire cell - - put %time at the beginning of line to time execution of just that line - - write time('some code') to executation of the contents of the string. - - If you want to time repeated execution of code for benchmarking purposes, use - the timeit command instead. - """ - def __init__(self, start=False): - if start: - from sage.all import walltime, cputime - self._start_walltime = walltime() - self._start_cputime = cputime() - - def before(self, code): - return Time(start=True) - - def after(self, code): - from sage.all import walltime, cputime - print(("\nCPU time: %.2f s, Wall time: %.2f s" % - (cputime(self._start_cputime), walltime(self._start_walltime)))) - self._start_cputime = self._start_walltime = None - - def __call__(self, code): - from sage.all import walltime, cputime - not_as_decorator = self._start_cputime is None - if not_as_decorator: - self.before(code) - salvus.execute(code) - if not_as_decorator: - self.after(code) - - -time = Time() - - -def file(path): - """ - Block decorator to write to a file. Use as follows: - - %file('filename') put this line in the file - - or - - %file('filename') - everything in the rest of the - cell goes into the file with given name. - - - As with all block decorators in Salvus, the arguments to file can - be arbitrary expressions. For examples, - - a = 'file'; b = ['name', 'txt'] - - %file(a+b[0]+'.'+b[1]) rest of line goes in 'filename.txt' - """ - return lambda content: open(path, 'w').write(content) - - -def timeit(*args, **kwds): - """ - Time execution of a command or block of commands. - - This command has been enhanced for Salvus so you may use it as - a block decorator as well, e.g., - - %timeit 2+3 - - and - - %timeit(number=10, preparse=False) 2^3 - - %timeit(number=10, seconds=True) 2^3 - - and - - %timeit(preparse=False) - - [rest of the cell] - - Here is the original docstring for timeit: - - """ - def go(code): - print((sage.misc.sage_timeit.sage_timeit(code, - globals_dict=salvus.namespace, - **kwds))) - - if len(args) == 0: - return lambda code: go(code) - else: - go(*args) - - -# TODO: these need to also give the argspec -timeit.__doc__ += sage.misc.sage_timeit.sage_timeit.__doc__ - - -class Capture: - """ - Capture or ignore the output from evaluating the given code. (SALVUS only). - - Use capture as a block decorator by placing either %capture or - %capture(optional args) at the beginning of a cell or at the - beginning of a line. If you use just plain %capture then stdout - and stderr are completely ignored. If you use %capture(args) - you can redirect or echo stdout and stderr to variables or - files. For example if you start a cell with this line:: - - %capture(stdout='output', stderr=open('error','w'), append=True, echo=True) - - then stdout is appended (because append=True) to the global - variable output, stderr is written to the file 'error', and the - output is still displayed in the output portion of the cell (echo=True). - - INPUT: - - - stdout -- string (or object with write method) to send stdout output to (string=name of variable) - - stderr -- string (or object with write method) to send stderr output to (string=name of variable) - - append -- (default: False) if stdout/stderr are a string, append to corresponding variable - - echo -- (default: False) if True, also echo stdout/stderr to the output cell. - """ - def __init__(self, stdout, stderr, append, echo): - self.v = (stdout, stderr, append, echo) - - def before(self, code): - (stdout, stderr, append, echo) = self.v - self._orig_stdout_f = orig_stdout_f = sys.stdout._f - if stdout is not None: - if hasattr(stdout, 'write'): - - def write_stdout(buf): - stdout.write(buf) - elif is_string(stdout): - if (stdout not in salvus.namespace) or not append: - salvus.namespace[stdout] = '' - if not is_string(salvus.namespace[stdout]): - salvus.namespace[stdout] = str(salvus.namespace[stdout]) - - def write_stdout(buf): - salvus.namespace[stdout] += buf - else: - raise TypeError( - "stdout must be None, a string, or have a write method") - - def f(buf, done): - write_stdout(buf) - if echo: - orig_stdout_f(buf, done) - elif done: - orig_stdout_f('', done) - - sys.stdout._f = f - elif not echo: - - def f(buf, done): - if done: - orig_stdout_f('', done) - - sys.stdout._f = f - - self._orig_stderr_f = orig_stderr_f = sys.stderr._f - if stderr is not None: - if hasattr(stderr, 'write'): - - def write_stderr(buf): - stderr.write(buf) - elif is_string(stderr): - if (stderr not in salvus.namespace) or not append: - salvus.namespace[stderr] = '' - if not is_string(salvus.namespace[stderr]): - salvus.namespace[stderr] = str(salvus.namespace[stderr]) - - def write_stderr(buf): - salvus.namespace[stderr] += buf - else: - raise TypeError( - "stderr must be None, a string, or have a write method") - - def f(buf, done): - write_stderr(buf) - if echo: - orig_stderr_f(buf, done) - elif done: - orig_stderr_f('', done) - - sys.stderr._f = f - elif not echo: - - def f(buf, done): - if done: - orig_stderr_f('', done) - - sys.stderr._f = f - - return self - - def __call__(self, - code=None, - stdout=None, - stderr=None, - append=False, - echo=False): - if code is None: - return Capture(stdout=stdout, - stderr=stderr, - append=append, - echo=echo) - if salvus._prefix: - if not code.startswith("%"): - code = salvus._prefix + '\n' + code - salvus.execute(code) - - def after(self, code): - sys.stdout._f = self._orig_stdout_f - sys.stderr._f = self._orig_stderr_f - - -capture = Capture(stdout=None, stderr=None, append=False, echo=False) - -import sage.misc.cython - - -def asy(code=None, **kwds): - # make a .pdf from .asy code and display it - # asy command can also be used to make .eps file - import tempfile - import subprocess - fname1 = tempfile.mkstemp(suffix=".asy")[1] - fname2 = tempfile.mkstemp(suffix=".png")[1] - with open(fname1, "w") as outf1: - outf1.write(code + '\n') - cmd = "/usr/bin/asy -offscreen -f png -o {} {}".format(fname2, fname1) - subprocess.call(cmd.split()) - salvus.file(fname2) - os.unlink(fname1) - os.unlink(fname2) - print('') - - -def cython(code=None, **kwds): - """ - Block decorator to easily include Cython code in CoCalc worksheets. - - Put %cython at the top of a cell, and the rest of that cell is compiled as - Cython code and made directly available to use in other cells. - - You can pass options to cython by typing "%cython(... var=value...)" - instead of just "%cython". - - If you give the option silent=True (not the default) then this won't - print what functions get globally defined as a result of evaluating code. - - This is a wrapper around Sage's own cython function, whose - docstring is below: - - ORIGINAL DOCSTRING: - - """ - if code is None: - return lambda code: cython(code, **kwds) - from sage.misc.temporary_file import tmp_dir - path = tmp_dir() - filename = os.path.join(path, 'a.pyx') - open(filename, 'w').write(code) - - silent = kwds.get('silent', False) - if 'silent' in kwds: - del kwds['silent'] - - if 'annotate' not in kwds and not silent: - kwds['annotate'] = True - - modname, path = sage.misc.cython.cython(filename, **kwds) - - try: - sys.path.insert(0, path) - module = __import__(modname) - finally: - del sys.path[0] - - defined = [] - for name, value in inspect.getmembers(module): - if not name.startswith('_') and name != 'init_memory_functions': - salvus.namespace[name] = value - defined.append(name) - if not silent: - if defined: - print(("Defined %s" % (', '.join(defined)))) - else: - print("No functions defined.") - - files = os.listdir(path) - html_filename = None - for n in files: - base, ext = os.path.splitext(n) - if ext.startswith('.html') and '_pyx_' in base: - html_filename = os.path.join(path, n) - if html_filename is not None: - salvus.file(html_filename, - raw=True, - show=True, - text="Auto-generated code...") - - -cython.__doc__ += sage.misc.cython.cython.__doc__ - - -class script: - r""" - Block decorator to run an arbitrary shell command with input from a - cell in Salvus. - - Put %script('shell command line') or %script(['command', 'arg1', - 'arg2', ...]) by itself on a line in a cell, and the command line - is run with stdin the rest of the contents of the cell. You can - also use script in single line mode, e.g.,:: - - %script('gp -q') factor(2^97 - 1) - - or - - %script(['gp', '-q']) factor(2^97 - 1) - - will launch a gp session, feed 'factor(2^97-1)' into stdin, and - display the resulting factorization. - - NOTE: the result is stored in the attribute "stdout", so you can do:: - - s = script('gp -q') - %s factor(2^97-1) - s.stdout - '\n[11447 1]\n\n[13842607235828485645766393 1]\n\n' - - and s.stdout will now be the output string. - - You may also specify the shell environment with the env keyword. - """ - def __init__(self, args, env=None): - self._args = args - self._env = env - - def __call__(self, code=''): - import subprocess - try: - s = None - s = subprocess.Popen(self._args, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - shell=is_string(self._args), - env=self._env) - s.stdin.write(code) - s.stdin.close() - finally: - if s is None: - return - try: - self.stdout = s.stdout.read() - sys.stdout.write(self.stdout) - finally: - try: - os.system("pkill -TERM -P %s" % s.pid) - except OSError: - pass - try: - os.kill(s.pid, 9) - except OSError: - pass - - -def python(code): - """ - Block decorator to run code in pure Python mode, without it being - preparsed by the Sage preparser. Otherwise, nothing changes. - - To use this, put %python by itself in a cell so that it applies to - the rest of the cell, or put it at the beginning of a line to - disable preparsing just for that line. - """ - salvus.execute(code, preparse=False) - - -def python3(code=None, **kwargs): - """ - Block decorator to run code in a pure Python3 mode session. - - To use this, put %python3 by itself in a cell so that it applies to - the rest of the cell, or put it at the beginning of a line to - run just that line using python3. - - You can combine %python3 with capture, if you would like to capture - the output to a variable. For example:: - - %capture(stdout='p3') - %python3 - x = set([1,2,3]) - print(x) - - Afterwards, p3 contains the output '{1, 2, 3}' and the variable x - in the controlling Sage session is in no way impacted. - - .. note:: - - State is preserved between cells. - CoCalc %python3 mode uses the jupyter `python3` kernel. - """ - if python3.jupyter_kernel is None: - python3.jupyter_kernel = jupyter("python3") - return python3.jupyter_kernel(code, **kwargs) - - -python3.jupyter_kernel = None - - -def anaconda(code=None, **kwargs): - """ - Block decorator to run code in a pure anaconda mode session. - - To use this, put %anaconda by itself in a cell so that it applies to - the rest of the cell, or put it at the beginning of a line to - run just that line using anaconda. - - You can combine %anaconda with capture, if you would like to capture - the output to a variable. For example:: - - %capture(stdout='a') - %anaconda - x = set([1,2,3]) - print(x) - - Afterwards, a contains the output '{1, 2, 3}' and the variable x - in the controlling Sage session is in no way impacted. - - .. note:: - - State is preserved between cells. - CoCalc %anaconda mode uses the jupyter `anaconda5` kernel. - """ - if anaconda.jupyter_kernel is None: - anaconda.jupyter_kernel = jupyter("anaconda5") - return anaconda.jupyter_kernel(code, **kwargs) - - -anaconda.jupyter_kernel = None - - -def singular_kernel(code=None, **kwargs): - """ - Block decorator to run code in a Singular mode session. - - To use this, put %singular_kernel by itself in a cell so that it applies to - the rest of the cell, or put it at the beginning of a line to - run just that line using singular_kernel. - - State is preserved between cells. - - This is completely different than the singular command in Sage itself, which - supports things like x = singular(sage_object), and *also* provides a way - to execute code by beginning cells with %singular. The singular interface in - Sage uses pexpect, so might be less robust than singular_kernel. - - .. note:: - - SMC %singular_kernel mode uses the jupyter `singular` kernel: - https://github.com/sebasguts/jupyter_kernel_singular - """ - if singular_kernel.jupyter_kernel is None: - singular_kernel.jupyter_kernel = jupyter("singular") - return singular_kernel.jupyter_kernel(code, **kwargs) - - -singular_kernel.jupyter_kernel = None - - -def perl(code): - """ - Block decorator to run code in a Perl session. - - To use this, put %perl by itself in a cell so that it applies to - the rest of the cell, or put it at the beginning of a line to - run just that line using perl. - - EXAMPLE: - - A perl cell:: - - %perl - $apple_count = 5; - $count_report = "There are $apple_count apples."; - print "The report is: $count_report\n"; - - Or use %perl on one line:: - - %perl $apple_count = 5; $count_report = "There are $apple_count apples."; print "The report is: $count_report\n"; - - You can combine %perl with capture, if you would like to capture - the output to a variable. For example:: - - %capture(stdout='p') - %perl print "hi" - - Afterwards, p contains 'hi'. - - NOTE: No state is preserved between calls. Each call is a separate process. - """ - script('sage-native-execute perl')(code) - - -def ruby(code): - """ - Block decorator to run code in a Ruby session. - - To use this, put %ruby by itself in a cell so that it applies to - the rest of the cell, or put it at the beginning of a line to - run just that line using ruby. - - EXAMPLE: - - A ruby cell:: - - %ruby - lang = "ruby" - print "Hello from #{lang}!" - - Or use %ruby on one line:: - - %ruby lang = "ruby"; print "Hello from #{lang}!" - - You can combine %ruby with capture, if you would like to capture - the output to a variable. For example:: - - %capture(stdout='p') - %ruby lang = "ruby"; print "Hello from #{lang}!" - - Afterwards, p contains 'Hello from ruby!'. - - NOTE: No state is preserved between calls. Each call is a separate process. - """ - script('sage-native-execute ruby')(code) - - -def fortran(x, library_paths=[], libraries=[], verbose=False): - """ - Compile Fortran code and make it available to use. - - INPUT: - - - x -- a string containing code - - Use this as a decorator. For example, put this in a cell and evaluate it:: - - %fortran - - C FILE: FIB1.F - SUBROUTINE FIB(A,N) - C - C CALCULATE FIRST N FIBONACCI NUMBERS - C - INTEGER N - REAL*8 A(N) - DO I=1,N - IF (I.EQ.1) THEN - A(I) = 0.0D0 - ELSEIF (I.EQ.2) THEN - A(I) = 1.0D0 - ELSE - A(I) = A(I-1) + A(I-2) - ENDIF - ENDDO - END - C END FILE FIB1.F - - - In the next cell, evaluate this:: - - import numpy - n = numpy.array(range(10),dtype=float) - fib(n,int(10)) - n - - This will produce this output: array([ 0., 1., 1., 2., 3., 5., 8., 13., 21., 34.]) - """ - import builtins - from sage.misc.temporary_file import tmp_dir - if len(x.splitlines()) == 1 and os.path.exists(x): - filename = x - x = open(x).read() - if filename.lower().endswith('.f90'): - x = '!f90\n' + x - - from numpy import f2py - from random import randint - - # Create everything in a temporary directory - mytmpdir = tmp_dir() - - try: - old_cwd = os.getcwd() - os.chdir(mytmpdir) - - old_import_path = os.sys.path - os.sys.path.append(mytmpdir) - - name = "fortran_module_%s" % randint(0, 2**64) # Python module name - # if the first line has !f90 as a comment, gfortran will - # treat it as Fortran 90 code - if x.startswith('!f90'): - fortran_file = name + '.f90' - else: - fortran_file = name + '.f' - - s_lib_path = "" - s_lib = "" - for s in library_paths: - s_lib_path = s_lib_path + "-L%s " % s - - for s in libraries: - s_lib = s_lib + "-l%s " % s - - log = name + ".log" - extra_args = '--quiet --f77exec=sage-inline-fortran --f90exec=sage-inline-fortran %s %s >"%s" 2>&1' % ( - s_lib_path, s_lib, log) - - f2py.compile(x, name, extra_args=extra_args, source_fn=fortran_file) - log_string = open(log).read() - - # f2py.compile() doesn't raise any exception if it fails. - # So we manually check whether the compiled file exists. - # NOTE: the .so extension is used expect on Cygwin, - # that is even on OS X where .dylib might be expected. - soname = name - uname = os.uname()[0].lower() - if uname[:6] == "cygwin": - soname += '.dll' - else: - soname += '.so' - if not os.path.isfile(soname): - raise RuntimeError("failed to compile Fortran code:\n" + - log_string) - - if verbose: - print(log_string) - - m = builtins.__import__(name) - - finally: - os.sys.path = old_import_path - os.chdir(old_cwd) - try: - import shutil - shutil.rmtree(mytmpdir) - except OSError: - # This can fail for example over NFS - pass - - for k, x in m.__dict__.items(): - if k[0] != '_': - salvus.namespace[k] = x - - -def sh(code=None, **kwargs): - """ - Run a bash script in Salvus. Uses jupyter bash kernel - which allows keeping state between cells. - - EXAMPLES: - - Use as a block decorator on a single line:: - - %sh pwd - - and multiline - - %sh - echo "hi" - pwd - ls -l - - You can also just directly call it:: - - sh('pwd') - - The output is printed. To capture it, use capture - - %capture(stdout='output') - %sh pwd - - After that, the variable output contains the current directory - - Remember shell state between cells - - %sh - FOO='xyz' - cd /tmp - ... new cell will show settings from previous cell ... - %sh - echo $FOO - pwd - - Display image file (this is a feature of jupyter bash kernel) - - %sh - display < sage_logo.png - - .. WARNING:: - - The jupyter bash kernel does not separate stdout and stderr as cell is running. - It only returns ok or error depending on exit status of last command in the cell. - So all cell output captured goes to either stdout or stderr variable, depending - on exit status of the last command in the %sh cell. - """ - if sh.jupyter_kernel is None: - sh.jupyter_kernel = jupyter("bash") - sh.jupyter_kernel( - 'function command_not_found_handle { printf "%s: command not found\n" "$1" >&2; return 127;}' - ) - return sh.jupyter_kernel(code, **kwargs) - - -sh.jupyter_kernel = None - - -# use jupyter kernel for GNU octave instead of sage interpreter interface -def octave(code=None, **kwargs): - r""" - Run GNU Octave code in a sage worksheet. - - INPUT: - - - ``code`` -- a string containing code - - Use as a decorator. For example, put this in a cell and evaluate it:: - - %octave - x = -10:0.1:10; - plot (x, sin (x)) - - .. note:: - - SMC %octave mode uses the jupyter `octave` kernel. - """ - if octave.jupyter_kernel is None: - octave.jupyter_kernel = jupyter("octave") - octave.jupyter_kernel.smc_image_scaling = 1 - return octave.jupyter_kernel(code, **kwargs) - - -octave.jupyter_kernel = None - - -# jupyter kernel for %ir mode -def r(code=None, **kwargs): - r""" - Run R code in a sage worksheet. - - INPUT: - - - ``code`` -- a string containing code - - Use as a decorator. For example, put this in a cell and evaluate it to see a scatter plot - of built-in mtcars dataframe variables `mpg` vs `wt`:: - - %r - with(mtcars,plot(wt,mpg)) - - .. note:: - - SMC %r mode uses the jupyter `ir` kernel. - """ - if r.jupyter_kernel is None: - r.jupyter_kernel = jupyter("ir") - r.jupyter_kernel('options(repr.plot.res = 240)') - r.jupyter_kernel.smc_image_scaling = .5 - return r.jupyter_kernel(code, **kwargs) - - -r.jupyter_kernel = None - - -# jupyter kernel for %scala mode -def scala211(code=None, **kwargs): - r""" - Run scala code in a sage worksheet. - - INPUT: - - - ``code`` -- a string containing code - - Use as a decorator. - - .. note:: - - SMC %scala211 mode uses the jupyter `scala211` kernel. - """ - if scala211.jupyter_kernel is None: - scala211.jupyter_kernel = jupyter("scala211") - return scala211.jupyter_kernel(code, **kwargs) - - -scala211.jupyter_kernel = None -# add alias for generic scala -scala = scala211 - - -def prun(code): - """ - Use %prun followed by a block of code to profile execution of that - code. This will display the resulting profile, along with a menu - to select how to sort the data. - - EXAMPLES: - - Profile computing a tricky integral (on a single line):: - - %prun integrate(sin(x^2),x) - - Profile a block of code:: - - %prun - E = EllipticCurve([1..5]) - v = E.anlist(10^5) - r = E.rank() - """ - import cProfile, pstats - from sage.misc.all import tmp_filename - - filename = tmp_filename() - cProfile.runctx(salvus.namespace['preparse'](code), salvus.namespace, - locals(), filename) - - @interact - def f(title=text_control('', "

CoCalc Profiler

"), - sort=( - "First sort by", - selector([ - ('calls', 'number of calls to the function'), - ('time', ' total time spent in the function'), - ('cumulative', - 'total time spent in this and all subfunctions (from invocation till exit)' - ), - ('module', 'name of the module that contains the function'), - ('name', 'name of the function') - ], - width="100%", - default='time')), - strip_dirs=True): - try: - p = pstats.Stats(filename) - if strip_dirs: - p.strip_dirs() - p.sort_stats(sort) - p.print_stats() - except Exception as msg: - print(msg) - - -############################################################## -# The %fork cell decorator. -############################################################## - - -def _wait_in_thread(pid, callback, filename): - from sage.structure.sage_object import load - - def wait(): - try: - os.waitpid(pid, 0) - callback(load(filename)) - except Exception as msg: - callback(msg) - - from threading import Thread - t = Thread(target=wait, args=tuple([])) - t.start() - - -def async_(f, args, kwds, callback): - """ - Run f in a forked subprocess with given args and kwds, then call the - callback function when f terminates. - """ - from sage.misc.all import tmp_filename - filename = tmp_filename() + '.sobj' - sys.stdout.flush() - sys.stderr.flush() - pid = os.fork() - if pid: - # The parent master process - try: - _wait_in_thread(pid, callback, filename) - return pid - finally: - if os.path.exists(filename): - os.unlink(filename) - else: - # The child process - try: - result = f(*args, **kwds) - except Exception as msg: - result = str(msg) - from sage.structure.sage_object import save - save(result, filename) - os._exit(0) - - -class Fork(object): - """ - The %fork block decorator evaluates its code in a forked subprocess - that does not block the main process. - - You may still use the @fork function decorator from Sage, as usual, - to run a function in a subprocess. Type "sage.all.fork?" to see - the help for the @fork decorator. - - WARNING: This is highly experimental and possibly flaky. Use with - caution. - - All (picklelable) global variables that are set in the forked - subprocess are set in the parent when the forked subprocess - terminates. However, the forked subprocess has no other side - effects, except what it might do to file handles and the - filesystem. - - To see currently running forked subprocesses, type - fork.children(), which returns a dictionary {pid:execute_uuid}. - To kill a given subprocess and stop the cell waiting for input, - type fork.kill(pid). This is currently the only way to stop code - running in %fork cells. - - TODO/WARNING: The subprocesses spawned by fork are not killed - if the parent process is killed first! - - NOTE: All pexpect interfaces are reset in the child process. - """ - def __init__(self): - self._children = {} - - def children(self): - return dict(self._children) - - def __call__(self, s): - - if isinstance(s, types.FunctionType): # check for decorator usage - import sage.parallel.decorate - return sage.parallel.decorate.fork(s) - - salvus._done = False - - id = salvus._id - - changed_vars = set([]) - - def change(var, val): - changed_vars.add(var) - - def f(): - # Run some commands to tell Sage that its - # pid has changed. - import sage.misc.misc - - # since python 3.12, there is no imp - try: - import imp - except: - import importlib as imp - imp.reload(sage.misc.misc) - - # The pexpect interfaces (and objects defined in them) are - # not valid. - sage.interfaces.quit.invalidate_all() - - salvus.namespace.on('change', None, change) - salvus.execute(s) - result = {} - from sage.structure.sage_object import dumps - for var in changed_vars: - try: - result[var] = dumps(salvus.namespace[var]) - except: - result[var] = 'unable to pickle %s' % var - return result - - from sage.structure.sage_object import loads - - def g(s): - if isinstance(s, Exception): - sys.stderr.write(str(s)) - sys.stderr.flush() - else: - for var, val in s.items(): - try: - salvus.namespace[var] = loads(val) - except: - print(("unable to unpickle %s" % var)) - salvus._conn.send_json({'event': 'output', 'id': id, 'done': True}) - if pid in self._children: - del self._children[pid] - - pid = async_(f, tuple([]), {}, g) - print(("Forked subprocess %s" % pid)) - self._children[pid] = id - - def kill(self, pid): - if pid in self._children: - salvus._conn.send_json({ - 'event': 'output', - 'id': self._children[pid], - 'done': True - }) - os.kill(pid, 9) - del self._children[pid] - else: - raise ValueError("Unknown pid = (%s)" % pid) - - -fork = Fork() - -#################################################### -# Display of 2d/3d graphics objects -#################################################### - -from sage.misc.all import tmp_filename -from sage.plot.animate import Animation -import matplotlib.figure - - -def show_animation(obj, delay=20, gif=False, **kwds): - if gif: - t = tmp_filename(ext='.gif') - obj.gif(delay, t, **kwds) - salvus.file(t, raw=False) - os.unlink(t) - else: - t = tmp_filename(ext='.webm') - obj.ffmpeg(t, delay=delay, **kwds) - # and let delete when worksheet ends - need this so can replay video. - salvus.file(t, raw=True) - - -def show_2d_plot_using_matplotlib(obj, svg, **kwds): - if isinstance(obj, matplotlib.image.AxesImage): - # The result of imshow, e.g., - # - # from matplotlib import numpy, pyplot - # pyplot.imshow(numpy.random.random_integers(255, size=(100,100,3))) - # - t = tmp_filename(ext='.png') - obj.write_png(t) - salvus.file(t) - os.unlink(t) - return - - if isinstance(obj, matplotlib.axes.Axes): - obj = obj.get_figure() - - if 'events' in kwds: - from smc_sagews.graphics import InteractiveGraphics - ig = InteractiveGraphics(obj, **kwds['events']) - n = '__a' + uuid().replace( - '-', '') # so it doesn't get garbage collected instantly. - obj.__setattr__(n, ig) - kwds2 = dict(kwds) - del kwds2['events'] - ig.show(**kwds2) - else: - t = tmp_filename(ext='.svg' if svg else '.png') - if isinstance(obj, matplotlib.figure.Figure): - obj.savefig(t, **kwds) - else: - obj.save(t, **kwds) - salvus.file(t) - os.unlink(t) - - -def show_3d_plot_using_tachyon(obj, **kwds): - t = tmp_filename(ext='.png') - obj.save(t, **kwds) - salvus.file(t) - os.unlink(t) - - -def show_graph_using_d3(obj, **kwds): - salvus.d3_graph(obj, **kwds) - - -def plot3d_using_matplotlib(expr, - rangeX, - rangeY, - density=40, - elev=45., - azim=35., - alpha=0.85, - cmap=None): - """ - Plots a symbolic expression in two variables on a two dimensional grid - and renders the function using matplotlib's 3D projection. - The purpose is to make it possible to create vectorized images (PDF, SVG) - for high-resolution images in publications -- instead of rasterized image formats. - - Example:: - %var x y - plot3d_using_matplotlib(x^2 + (1-y^2), (x, -5, 5), (y, -5, 5)) - - Arguments:: - - * expr: symbolic expression, e.g. x^2 - (1-y)^2 - * rangeX: triple: (variable, minimum, maximum), e.g. (x, -10, 10) - * rangeY: like rangeX - * density: grid density - * elev: elevation, e.g. 45 - * azim: azimuth, e.g. 35 - * alpha: alpha transparency of plot (default: 0.85) - * cmap: matplotlib colormap, e.g. matplotlib.cm.Blues (default) - """ - from matplotlib import cm - import matplotlib.pyplot as plt - from mpl_toolkits.mplot3d import axes3d - import numpy as np - - cmap = cmap or cm.Blues - - plt.cla() - fig = plt.figure() - ax = fig.gca(projection='3d') - ax.view_init(elev=elev, azim=azim) - - xx = np.linspace(rangeX[1], rangeX[2], density) - yy = np.linspace(rangeY[1], rangeY[2], density) - X, Y = np.meshgrid(xx, yy) - - import numpy as np - exprv = np.vectorize(lambda x1, x2 : \ - float(expr.subs({rangeX[0] : x1, rangeY[0] : x2}))) - Z = exprv(X, Y) - zlim = np.min(Z), np.max(Z) - - ax.plot_surface(X, - Y, - Z, - alpha=alpha, - cmap=cmap, - linewidth=.5, - shade=True, - rstride=int(len(xx) / 10), - cstride=int(len(yy) / 10)) - - ax.set_xlabel('X') - ax.set_xlim(*rangeX[1:]) - ax.set_ylabel('Y') - ax.set_ylim(*rangeY[1:]) - ax.set_zlabel('Z') - ax.set_zlim(*zlim) - - plt.show() - - -# Sage version 8.9 introduced -# https://doc.sagemath.org/html/en/reference/plotting/sage/plot/multigraphics.html#sage.plot.multigraphics.MultiGraphics -# which complicates the logic below. -try: - # Try to import both GraphicsArray and MultiGraphics - from sage.plot.multigraphics import GraphicsArray, MultiGraphics -except: - # Import failed, so probably 8.9 -- we try to import GraphicsArray. - # If this also fails, then Sage has changed a lot and some manual work is needed. - from sage.plot.graphics import GraphicsArray - # Also ensure MultiGraphics is defined but None. We'll have to - # check for None in the places where MultiGraphics is used below. - MultiGraphics = None - -from sage.plot.graphics import Graphics -from sage.plot.plot3d.base import Graphics3d -from sage.plot.plot3d.tachyon import Tachyon - -# used in show function -GRAPHICS_MODULES_SHOW = [ - Graphics, - GraphicsArray, - matplotlib.figure.Figure, - matplotlib.axes.Axes, - matplotlib.image.AxesImage, -] - -if MultiGraphics is not None: - GRAPHICS_MODULES_SHOW.append(MultiGraphics) - -GRAPHICS_MODULES_SHOW = tuple(GRAPHICS_MODULES_SHOW) - - -def show(*objs, **kwds): - """ - Show a 2d or 3d graphics object (or objects), animation, or matplotlib figure, or show an - expression typeset nicely using LaTeX. - - - display: (default: True); if True, use display math for expression (big and centered). - - - svg: (default: True); if True, show 2d plots using svg (otherwise use png) - - - d3: (default: True); if True, show graphs (vertices and edges) using an interactive D3 viewer - for the many options for this viewer, type - - import smc_sagews.graphics - smc_sagews.graphics.graph_to_d3_jsonable? - - If false, graphs are converted to plots and displayed as usual. - - - renderer: (default: 'webgl'); for 3d graphics - - 'webgl' (fastest) using hardware accelerated 3d; - - 'canvas' (slower) using a 2d canvas, but may work better with transparency; - - 'tachyon' -- a ray traced static image. - - - spin: (default: False); spins 3d plot, with number determining speed (requires mouse over plot) - - - events: if given, {'click':foo, 'mousemove':bar}; each time the user clicks, - the function foo is called with a 2-tuple (x,y) where they clicked. Similarly - for mousemove. This works for Sage 2d graphics and matplotlib figures. - - - viewer: optional string, set to "tachyon" for static ray-tracing view of 3d image - - - background: string (default: 'transparent'), specifies background color for 3d images. - Ignored if viewer is set to 'tachyon' or if object type is Tachyon. - May be 'transparent' or any valid CSS color string, e.g.: 'red', '#00ff00', 'rgb(0,0,255)'. - - - foreground: string, specifies frame color for 3d images. Defaults to 'gray' when - background is 'transparent', otherwise default is computed for visibility based on canvas - background. - - ANIMATIONS: - - - animations are by default encoded and displayed using an efficiently web-friendly - format (currently webm, which is **not supported** by Safari or IE). - - - ``delay`` - integer (default: 20); delay in hundredths of a - second between frames. - - - gif=False -- if you set gif=True, instead use an animated gif, - which is much less efficient, but works on all browsers. - - You can also use options directly to the animate command, e.g., the figsize option below: - - a = animate([plot(sin(x + a), (x, 0, 2*pi)) for a in [0, pi/4, .., 2*pi]], figsize=6) - show(a, delay=30) - - - EXAMPLES: - - Some examples: - - show(2/3) - show([1, 4/5, pi^2 + e], 1+pi) - show(x^2, display=False) - show(e, plot(sin)) - - Here's an example that illustrates creating a clickable image with events:: - - @interact - def f0(fun=x*sin(x^2), mousemove='', click='(0,0)'): - click = sage_eval(click) - g = plot(fun, (x,0,5), zorder=0) + point(click, color='red', pointsize=100, zorder=10) - ymax = g.ymax(); ymin = g.ymin() - m = fun.derivative(x)(x=click[0]) - b = fun(x=click[0]) - m*click[0] - g += plot(m*x + b, (click[0]-1,click[0]+1), color='red', zorder=10) - def h(p): - f0.mousemove = p - def c(p): - f0(click=p) - show(g, events={'click':c, 'mousemove':h}, svg=True, gridlines='major', ymin=ymin, ymax=ymax) - """ - # svg=True, d3=True, - svg = kwds.get('svg', True) - d3 = kwds.get('d3', True) - display = kwds.get('display', True) - for t in ['svg', 'd3', 'display']: - if t in kwds: - del kwds[t] - from smc_sagews import graphics - - def show0(obj, combine_all=False): - # Either show the object and return None or - # return a string of html to represent obj. - if isinstance(obj, GRAPHICS_MODULES_SHOW): - show_2d_plot_using_matplotlib(obj, svg=svg, **kwds) - elif isinstance(obj, Animation): - show_animation(obj, **kwds) - elif isinstance(obj, Graphics3d): - - # _extra_kwds processing follows the example of - extra_kwds = {} if obj._extra_kwds is None else obj._extra_kwds - for k in [ - 'spin', - 'renderer', - 'viewer', - 'frame', - 'height', - 'width', - 'background', - 'foreground', - 'aspect_ratio', - ]: - if k in extra_kwds and k not in kwds: - kwds[k] = obj._extra_kwds[k] - - if kwds.get('viewer') == 'tachyon': - show_3d_plot_using_tachyon(obj, **kwds) - else: - if kwds.get('viewer') == 'threejs': - del kwds['viewer'] - if kwds.get('online'): - del kwds['online'] - salvus.threed(obj, **kwds) - elif isinstance(obj, Tachyon): - show_3d_plot_using_tachyon(obj, **kwds) - elif isinstance( - obj, (sage.graphs.graph.Graph, sage.graphs.digraph.DiGraph)): - if d3: - show_graph_using_d3(obj, **kwds) - else: - show(obj.plot(), **kwds) - elif is_string(obj): - return obj - elif isinstance(obj, (list, tuple)): - v = [] - for a in obj: - b = show0(a) - if b is not None: - v.append(b) - if combine_all: - return ' '.join(v) - s = ', '.join(v) - if isinstance(obj, list): - return '[%s]' % s - else: - return '(%s)' % s - elif is_dataframe(obj): - html(obj.to_html(), hide=False) - else: - __builtins__['_'] = obj - s = str(sage.misc.latex.latex(obj)) - if r'\text{\texttt' in s and 'tikzpicture' not in s: - # In this case the mathjax latex mess is so bad, it is better to just print and give up! - print(obj) - return - # Add anything here that Sage produces and mathjax can't handle, and - # which people complain about... (obviously, I wish there were a way to - # know -- e.g., if Sage had a way to tell whether latex it produces - # will work with mathjax or not). - if '\\begin{tikzpicture}' in s or '\\raisebox' in s: - # special case -- mathjax has no support for tikz or \raisebox so we just immediately display it (as a png); this is - # better than nothing. - sage.misc.latex.latex.eval(s) - return '' - elif r'\begin{tabular}' in s: - # tabular is an environment for text, not formular. - # Sage's `tabular` should actually use \array! - sage.misc.latex.latex.eval(s) - return '' - # default - elif display: - return "$\\displaystyle %s$" % s - else: - return "$%s$" % s - - sys.stdout.flush() - sys.stderr.flush() - s = show0(objs, combine_all=True) - - if six.PY3: - from html import escape - elif six.PY2: - # deprecated in py3 - from cgi import escape - - if s is not None: - if len(s) > 0: - if display: - salvus.html("
%s
" % escape(s)) - else: - salvus.html("
%s
" % escape(s)) - sys.stdout.flush() - sys.stderr.flush() - - -# Make it so plots plot themselves correctly when they call their repr. -Graphics.show = show -GraphicsArray.show = show -if MultiGraphics is not None: - MultiGraphics.show = show -Animation.show = show - -# Very "evil" abuse of the display manager, so sphere().show() works: -try: - from sage.repl.rich_output import get_display_manager - get_display_manager().display_immediately = show -except: - # so doesn't crash on older versions of Sage. - pass - - -################################################### -# %auto -- automatically evaluate a cell on load -################################################### -def auto(s): - """ - The %auto decorator sets a cell so that it will be automatically - executed when the Sage process first starts. Make it the first - line of a cell. - - Thus %auto allows you to initialize functions, variables, interacts, - etc., e.g., when loading a worksheet. - """ - return s # the do-nothing block decorator. - - -def hide(component='input'): - """ - Hide a component of a cell. By default, hide hides the the code - editor part of the cell, but you can hide other parts by passing - in an optional argument: - - 'input', 'output' - - Use the cell.show(...) function to reveal a cell component. - """ - if component not in ['input', 'output']: - # Allow %hide to work, for compatability with sagenb. - hide('input') - return component - cell.hide(component) - - -def hideall(code=None): - cell.hideall() - if code is not None: # for backwards compat with sagenb - return code - - -########################################################## -# A "%exercise" cell mode -- a first step toward -# automated homework. -########################################################## -class Exercise: - def __init__(self, question, answer, check=None, hints=None): - import sage.all - from sage.structure.element import is_Matrix - if not (isinstance(answer, (tuple, list)) and len(answer) == 2): - if is_Matrix(answer): - default = sage.all.parent(answer)(0) - else: - default = '' - answer = [answer, default] - - if check is None: - R = sage.all.parent(answer[0]) - - def check(attempt): - return R(attempt) == answer[0] - - if hints is None: - hints = ['', '', '', "The answer is %s." % answer[0]] - - self._question = question - self._answer = answer - self._check = check - self._hints = hints - - def _check_attempt(self, attempt, interact): - from sage.misc.all import walltime - response = "
" - try: - r = self._check(attempt) - if isinstance(r, tuple) and len(r) == 2: - correct = r[0] - comment = r[1] - else: - correct = bool(r) - comment = '' - except TypeError as msg: - response += "

Huh? -- %s (attempt=%s)

" % ( - msg, attempt) - else: - if correct: - response += "

RIGHT!

" - if self._start_time: - response += "

Time: %.1f seconds

" % ( - walltime() - self._start_time, ) - if self._number_of_attempts == 1: - response += "

You got it first try!

" - else: - response += "

It took you %s attempts.

" % ( - self._number_of_attempts, ) - else: - response += "

Not correct yet...

" - if self._number_of_attempts == 1: - response += "

(first attempt)

" - else: - response += "

(%s attempts)

" % self._number_of_attempts - - if self._number_of_attempts > len(self._hints): - hint = self._hints[-1] - else: - hint = self._hints[self._number_of_attempts - 1] - if hint: - response += "(HINT: %s)" % ( - hint, ) - if comment: - response += '

%s

' % comment - - response += "
" - - interact.feedback = text_control(response, label='') - - return correct - - def ask(self, cb): - from sage.misc.all import walltime - self._start_time = walltime() - self._number_of_attempts = 0 - attempts = [] - - @interact(layout=[[('question', 12)], [('attempt', 12)], - [('feedback', 12)]]) - def f(question=("Question:", text_control(self._question)), - attempt=('Answer:', self._answer[1])): - if 'attempt' in interact.changed() and attempt != '': - attempts.append(attempt) - if self._start_time == 0: - self._start_time = walltime() - self._number_of_attempts += 1 - if self._check_attempt(attempt, interact): - cb({ - 'attempts': attempts, - 'time': walltime() - self._start_time - }) - - -def exercise(code): - r""" - Use the %exercise cell decorator to create interactive exercise - sets. Put %exercise at the top of the cell, then write Sage code - in the cell that defines the following (all are optional): - - - a ``question`` variable, as an HTML string with math in dollar - signs - - - an ``answer`` variable, which can be any object, or a pair - (correct_value, interact control) -- see the docstring for - interact for controls. - - - an optional callable ``check(answer)`` that returns a boolean or - a 2-tuple - - (True or False, message), - - where the first argument is True if the answer is correct, and - the optional second argument is a message that should be - displayed in response to the given answer. NOTE: Often the - input "answer" will be a string, so you may have to use Integer, - RealNumber, or sage_eval to evaluate it, depending - on what you want to allow the user to do. - - - hints -- optional list of strings to display in sequence each - time the user enters a wrong answer. The last string is - displayed repeatedly. If hints is omitted, the correct answer - is displayed after three attempts. - - NOTE: The code that defines the exercise is executed so that it - does not impact (and is not impacted by) the global scope of your - variables elsewhere in your session. Thus you can have many - %exercise cells in a single worksheet with no interference between - them. - - The following examples further illustrate how %exercise works. - - An exercise to test your ability to sum the first $n$ integers:: - - %exercise - title = "Sum the first n integers, like Gauss did." - n = randint(3, 100) - question = "What is the sum $1 + 2 + \\cdots + %s$ of the first %s positive integers?"%(n,n) - answer = n*(n+1)//2 - - Transpose a matrix:: - - %exercise - title = r"Transpose a $2 \times 2$ Matrix" - A = random_matrix(ZZ,2) - question = "What is the transpose of $%s?$"%latex(A) - answer = A.transpose() - - Add together a few numbers:: - - %exercise - k = randint(2,5) - title = "Add %s numbers"%k - v = [randint(1,10) for _ in range(k)] - question = "What is the sum $%s$?"%(' + '.join([str(x) for x in v])) - answer = sum(v) - - The trace of a matrix:: - - %exercise - title = "Compute the trace of a matrix." - A = random_matrix(ZZ, 3, x=-5, y = 5)^2 - question = "What is the trace of $$%s?$$"%latex(A) - answer = A.trace() - - Some basic arithmetic with hints and dynamic feedback:: - - %exercise - k = randint(2,5) - title = "Add %s numbers"%k - v = [randint(1,10) for _ in range(k)] - question = "What is the sum $%s$?"%(' + '.join([str(x) for x in v])) - answer = sum(v) - hints = ['This is basic arithmetic.', 'The sum is near %s.'%(answer+randint(1,5)), "The answer is %s."%answer] - def check(attempt): - c = Integer(attempt) - answer - if c == 0: - return True - if abs(c) >= 10: - return False, "Gees -- not even close!" - if c < 0: - return False, "too low" - if c > 0: - return False, "too high" - """ - f = closure(code) - - def g(): - x = f() - return x.get('title', - ''), x.get('question', ''), x.get('answer', ''), x.get( - 'check', None), x.get('hints', None) - - title, question, answer, check, hints = g() - obj = {} - obj['E'] = Exercise(question, answer, check, hints) - obj['title'] = title - - def title_control(t): - return text_control('

%s

' % t) - - the_times = [] - - @interact(layout=[[('go', 1), ('title', 11, '')], [('')], - [('times', 12, "Times:")]], - flicker=True) - def h(go=button(" " * 5 + "Go" + " " * 7, - label='', - icon='fa-refresh', - classes="btn-large btn-success"), - title=title_control(title), - times=text_control('')): - c = interact.changed() - if 'go' in c or 'another' in c: - interact.title = title_control(obj['title']) - - def cb(obj): - the_times.append("%.1f" % obj['time']) - h.times = ', '.join(the_times) - - obj['E'].ask(cb) - - title, question, answer, check, hints = g( - ) # get ready for next time. - obj['title'] = title - obj['E'] = Exercise(question, answer, check, hints) - - -def closure(code): - """ - Wrap the given code block (a string) in a closure, i.e., a - function with an obfuscated random name. - - When called, the function returns locals(). - """ - import uuid - # TODO: strip string literals first - code = ' ' + ('\n '.join(code.splitlines())) - fname = "__" + str(uuid.uuid4()).replace('-', '_') - closure = "def %s():\n%s\n return locals()" % (fname, code) - - class Closure: - def __call__(self): - return self._f() - - c = Closure() - salvus.execute(closure) - c._f = salvus.namespace[fname] - del salvus.namespace[fname] - return c - - -######################################### -# Dynamic variables (linked to controls) -######################################### - - -def _dynamic(var, control=None): - if control is None: - control = salvus.namespace.get(var, '') - - @interact(layout=[[(var, 12)]], output=False) - def f(x=(var, control)): - salvus.namespace.set(var, x, do_not_trigger=[var]) - - def g(y): - f.x = y - - salvus.namespace.on('change', var, g) - - if var in salvus.namespace: - x = salvus.namespace[var] - - -def dynamic(*args, **kwds): - """ - Make variables in the global namespace dynamically linked to a control from the - interact label (see the documentation for interact). - - EXAMPLES: - - Make a control linked to a variable that doesn't yet exist:: - - dynamic('xyz') - - Make a slider and a selector, linked to t and x:: - - dynamic(t=(1..10), x=[1,2,3,4]) - t = 5 # this changes the control - """ - for var in args: - if not is_string(var): - i = id(var) - for k, v in salvus.namespace.items(): - if id(v) == i: - _dynamic(k) - return - else: - _dynamic(var) - - for var, control in kwds.items(): - _dynamic(var, control) - - -import sage.all - - -def var0(*args, **kwds): - if len(args) == 1: - name = args[0] - else: - name = args - G = salvus.namespace - v = sage.all.SR.var(name, **kwds) - if isinstance(v, tuple): - for x in v: - G[repr(x)] = x - else: - G[repr(v)] = v - return v - - -def var(*args, **kwds): - r""" - Create symbolic variables and inject them into the global namespace. - - NOTE: In CoCalc, you can use var as a line decorator:: - - %var x - %var a,b,theta # separate with commas - %var x y z t # separate with spaces - - Use latex_name to customizing how the variables is typeset: - - var1 = var('var1', latex_name=r'\sigma^2_1') - show(e^(var1**2)) - - Multicolored variables made using the %var line decorator: - - %var(latex_name=r"\color{green}{\theta}") theta - %var(latex_name=r"\color{red}{S_{u,i}}") sui - show(expand((sui + x^3 + theta)^2)) - - - - Here is the docstring for var in Sage: - - """ - if 'latex_name' in kwds: - # wrap with braces -- sage should probably do this, but whatever. - kwds['latex_name'] = '{%s}' % kwds['latex_name'] - if len(args) > 0: - return var0(*args, **kwds) - else: - - def f(s): - return var0(s, *args, **kwds) - - return f - - -var.__doc__ += sage.all.var.__doc__ - -############################################# -# Variable reset -- we have to rewrite -# this because of all the monkey patching -# that we do. -############################################# - -import sage.misc.reset - - -def reset(vars=None, attached=False): - """ - If vars is specified, just restore the value of vars and leave - all other variables alone. In CoCalc, you can also use - reset as a line decorator:: - - %reset x, pi, sin # comma-separated - %reset x pi sin # commas are optional - - If vars is not given, delete all user-defined variables, reset - all global variables back to their default states, and reset - all interfaces to other computer algebra systems. - - Original reset docstring:: - - """ - if vars is not None: - restore(vars) - return - G = salvus.namespace - T = type(sys) # module type - for k in list(G.keys()): - if k[0] != '_' and type(k) != T: - try: - if k != 'salvus': - del G[k] - except KeyError: - pass - restore() - from sage.symbolic.assumptions import forget - forget() - sage.misc.reset.reset_interfaces() - if attached: - sage.misc.reset.reset_attached() - # reset() adds 'pretty_print' and 'view' to show_identifiers() - # user can shadow these and they will appear in show_identifiers() - # 'sage_salvus' is added when the following line runs; user may not shadow it - exec('sage.misc.session.state_at_init = dict(globals())', salvus.namespace) - - -reset.__doc__ += sage.misc.reset.reset.__doc__ - - -def restore(vars=None): - "" - if is_string(vars): - vars = str(vars) # sage.misc.reset is unicode ignorant - if ',' in vars: # sage.misc.reset is stupid about commas and space -- TODO: make a patch to sage - vars = [v.strip() for v in vars.split(',')] - import sage.calculus.calculus - sage.misc.reset._restore(salvus.namespace, default_namespace, vars) - sage.misc.reset._restore(sage.calculus.calculus.syms_cur, - sage.calculus.calculus.syms_default, vars) - - -restore.__doc__ += sage.misc.reset.restore.__doc__ - - -# NOTE: this is not used anymore -def md2html(s): - from .markdown2Mathjax import sanitizeInput, reconstructMath - from markdown2 import markdown - - delims = [('\\(', '\\)'), ('$$', '$$'), ('\\[', '\\]'), - ('\\begin{equation}', '\\end{equation}'), - ('\\begin{equation*}', '\\end{equation*}'), - ('\\begin{align}', '\\end{align}'), - ('\\begin{align*}', '\\end{align*}'), - ('\\begin{eqnarray}', '\\end{eqnarray}'), - ('\\begin{eqnarray*}', '\\end{eqnarray*}'), - ('\\begin{math}', '\\end{math}'), - ('\\begin{displaymath}', '\\end{displaymath}')] - - tmp = [((s, None), None)] - for d in delims: - tmp.append((sanitizeInput(tmp[-1][0][0], equation_delims=d), d)) - - extras = ['code-friendly', 'footnotes', 'smarty-pants', 'wiki-tables'] - markedDownText = markdown(tmp[-1][0][0], extras=extras) - - while len(tmp) > 1: - markedDownText = reconstructMath(markedDownText, - tmp[-1][0][1], - equation_delims=tmp[-1][1]) - del tmp[-1] - - return markedDownText - - -# NOTE: this is not used anymore -class Markdown(object): - r""" - Cell mode that renders everything after %md as markdown. - - EXAMPLES:: - - --- - %md - # A Title - - ## A subheading - - --- - %md(hide=True) - # A title - - - a list - - --- - md("# A title") - - - --- - %md `some code` - - - This uses the Python markdown2 library with the following - extras enabled: - - 'code-friendly', 'footnotes', - 'smarty-pants', 'wiki-tables' - - See https://github.com/trentm/python-markdown2/wiki/Extras - We also use markdown2Mathjax so that LaTeX will be properly - typeset if it is wrapped in $'s and $$'s, \(, \), \[, \], - \begin{equation}, \end{equation}, \begin{align}, \end{align}., - """ - def __init__(self, hide=False): - self._hide = hide - - def __call__(self, *args, **kwds): - if len(kwds) > 0 and len(args) == 0: - return Markdown(**kwds) - if len(args) > 0: - self._render(args[0], **kwds) - - def _render(self, s, hide=None): - if hide is None: - hide = self._hide - html(md2html(s), hide=hide) - - -# not used -#md = Markdown() - -# Instead... of the above server-side markdown, we use this client-side markdown. - - -class Marked(object): - r""" - Cell mode that renders everything after %md as Github flavored - markdown [1] with mathjax and hides the input by default. - - [1] https://help.github.com/articles/github-flavored-markdown - - The rendering is done client-side using marked and mathjax. - - EXAMPLES:: - - --- - %md - # A Title - - ## A subheading - - --- - %md(hide=False) - # A title - - - a list - - --- - md("# A title", hide=False) - - - --- - %md(hide=False) `some code` - - """ - def __init__(self, hide=False): - self._hide = hide - - def __call__(self, *args, **kwds): - if len(kwds) > 0 and len(args) == 0: - return Marked(**kwds) - if len(args) > 0: - self._render(args[0], **kwds) - - def _render(self, s, hide=None): - if hide is None: - hide = self._hide - if hide: - salvus.hide('input') - salvus.md(s) - - -md = Marked() - - -##### -## Raw Input -# - this is the Python 2.x interpretation. In Python 3.x there is no raw_input, -# and raw_input is renamed input (to cause more confusion). -##### -def raw_input(prompt='', - default='', - placeholder='', - input_width=None, - label_width=None, - type=None): - """ - Read a string from the user in the worksheet interface to Sage. - - INPUTS: - - - prompt -- (default: '') a label to the left of the input - - default -- (default: '') default value to put in input box - - placeholder -- (default: '') default placeholder to put in grey when input box empty - - input_width -- (default: None) css that gives the width of the input box - - label_width -- (default: None) css that gives the width of the label - - type -- (default: None) if not given, returns a unicode string representing the exact user input. - Other options include: - - type='sage' -- will evaluate it to a sage expression in the global scope. - - type=anything that can be called, e.g., type=int, type=float. - - OUTPUT: - - - By default, returns a **unicode** string (not a normal Python str). However, can be customized - by changing the type. - - EXAMPLE:: - - print(raw_input("What is your full name?", default="Sage Math", input_width="20ex", label_width="25ex")) - - """ - return salvus.raw_input(prompt=prompt, - default=default, - placeholder=placeholder, - input_width=input_width, - label_width=label_width, - type=type) - - -def input(*args, **kwds): - """ - Read a string from the user in the worksheet interface to Sage and return evaluated object. - - Type raw_input? for more help; this function is the same as raw_input, except with type='sage'. - - EXAMPLE:: - - print(type(input("What is your age", default=18, input_width="20ex", label_width="25ex"))) - - """ - kwds['type'] = 'sage' - return salvus.raw_input(*args, **kwds) - - -##### -## Clear -def clear(): - """ - Clear the output of the current cell. You can use this to - dynamically animate the output of a cell using a for loop. - - SEE ALSO: delete_last_output - """ - salvus.clear() - - -def delete_last_output(): - """ - Delete the last output message. - - SEE ALSO: clear - """ - salvus.delete_last_output() - - -##### -# Generic Pandoc cell decorator - - -def pandoc(fmt, doc=None, hide=True): - """ - INPUT: - - - fmt -- one of 'docbook', 'haddock', 'html', 'json', 'latex', 'markdown', 'markdown_github', - 'markdown_mmd', 'markdown_phpextra', 'markdown_strict', 'mediawiki', - 'native', 'opml', 'rst', 'textile' - - - doc -- a string in the given format - - OUTPUT: - - - Called directly, you get the HTML rendered version of doc as a string. - - - If you use this as a cell decorator, it displays the HTML output, e.g., - - %pandoc('mediawiki') - * ''Unordered lists'' are easy to do: - ** Start every line with a star. - *** More stars indicate a deeper level. - - """ - if doc is None: - return lambda x: html(pandoc(fmt, x), hide=hide - ) if x is not None else '' - - import subprocess - p = subprocess.Popen(['pandoc', '-f', fmt, '--mathjax'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - stdin=subprocess.PIPE) - if not is_string(doc): - if six.PY2: - doc = str(doc).encode('utf-8') - else: - doc = str(doc, 'utf8') - p.stdin.write(doc.encode('UTF-8')) - p.stdin.close() - err = p.stderr.read() - if err: - raise RuntimeError(err) - return p.stdout.read() - - -def wiki(doc=None, hide=True): - """ - Mediawiki markup cell decorator. E.g., - - EXAMPLE:: - - %wiki(hide=False) - * ''Unordered lists'' and math like $x^3 - y^2$ are both easy - ** Start every line with a star. - *** More stars indicate a deeper level. """ - if doc is None: - return lambda doc: wiki(doc=doc, hide=hide) if doc else '' - html(pandoc('mediawiki', doc=doc), hide=hide) - - -mediawiki = wiki - -###### - - -def load_html_resource(filename): - fl = filename.lower() - if fl.startswith('http://') or fl.startswith('https://'): - # remote url - url = fl - else: - # local file - url = salvus.file(filename, show=False) - ext = os.path.splitext(filename)[1][1:].lower() - if ext == "css": - salvus.javascript( - '''$.get("%s", function(css) { $('').html(css).appendTo("body")});''' - % url) - elif ext == "html": - salvus.javascript('element.append($("
").load("%s"))' % url) - elif ext == "coffee": - salvus.coffeescript( - '$.ajax({url:"%s"}).done (data) ->\n eval(CoffeeScript.compile(data))' - % url) - elif ext == "js": - salvus.html('' % url) - - -def attach(*args): - r""" - Load file(s) into the Sage worksheet process and add to list of attached files. - All attached files that have changed since they were last loaded are reloaded - the next time a worksheet cell is executed. - - INPUT: - - - ``files`` - list of strings, filenames to attach - - .. SEEALSO:: - - :meth:`sage.repl.attach.attach` docstring has details on how attached files - are handled - """ - # can't (yet) pass "attach = True" to load(), so do this - - if len(args) == 1: - if is_string(args[0]): - args = tuple(args[0].replace(',', ' ').split()) - if isinstance(args[0], (list, tuple)): - args = args[0] - try: - from sage.repl.attach import load_attach_path - except ImportError: - raise NotImplementedError("sage_salvus: attach not available") - - for fname in args: - for path in load_attach_path(): - fpath = os.path.join(path, fname) - fpath = os.path.expanduser(fpath) - if os.path.isfile(fpath): - load(fname) - sage.repl.attach.add_attached_file(fpath) - break - else: - raise IOError('did not find file %r to attach' % fname) - - -# Monkey-patched the load command -def load(*args, **kwds): - """ - Load Sage object from the file with name filename, which will have - an .sobj extension added if it doesn't have one. Or, if the input - is a filename ending in .py, .pyx, or .sage, load that file into - the current running session. Loaded files are not loaded into - their own namespace, i.e., this is much more like Python's - "execfile" than Python's "import". - - You may also load an sobj or execute a code file available on the web - by specifying the full URL to the file. (Set ``verbose = False`` to - supress the download progress indicator.) - - INPUT: - - - args -- any number of filename strings with any of the following extensions: - - .sobj, .sage, .py, .pyx, .html, .css, .js, .coffee, .pdf - - - ``verbose`` -- (default: True) load file over the network. - - If you load any of the web types (.html, .css, .js, .coffee), they are loaded - into the web browser DOM (or Javascript session), not the Python process. - - If you load a pdf, it is displayed in the output of the worksheet. The extra - options are passed to smc.pdf -- see the docstring for that. - - In CoCalc you may also use load as a decorator, with exactly one filename as input:: - - %load foo.sage - - This loads a single file whose name has a space in it:: - - %load a b.sage - - The following are all valid ways to use load:: - - %load a.html - %load a.css - %load a.js - %load a.coffee - %load a.css - load('a.css', 'a.js', 'a.coffee', 'a.html') - load(['a.css', 'a.js', 'a.coffee', 'a.html']) - - ALIAS: %runfile is the same as %load, for compatibility with IPython. - """ - if len(args) == 1: - if is_string(args[0]): - args = (args[0].strip(), ) - if isinstance(args[0], (list, tuple)): - args = args[0] - - if len(args) == 0 and len(kwds) == 1: - # This supports - # %load(verbose=False) a.sage - # which doesn't really matter right now, since there is a bug in Sage's own - # load command, where it isn't verbose for network code, but is for objects. - def f(*args): - return load(*args, **kwds) - - return f - - t = '__tmp__' - i = 0 - while t + str(i) in salvus.namespace: - i += 1 - t += str(i) - - # First handle HTML related args -- these are all very oriented toward cloud.sagemath worksheets - html_extensions = set(['js', 'css', 'coffee', 'html']) - other_args = [] - for arg in args: - i = arg.rfind('.') - if i != -1 and arg[i + 1:].lower() in html_extensions: - load_html_resource(arg) - elif i != -1 and arg[i + 1:].lower() == 'pdf': - show_pdf(arg, **kwds) - else: - other_args.append(arg) - - # pdf? - for arg in args: - i = arg.find('.') - - # now handle remaining non-web arguments. - if other_args: - try: - exec( - 'salvus.namespace["%s"] = sage.misc.persist.load(*__args, **__kwds)' - % t, salvus.namespace, { - '__args': other_args, - '__kwds': kwds - }) - return salvus.namespace[t] - finally: - try: - del salvus.namespace[t] - except: - pass - - -# add alias, due to IPython. -runfile = load - -## Make it so pylab (matplotlib) figures display, at least using pylab.show -import pylab - - -def _show_pylab(svg=True): - """ - Show a Pylab plot in a Sage Worksheet. - - INPUTS: - - - svg -- boolean (default: True); if True use an svg; otherwise, use a png. - """ - try: - ext = '.svg' if svg else '.png' - filename = uuid() + ext - pylab.savefig(filename) - salvus.file(filename) - finally: - try: - os.unlink(filename) - except: - pass - - -pylab.show = _show_pylab -matplotlib.figure.Figure.show = show - -import matplotlib.pyplot - - -def _show_pyplot(svg=True): - """ - Show a Pylab plot in a Sage Worksheet. - - INPUTS: - - - svg -- boolean (default: True); if True use an svg; otherwise, use a png. - """ - try: - ext = '.svg' if svg else '.png' - filename = uuid() + ext - matplotlib.pyplot.savefig(filename) - salvus.file(filename) - finally: - try: - os.unlink(filename) - except: - pass - - -matplotlib.pyplot.show = _show_pyplot - -## Our own displayhook - -_system_sys_displayhook = sys.displayhook - -DISPLAYHOOK_MODULES_SHOW = [ - Graphics3d, - Graphics, - GraphicsArray, - matplotlib.figure.Figure, - matplotlib.axes.Axes, - matplotlib.image.AxesImage, - Animation, - Tachyon, -] - -if MultiGraphics is not None: - DISPLAYHOOK_MODULES_SHOW.append(MultiGraphics) - -DISPLAYHOOK_MODULES_SHOW = tuple(DISPLAYHOOK_MODULES_SHOW) - - -def displayhook(obj): - if isinstance(obj, DISPLAYHOOK_MODULES_SHOW): - show(obj) - else: - _system_sys_displayhook(obj) - - -sys.displayhook = displayhook -import sage.misc.latex, types - -# We make this a list so that users can append to it easily. -TYPESET_MODE_EXCLUDES = [ - sage.misc.latex.LatexExpr, - type(None), - type, - sage.plot.plot3d.base.Graphics3d, - sage.plot.graphics.Graphics, - GraphicsArray, -] - -if MultiGraphics is not None: - TYPESET_MODE_EXCLUDES.append(MultiGraphics) - -TYPESET_MODE_EXCLUDES = tuple(TYPESET_MODE_EXCLUDES) - - -def typeset_mode(on=True, display=True, **args): - """ - Turn typeset mode on or off. When on, each output is typeset using LaTeX. - - EXAMPLES:: - - typeset_mode() # turns typesetting on - - typeset_mode(False) # turn typesetting off - - typeset_mode(True, display=False) # typesetting mode on, but do not make output big and centered - - """ - if is_string(on): # e.g., %typeset_mode False - on = sage_eval(on, {'false': False, 'true': True}) - if on: - - def f(obj): - if isinstance(obj, TYPESET_MODE_EXCLUDES): - displayhook(obj) - else: - show(obj, display=display) - - sys.displayhook = f - else: - sys.displayhook = displayhook - - -def python_future_feature(feature=None, enable=None): - """ - Enable python features from the __future__ system. - - EXAMPLES:: - - Enable python3 printing: - - python_future_feature('print_function', True) - python_future_feature('print_function') # returns True - print("hello", end="") - - Then switch back to python2 printing - python_future_feature('print_function', False) - print "hello" - - """ - return salvus.python_future_feature(feature, enable) - - -def py3print_mode(enable=None): - """ - Enable python3 print syntax. - - EXAMPLES:: - - Enable python3 printing: - - py3print_mode(True) - py3print_mode() # returns True - print("hello", end="") - - Then switch back to python2 printing - py3print_mode(False) - print "hello" - - """ - return salvus.python_future_feature('print_function', enable) - - -def default_mode(mode): - """ - Set the default mode for cell evaluation. This is equivalent - to putting %mode at the top of any cell that does not start - with %. Use default_mode() to return the current mode. - Use default_mode("") to have no default mode. - - EXAMPLES:: - - Make Pari/GP the default mode: - - default_mode("gp") - default_mode() # outputs "gp" - - Then switch back to Sage:: - - default_mode("") # or default_mode("sage") - - You can also use default_mode as a line decorator:: - - %default_mode gp # equivalent to default_mode("gp") - """ - return salvus.default_mode(mode) - - -####################################################### -# Monkey patching and deprecation -- -####################################################### - -# Monkey patch around a bug in Python's findsource that breaks deprecation in cloud worksheets. -# This won't matter if we switch to not using exec, since then there will be a file behind -# each block of code. However, for now we have to do this. -import inspect -_findsource = inspect.findsource - - -def findsource(object): - try: - return _findsource(object) - except: - raise IOError( - 'source code not available') # as *claimed* by the Python docs! - - -inspect.findsource = findsource - -####################################################### -# Viewing pdf's -####################################################### - - -def show_pdf(filename, viewer="object", width=1000, height=600, scale=1.6): - """ - Display a PDF file from the file system in an output cell of a worksheet. - - It uses the HTML object tag, which uses either the browser plugin, - or provides a download link in case the browser can't display pdf's. - - INPUT: - - - filename - - width -- (default: 1000) -- pixel width of viewer - - height -- (default: 600) -- pixel height of viewer - """ - url = salvus.file(filename, show=False) - s = ''' -

Your browser doesn't support embedded PDF's, but you can download %s

- ''' % (url, width, height, url, filename) - salvus.html(s) - - -######################################################## -# Documentation of modes -######################################################## -def modes(): - """ - To use a mode command, either type - - %command - - or - - %command - [rest of cell] - - Create your own mode command by defining a function that takes - a string as input and outputs a string. (Yes, it is that simple.) - """ - import re - mode_cmds = set() - for s in open(os.path.realpath(__file__), 'r'): - s = s.strip() - if s.startswith('%'): - sm = (re.findall(r'%[a-zA-Z]+', s)) - if len(sm) > 0: - mode_cmds.add(sm[0]) - mode_cmds.discard('%s') - for k, v in sage.interfaces.all.__dict__.items(): - if isinstance(v, sage.interfaces.expect.Expect): - mode_cmds.add('%' + k) - mode_cmds.update([ - '%cython', '%time', '%auto', '%hide', '%hideall', '%fork', '%runfile', - '%default_mode', '%typeset_mode' - ]) - v = list(sorted(mode_cmds)) - return v - - -######################################################## -# Go mode -######################################################## -def go(s): - """ - Run a go program. For example, - - %go - func main() { fmt.Println("Hello World") } - - You can set the whole worksheet to be in go mode by typing - - %default_mode go - - NOTES: - - - The official Go tutorial as a long Sage Worksheet is available here: - - https://github.com/sagemath/cloud-examples/tree/master/go - - - There is no relation between one cell and the next. Each is a separate - self-contained go program, which gets compiled and run, with the only - side effects being changes to the file system. The program itself is - stored in a random file that is deleted after it is run. - - - The %go command automatically adds 'package main' and 'import "fmt"' - (if fmt. is used) to the top of the program, since the assumption - is that you're using %go interactively. - """ - import uuid - name = str(uuid.uuid4()) - if 'fmt.' in s and '"fmt"' not in s and "'fmt'" not in s: - s = 'import "fmt"\n' + s - if 'package main' not in s: - s = 'package main\n' + s - try: - open(name + '.go', 'w').write(s.encode("UTF-8")) - (child_stdin, child_stdout, - child_stderr) = os.popen3('go build %s.go' % name) - err = child_stderr.read() - sys.stdout.write(child_stdout.read()) - sys.stderr.write(err) - sys.stdout.flush() - sys.stderr.flush() - if not os.path.exists(name): # failed to produce executable - return - (child_stdin, child_stdout, child_stderr) = os.popen3("./" + name) - sys.stdout.write(child_stdout.read()) - sys.stderr.write(child_stderr.read()) - sys.stdout.flush() - sys.stderr.flush() - finally: - try: - os.unlink(name + '.go') - except: - pass - try: - os.unlink(name) - except: - pass - - -######################################################## -# Java mode -######################################################## -def java(s): - """ - Run a Java program. For example, - - %java - public class YourName { public static void main(String[] args) { System.out.println("Hello world"); } } - - You can set the whole worksheet to be in java mode by typing - - %default_mode java - - NOTE: - - - There is no relation between one cell and the next. Each is a separate - self-contained java program, which gets compiled and run, with the only - side effects being changes to the file system. The program itself is - stored in a file named as the public class that is deleted after it is run. - """ - name = re.search('public class (?P[a-zA-Z0-9]+)', s) - if name: - name = name.group('name') - else: - print('error public class name not found') - return - try: - open(name + '.java', 'w').write(s.encode("UTF-8")) - (child_stdin, child_stdout, - child_stderr) = os.popen3('javac %s.java' % name) - err = child_stderr.read() - sys.stdout.write(child_stdout.read()) - sys.stderr.write(err) - sys.stdout.flush() - sys.stderr.flush() - if not os.path.exists(name + '.class'): # failed to produce executable - return - (child_stdin, child_stdout, child_stderr) = os.popen3('java %s' % name) - sys.stdout.write(child_stdout.read()) - sys.stderr.write('\n' + child_stderr.read()) - sys.stdout.flush() - sys.stderr.flush() - finally: - pass - try: - os.unlink(name + '.java') - except: - pass - try: - os.unlink(name + '.class') - except: - pass - - -######################################################## -# Julia mode -######################################################## - - -def julia(code=None, **kwargs): - """ - Block decorator to run Julia over Jupyter bridge. - - To use this, put %julia on a line by itself in a cell so that it applies to - the rest of the cell, or put it at the beginning of a line to - run just that line using julia. - - State is preserved between cells. - - This is different than the julia command in Sage itself (which you can - access via sage.interfaces.julia), which uses a more brittle pexpect interface. - - """ - if julia.jupyter_kernel is None: - julia.jupyter_kernel = jupyter("julia-1.7") - return julia.jupyter_kernel(code, **kwargs) - - -julia.jupyter_kernel = None - -# Help command -import sage.misc.sagedoc -import sage.version -import sage.misc.sagedoc - - -def help(*args, **kwds): - if len(args) > 0 or len(kwds) > 0: - sage.misc.sagedoc.help(*args, **kwds) - else: - s = """ -## Welcome to Sage %s! - -- **Online documentation:** [View the Sage documentation online](http://www.sagemath.org/doc/). - -- **Help:** For help on any object or function, for example `matrix_plot`, enter `matrix_plot?` followed by tab or shift+enter. For help on any module (or object or function), for example, `sage.matrix`, enter `help(sage.matrix)`. - -- **Tab completion:** Type `obj` followed by tab to see all completions of obj. To see all methods you may call on `obj`, type `obj.` followed by tab. - -- **Source code:** Enter `matrix_plot??` followed by tab or shift+enter to look at the source code of `matrix_plot`. - -- **License information:** For license information about Sage and its components, enter `license()`.""" % sage.version.version - salvus.md(s) - - -# Import the jupyter kernel client. -try: - from .sage_jupyter import jupyter -except: - from sage_jupyter import jupyter - - -# license() workaround for IPython pager -# could also set os.environ['TERM'] to 'dumb' to workaround the pager -def license(): - r""" - Display Sage license file COPYING.txt - - You can also view this information in an SMC terminal session: - - | $ sage - | sage: license() - - """ - print((sage.misc.copying.license)) - - -# search_src -import glob - - -# from http://stackoverflow.com/questions/9877462/is-there-a-python-equivalent-to-the-which-commane -# in python 3.3+ there is shutil.which() -def which(pgm): - path = os.getenv('PATH') - for p in path.split(os.path.pathsep): - p = os.path.join(p, pgm) - if os.path.exists(p) and os.access(p, os.X_OK): - return p - - -try: - from .sage_server import MAX_CODE_SIZE -except: - from sage_server import MAX_CODE_SIZE - - -def search_src(str, max_chars=MAX_CODE_SIZE): - r""" - Get file names resulting from git grep of smc repo - - INPUT: - - - ``str`` -- string, expression to search for; will be quoted - - ``max_chars`` -- integer, max characters to display from selected file - - OUTPUT: - - Interact selector of matching filenames. Choosing one causes its - contents to be shown in salvus.code() output. - """ - sage_cmd = which("sage") - if os.path.islink(sage_cmd): - sage_cmd = os.readlink(sage_cmd) - - # /projects/sage/sage-x.y/src/bin - sdir = os.path.dirname(sage_cmd) - - # /projects/sage/sage-x.y - sdir = os.path.dirname(os.path.dirname(sdir)) - - # /projects/sage/sage-x.y/src - sdir = glob.glob(sdir + "/src/sage")[0] - - cmd = 'cd %s;timeout 5 git grep -il "%s"' % (sdir, str) - srch = os.popen(cmd).read().splitlines() - header = "files matched" - nftext = header + ": %s" % len(srch) - - @interact - def _(fname=selector([nftext] + srch, "view source file:")): - if not fname.startswith(header): - with open(os.path.join(sdir, fname), 'r') as infile: - code = infile.read(max_chars) - salvus.code(code, mode="python", filename=fname) - - -# search_doc -def search_doc(str): - r""" - Create link to Google search of sage docs. - - INPUT: - - - ``str`` -- string, expression to search for; will be quoted - - OUTPUT: - - HTML hyperlink to google search - """ - txt = 'Use this link to search: ' + \ - ''+str+'' - salvus.html(txt) - - -import sage.misc.session - - -def show_identifiers(): - """ - Returns a list of all variable names that have been defined during this session. - - SMC introduces worksheet variables, including 'smc','salvus', 'require', and after reset(), 'sage_salvus'. - These identifiers are removed from the output of sage.misc.session.show_identifiers() on return. - User should not assign to these variables when running code in a worksheet. - """ - si = eval('show_identifiers.fn()', salvus.namespace) - si2 = [ - v for v in si if v not in ['smc', 'salvus', 'require', 'sage_salvus'] - ] - return si2 - - -show_identifiers.fn = sage.misc.session.show_identifiers diff --git a/src/smc_sagews/smc_sagews/sage_server.py b/src/smc_sagews/smc_sagews/sage_server.py deleted file mode 100755 index 6366005292..0000000000 --- a/src/smc_sagews/smc_sagews/sage_server.py +++ /dev/null @@ -1,2414 +0,0 @@ -#!/usr/bin/env python -""" -sage_server.py -- unencrypted forking TCP server. - -Note: I wrote functionality so this can run as root, create accounts on the fly, -and serve sage as those accounts. Doing this is horrendous from a security point of -view, and I'm definitely not doing this. - -None of that functionality is actually used in https://cocalc.com! - -For debugging, this may help: - - killemall sage_server.py && sage --python sage_server.py -p 6000 - -""" - -# NOTE: This file is GPL'd -# because it imports the Sage library. This file is not directly -# imported by anything else in CoCalc; the Python process it runs is -# used over a TCP connection. - -######################################################################################### -# Copyright (C) 2016, Sagemath Inc. -# # -# Distributed under the terms of the GNU General Public License (GPL), version 2+ # -# # -# http://www.gnu.org/licenses/ # -######################################################################################### - -# Add the path that contains this file to the Python load path, so we -# can import other files from there. -from __future__ import absolute_import -import six -import os, sys, time, operator -import __future__ as future -from functools import reduce - - -def is_string(s): - return isinstance(s, six.string_types) - - -def unicode8(s): - # I evidently don't understand Python unicode... Do the following for now: - # TODO: see http://stackoverflow.com/questions/21897664/why-does-unicodeu-passed-an-errors-parameter-raise-typeerror for how to fix. - try: - if six.PY2: - return str(s).encode('utf-8') - else: - return str(s, 'utf-8') - except: - try: - return str(s) - except: - return s - - -LOGFILE = os.path.realpath(__file__)[:-3] + ".log" -PID = os.getpid() -from datetime import datetime - - -def log(*args): - try: - debug_log = open(LOGFILE, 'a') - from sys import version_info - if version_info >= (3, 12): - from datetime import UTC - d = datetime.now(UTC) - else: - d = datetime.utcnow() - d_txt = d.strftime('%Y-%m-%d %H:%M:%S.%f')[:-3] - args_str = ' '.join([unicode8(x) for x in args]) - mesg = "%s (%s): %s\n" % (PID, d_txt, args_str) - debug_log.write(mesg) - debug_log.flush() - except Exception as err: - print(("an error writing a log message (ignoring) -- %s" % err, args)) - - -# used for clearing pylab figure -pylab = None - -# Maximum number of distinct (non-once) output messages per cell; when this number is -# exceeded, an exception is raised; this reduces the chances of the user creating -# a huge unusable worksheet. -MAX_OUTPUT_MESSAGES = 256 -# stdout, stderr, html, etc. that exceeds this many characters will be truncated to avoid -# killing the client. -MAX_STDOUT_SIZE = MAX_STDERR_SIZE = MAX_CODE_SIZE = MAX_HTML_SIZE = MAX_MD_SIZE = MAX_TEX_SIZE = 40000 - -MAX_OUTPUT = 150000 - -# Standard imports. -import json, resource, shutil, signal, socket, struct, \ - tempfile, time, traceback, pwd, re - -# for "3x^2 + 4xy - 5(1+x) - 3 abc4ok", this pattern matches "3x", "5(" and "4xy" but not "abc4ok" -# to understand it, see https://regex101.com/ or https://www.debuggex.com/ -RE_POSSIBLE_IMPLICIT_MUL = re.compile(r'(?:(?<=[^a-zA-Z])|^)(\d+[a-zA-Z\(]+)') - -try: - from . import sage_parsing, sage_salvus -except: - import sage_parsing, sage_salvus - -uuid = sage_salvus.uuid - -reload_attached_files_if_mod_smc_available = True - - -def reload_attached_files_if_mod_smc(): - # CRITICAL: do NOT impor sage.repl.attach!! That will import IPython, wasting several seconds and - # killing the user experience for no reason. - try: - import sage.repl - sage.repl.attach - except: - # nothing to do -- attach has not been used and is not yet available. - return - global reload_attached_files_if_mod_smc_available - if not reload_attached_files_if_mod_smc_available: - return - try: - from sage.repl.attach import load_attach_path, modified_file_iterator - except: - print("sage_server: attach not available") - reload_attached_files_if_mod_smc_available = False - return - # see sage/src/sage/repl/attach.py reload_attached_files_if_modified() - for filename, mtime in modified_file_iterator(): - basename = os.path.basename(filename) - timestr = time.strftime('%T', mtime) - log('reloading attached file {0} modified at {1}'.format( - basename, timestr)) - from .sage_salvus import load - load(filename) - - -# Determine the info object, if available. There's no good reason -# it wouldn't be available, unless a user explicitly deleted it, but -# we may as well try to be robust to this, especially if somebody -# were to try to use this server outside of cloud.sagemath.com. -_info_path = os.path.join(os.environ['SMC'], 'info.json') -if os.path.exists(_info_path): - try: - INFO = json.loads(open(_info_path).read()) - except: - # This will fail, e.g., if info.json is invalid (maybe a blank file). - # We definitely don't want sage server startup to be completely broken - # in this case, so we fall back to "no info". - INFO = {} -else: - INFO = {} -if 'base_url' not in INFO: - INFO['base_url'] = '' - -# Configure logging -#logging.basicConfig() -#log = logging.getLogger('sage_server') -#log.setLevel(logging.INFO) - -# A CoffeeScript version of this function is in misc_node.coffee. -import hashlib - - -def uuidsha1(data): - sha1sum = hashlib.sha1() - sha1sum.update(data) - s = sha1sum.hexdigest() - t = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx' - r = list(t) - j = 0 - for i in range(len(t)): - if t[i] == 'x': - r[i] = s[j] - j += 1 - elif t[i] == 'y': - # take 8 + low order 3 bits of hex number. - r[i] = hex((int(s[j], 16) & 0x3) | 0x8)[-1] - j += 1 - return ''.join(r) - - -# A tcp connection with support for sending various types of messages, especially JSON. -class ConnectionJSON(object): - - def __init__(self, conn): - # avoid common mistake -- conn is supposed to be from socket.socket... - assert not isinstance(conn, ConnectionJSON) - self._conn = conn - - def close(self): - self._conn.close() - - def _send(self, s): - if six.PY3 and type(s) == str: - s = s.encode('utf8') - length_header = struct.pack(">L", len(s)) - # py3: TypeError: can't concat str to bytes - self._conn.send(length_header + s) - - def send_json(self, m): - m = json.dumps(m) - if '\\u0000' in m: - raise RuntimeError("NULL bytes not allowed") - log("sending message '", truncate_text(m, 256), "'") - self._send('j' + m) - return len(m) - - def send_blob(self, blob): - if six.PY3 and type(blob) == str: - # unicode objects must be encoded before hashing - blob = blob.encode('utf8') - - s = uuidsha1(blob) - if six.PY3 and type(blob) == bytes: - # we convert all to bytes first, to avoid unnecessary conversions - self._send(('b' + s).encode('utf8') + blob) - else: - # old sage py2 code - self._send('b' + s + blob) - return s - - def send_file(self, filename): - log("sending file '%s'" % filename) - f = open(filename, 'rb') - data = f.read() - f.close() - return self.send_blob(data) - - def _recv(self, n): - #print("_recv(%s)"%n) - # see http://stackoverflow.com/questions/3016369/catching-blocking-sigint-during-system-call - for i in range(20): - try: - #print "blocking recv (i = %s), pid=%s"%(i, os.getpid()) - r = self._conn.recv(n) - #log("n=%s; received: '%s' of len %s"%(n,r, len(r))) - return r - except OSError as e: - #print("socket.error, msg=%s"%msg) - if e.errno != 4: - raise - raise EOFError - - def recv(self): - n = self._recv(4) - if len(n) < 4: - raise EOFError - n = struct.unpack('>L', n)[0] # big endian 32 bits - s = self._recv(n) - while len(s) < n: - t = self._recv(n - len(s)) - if len(t) == 0: - raise EOFError - s += t - - if six.PY3: - # bystream to string, in particular s[0] will be e.g. 'j' and not 106 - #log("ConnectionJSON::recv s=%s... (type %s)" % (s[:5], type(s))) - # is s always of type bytes? - if type(s) == bytes: - s = s.decode('utf8') - - if s[0] == 'j': - try: - return 'json', json.loads(s[1:]) - except Exception as msg: - log("Unable to parse JSON '%s'" % s[1:]) - raise - - elif s[0] == 'b': - return 'blob', s[1:] - raise ValueError("unknown message type '%s'" % s[0]) - - -def truncate_text(s, max_size): - if len(s) > max_size: - return s[:max_size] + "[...]", True - else: - return s, False - - -def truncate_text_warn(s, max_size, name): - r""" - Truncate text if too long and format a warning message. - - INPUT: - - - ``s`` -- string to be truncated - - ``max-size`` - integer truncation limit - - ``name`` - string, name of limiting parameter - - OUTPUT: - - a triple: - - - string -- possibly truncated input string - - boolean -- true if input string was truncated - - string -- warning message if input string was truncated - """ - tmsg = "WARNING: Output: %s truncated by %s to %s. Type 'smc?' to learn how to raise the output limit." - lns = len(s) - if lns > max_size: - tmsg = tmsg % (lns, name, max_size) - return s[:max_size] + "[...]", True, tmsg - else: - return s, False, '' - - -class Message(object): - - def _new(self, event, props={}): - m = {'event': event} - for key, val in props.items(): - if key != 'self': - m[key] = val - return m - - def start_session(self): - return self._new('start_session') - - def session_description(self, pid): - return self._new('session_description', {'pid': pid}) - - def send_signal(self, pid, signal=signal.SIGINT): - return self._new('send_signal', locals()) - - def terminate_session(self, done=True): - return self._new('terminate_session', locals()) - - def execute_code(self, id, code, preparse=True): - return self._new('execute_code', locals()) - - def execute_javascript(self, code, obj=None, coffeescript=False): - return self._new('execute_javascript', locals()) - - def output( - self, - id, - stdout=None, - stderr=None, - code=None, - html=None, - javascript=None, - coffeescript=None, - interact=None, - md=None, - tex=None, - d3=None, - file=None, - raw_input=None, - obj=None, - once=None, - hide=None, - show=None, - events=None, - clear=None, - delete_last=None, - done=False # CRITICAL: done must be specified for multi-response; this is assumed by sage_session.coffee; otherwise response assumed single. - ): - m = self._new('output') - m['id'] = id - t = truncate_text_warn - did_truncate = False - from . import sage_server # we do this so that the user can customize the MAX's below. - if code is not None: - code['source'], did_truncate, tmsg = t(code['source'], - sage_server.MAX_CODE_SIZE, - 'MAX_CODE_SIZE') - m['code'] = code - if stderr is not None and len(stderr) > 0: - m['stderr'], did_truncate, tmsg = t(stderr, - sage_server.MAX_STDERR_SIZE, - 'MAX_STDERR_SIZE') - if stdout is not None and len(stdout) > 0: - m['stdout'], did_truncate, tmsg = t(stdout, - sage_server.MAX_STDOUT_SIZE, - 'MAX_STDOUT_SIZE') - if html is not None and len(html) > 0: - m['html'], did_truncate, tmsg = t(html, sage_server.MAX_HTML_SIZE, - 'MAX_HTML_SIZE') - if md is not None and len(md) > 0: - m['md'], did_truncate, tmsg = t(md, sage_server.MAX_MD_SIZE, - 'MAX_MD_SIZE') - if tex is not None and len(tex) > 0: - tex['tex'], did_truncate, tmsg = t(tex['tex'], - sage_server.MAX_TEX_SIZE, - 'MAX_TEX_SIZE') - m['tex'] = tex - if javascript is not None: m['javascript'] = javascript - if coffeescript is not None: m['coffeescript'] = coffeescript - if interact is not None: m['interact'] = interact - if d3 is not None: m['d3'] = d3 - if obj is not None: m['obj'] = json.dumps(obj) - if file is not None: m['file'] = file # = {'filename':..., 'uuid':...} - if raw_input is not None: m['raw_input'] = raw_input - if done is not None: m['done'] = done - if once is not None: m['once'] = once - if hide is not None: m['hide'] = hide - if show is not None: m['show'] = show - if events is not None: m['events'] = events - if clear is not None: m['clear'] = clear - if delete_last is not None: m['delete_last'] = delete_last - if did_truncate: - if 'stderr' in m: - m['stderr'] += '\n' + tmsg - else: - m['stderr'] = '\n' + tmsg - return m - - def introspect_completions(self, id, completions, target): - m = self._new('introspect_completions', locals()) - m['id'] = id - return m - - def introspect_docstring(self, id, docstring, target): - m = self._new('introspect_docstring', locals()) - m['id'] = id - return m - - def introspect_source_code(self, id, source_code, target): - m = self._new('introspect_source_code', locals()) - m['id'] = id - return m - - -message = Message() - -whoami = os.environ['USER'] - - -def client1(port, hostname): - conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - conn.connect((hostname, int(port))) - conn = ConnectionJSON(conn) - - conn.send_json(message.start_session()) - typ, mesg = conn.recv() - pid = mesg['pid'] - print(("PID = %s" % pid)) - - id = 0 - while True: - try: - code = sage_parsing.get_input('sage [%s]: ' % id) - if code is None: # EOF - break - conn.send_json(message.execute_code(code=code, id=id)) - while True: - typ, mesg = conn.recv() - if mesg['event'] == 'terminate_session': - return - elif mesg['event'] == 'output': - if 'stdout' in mesg: - sys.stdout.write(mesg['stdout']) - sys.stdout.flush() - if 'stderr' in mesg: - print(('! ' + - '\n! '.join(mesg['stderr'].splitlines()))) - if 'done' in mesg and mesg['id'] >= id: - break - id += 1 - - except KeyboardInterrupt: - print("Sending interrupt signal") - conn2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - conn2.connect((hostname, int(port))) - conn2 = ConnectionJSON(conn2) - conn2.send_json(message.send_signal(pid)) - del conn2 - id += 1 - - conn.send_json(message.terminate_session()) - print("\nExiting Sage client.") - - -class BufferedOutputStream(object): - - def __init__(self, f, flush_size=4096, flush_interval=.1): - self._f = f - self._buf = '' - self._flush_size = flush_size - self._flush_interval = flush_interval - self.reset() - - def reset(self): - self._last_flush_time = time.time() - - def fileno(self): - return 0 - - def write(self, output): - # CRITICAL: we need output to valid PostgreSQL TEXT, so no null bytes - # This is not going to silently corrupt anything -- it's just output that - # is destined to be *rendered* in the browser. This is only a partial - # solution to a more general problem, but it is safe. - try: - self._buf += output.replace('\x00', '') - except UnicodeDecodeError: - self._buf += output.decode('utf-8').replace('\x00', '') - #self.flush() - t = time.time() - if ((len(self._buf) >= self._flush_size) - or (t - self._last_flush_time >= self._flush_interval)): - self.flush() - self._last_flush_time = t - - def flush(self, done=False): - if not self._buf and not done: - # no point in sending an empty message - return - try: - self._f(self._buf, done=done) - except UnicodeDecodeError: - if six.PY2: # str doesn't have errors option in python2! - self._f(unicode(self._buf, errors='replace'), done=done) - else: - self._f(str(self._buf, errors='replace'), done=done) - self._buf = '' - - def isatty(self): - return False - - -# This will *have* to be re-done using Cython for speed. -class Namespace(dict): - - def __init__(self, x): - self._on_change = {} - self._on_del = {} - dict.__init__(self, x) - - def on(self, event, x, f): - if event == 'change': - if x not in self._on_change: - self._on_change[x] = [] - self._on_change[x].append(f) - elif event == 'del': - if x not in self._on_del: - self._on_del[x] = [] - self._on_del[x].append(f) - - def remove(self, event, x, f): - if event == 'change' and x in self._on_change: - v = self._on_change[x] - i = v.find(f) - if i != -1: - del v[i] - if len(v) == 0: - del self._on_change[x] - elif event == 'del' and x in self._on_del: - v = self._on_del[x] - i = v.find(f) - if i != -1: - del v[i] - if len(v) == 0: - del self._on_del[x] - - def __setitem__(self, x, y): - dict.__setitem__(self, x, y) - try: - if x in self._on_change: - for f in self._on_change[x]: - f(y) - if None in self._on_change: - for f in self._on_change[None]: - f(x, y) - except Exception as mesg: - print(mesg) - - def __delitem__(self, x): - try: - if x in self._on_del: - for f in self._on_del[x]: - f() - if None in self._on_del: - for f in self._on_del[None]: - f(x) - except Exception as mesg: - print(mesg) - dict.__delitem__(self, x) - - def set(self, x, y, do_not_trigger=None): - dict.__setitem__(self, x, y) - if x in self._on_change: - if do_not_trigger is None: - do_not_trigger = [] - for f in self._on_change[x]: - if f not in do_not_trigger: - f(y) - if None in self._on_change: - for f in self._on_change[None]: - f(x, y) - - -class TemporaryURL: - - def __init__(self, url, ttl): - self.url = url - self.ttl = ttl - - def __repr__(self): - return repr(self.url) - - def __str__(self): - return self.url - - -namespace = Namespace({}) - - -class Salvus(object): - """ - Cell execution state object and wrapper for access to special CoCalc Server functionality. - - An instance of this object is created each time you execute a cell. It has various methods - for sending different types of output messages, links to files, etc. Type 'help(smc)' for - more details. - - OUTPUT LIMITATIONS -- There is an absolute limit on the number of messages output for a given - cell, and also the size of the output message for each cell. You can access or change - those limits dynamically in a worksheet as follows by viewing or changing any of the - following variables:: - - sage_server.MAX_STDOUT_SIZE # max length of each stdout output message - sage_server.MAX_STDERR_SIZE # max length of each stderr output message - sage_server.MAX_MD_SIZE # max length of each md (markdown) output message - sage_server.MAX_HTML_SIZE # max length of each html output message - sage_server.MAX_TEX_SIZE # max length of tex output message - sage_server.MAX_OUTPUT_MESSAGES # max number of messages output for a cell. - - And:: - - sage_server.MAX_OUTPUT # max total character output for a single cell; computation - # terminated/truncated if sum of above exceeds this. - """ - Namespace = Namespace - _prefix = '' - _postfix = '' - _default_mode = 'sage' - _py_features = {} - - def _flush_stdio(self): - """ - Flush the standard output streams. This should be called before sending any message - that produces output. - """ - sys.stdout.flush() - sys.stderr.flush() - - def __repr__(self): - return '' - - def __init__(self, conn, id, data=None, cell_id=None, message_queue=None): - self._conn = conn - self._num_output_messages = 0 - self._total_output_length = 0 - self._output_warning_sent = False - self._id = id - self._done = True # done=self._done when last execute message is sent; e.g., set self._done = False to not close cell on code term. - self.data = data - self.cell_id = cell_id - self.namespace = namespace - self.message_queue = message_queue - self.code_decorators = [] # gets reset if there are code decorators - # Alias: someday remove all references to "salvus" and instead use smc. - # For now this alias is easier to think of and use. - namespace['smc'] = namespace[ - 'salvus'] = self # beware of circular ref? - # Monkey patch in our "require" command. - namespace['require'] = self.require - # Make the salvus object itself available when doing "from sage.all import *". - import sage.all - sage.all.salvus = self - - def _send_output(self, *args, **kwds): - if self._output_warning_sent: - raise KeyboardInterrupt - mesg = message.output(*args, **kwds) - if not mesg.get('once', False): - self._num_output_messages += 1 - from . import sage_server - - if self._num_output_messages > sage_server.MAX_OUTPUT_MESSAGES: - self._output_warning_sent = True - err = "\nToo many output messages: %s (at most %s per cell -- type 'smc?' to learn how to raise this limit): attempting to terminate..." % ( - self._num_output_messages, sage_server.MAX_OUTPUT_MESSAGES) - self._conn.send_json( - message.output(stderr=err, id=self._id, once=False, done=True)) - raise KeyboardInterrupt - - n = self._conn.send_json(mesg) - self._total_output_length += n - - if self._total_output_length > sage_server.MAX_OUTPUT: - self._output_warning_sent = True - err = "\nOutput too long: %s -- MAX_OUTPUT (=%s) exceeded (type 'smc?' to learn how to raise this limit): attempting to terminate..." % ( - self._total_output_length, sage_server.MAX_OUTPUT) - self._conn.send_json( - message.output(stderr=err, id=self._id, once=False, done=True)) - raise KeyboardInterrupt - - def obj(self, obj, done=False): - self._send_output(obj=obj, id=self._id, done=done) - return self - - def link(self, filename, label=None, foreground=True, cls=''): - """ - Output a clickable link to a file somewhere in this project. The filename - path must be relative to the current working directory of the Python process. - - The simplest way to use this is - - salvus.link("../name/of/file") # any relative path to any file - - This creates a link, which when clicked on, opens that file in the foreground. - - If the filename is the name of a directory, clicking will instead - open the file browser on that directory: - - salvus.link("../name/of/directory") # clicking on the resulting link opens a directory - - If you would like a button instead of a link, pass cls='btn'. You can use any of - the standard Bootstrap button classes, e.g., btn-small, btn-large, btn-success, etc. - - If you would like to change the text in the link (or button) to something - besides the default (filename), just pass arbitrary HTML to the label= option. - - INPUT: - - - filename -- a relative path to a file or directory - - label -- (default: the filename) html label for the link - - foreground -- (default: True); if True, opens link in the foreground - - cls -- (default: '') optional CSS classes, such as 'btn'. - - EXAMPLES: - - Use as a line decorator:: - - %salvus.link name/of/file.foo - - Make a button:: - - salvus.link("foo/bar/", label="The Bar Directory", cls='btn') - - Make two big blue buttons with plots in them:: - - plot(sin, 0, 20).save('sin.png') - plot(cos, 0, 20).save('cos.png') - for img in ['sin.png', 'cos.png']: - salvus.link(img, label=""%salvus.file(img, show=False), cls='btn btn-large btn-primary') - - - - """ - path = os.path.abspath(filename)[len(os.environ['HOME']) + 1:] - if label is None: - label = filename - id = uuid() - self.html("" % - (cls, id)) - - s = "$('#%s').html(obj.label).click(function() {%s; return false;});" % ( - id, self._action(path, foreground)) - self.javascript(s, - obj={ - 'label': label, - 'path': path, - 'foreground': foreground - }, - once=False) - - def _action(self, path, foreground): - if os.path.isdir(path): - if foreground: - action = "worksheet.project_page.open_directory(obj.path);" - else: - action = "worksheet.project_page.set_current_path(obj.path);" - else: - action = "worksheet.project_page.open_file({'path':obj.path, 'foreground': obj.foreground});" - return action - - def open_tab(self, filename, foreground=True): - """ - Open a new file (or directory) document in another tab. - See the documentation for salvus.link. - """ - path = os.path.abspath(filename)[len(os.environ['HOME']) + 1:] - self.javascript(self._action(path, foreground), - obj={ - 'path': path, - 'foreground': foreground - }, - once=True) - - def close_tab(self, filename): - """ - Close an open file tab. The filename is relative to the current working directory. - """ - self.javascript("worksheet.project_page.close_file(obj)", - obj=filename, - once=True) - - def threed( - self, - g, # sage Graphic3d object. - width=None, - height=None, - frame=True, # True/False or {'color':'black', 'thickness':.4, 'labels':True, 'fontsize':14, 'draw':True, - # 'xmin':?, 'xmax':?, 'ymin':?, 'ymax':?, 'zmin':?, 'zmax':?} - background=None, - foreground=None, - spin=False, - aspect_ratio=None, - frame_aspect_ratio=None, # synonym for aspect_ratio - done=False, - renderer=None, # None, 'webgl', or 'canvas' - ): - - from .graphics import graphics3d_to_jsonable, json_float as f - - # process options, combining ones set explicitly above with ones inherited from 3d scene - opts = { - 'width': width, - 'height': height, - 'background': background, - 'foreground': foreground, - 'spin': spin, - 'aspect_ratio': aspect_ratio, - 'renderer': renderer - } - - extra_kwds = {} if g._extra_kwds is None else g._extra_kwds - - # clean up and normalize aspect_ratio option - if aspect_ratio is None: - if frame_aspect_ratio is not None: - aspect_ratio = frame_aspect_ratio - elif 'frame_aspect_ratio' in extra_kwds: - aspect_ratio = extra_kwds['frame_aspect_ratio'] - elif 'aspect_ratio' in extra_kwds: - aspect_ratio = extra_kwds['aspect_ratio'] - if aspect_ratio is not None: - if aspect_ratio == 1 or aspect_ratio == "automatic": - aspect_ratio = None - elif not (isinstance(aspect_ratio, - (list, tuple)) and len(aspect_ratio) == 3): - raise TypeError( - "aspect_ratio must be None, 1 or a 3-tuple, but it is '%s'" - % (aspect_ratio, )) - else: - aspect_ratio = [f(x) for x in aspect_ratio] - - opts['aspect_ratio'] = aspect_ratio - - for k in [ - 'spin', - 'height', - 'width', - 'background', - 'foreground', - 'renderer', - ]: - if k in extra_kwds and not opts.get(k, None): - opts[k] = extra_kwds[k] - - if not isinstance(opts['spin'], bool): - opts['spin'] = f(opts['spin']) - opts['width'] = f(opts['width']) - opts['height'] = f(opts['height']) - - # determine the frame - b = g.bounding_box() - xmin, xmax, ymin, ymax, zmin, zmax = b[0][0], b[1][0], b[0][1], b[1][ - 1], b[0][2], b[1][2] - fr = opts['frame'] = { - 'xmin': f(xmin), - 'xmax': f(xmax), - 'ymin': f(ymin), - 'ymax': f(ymax), - 'zmin': f(zmin), - 'zmax': f(zmax) - } - - if isinstance(frame, dict): - for k in list(fr.keys()): - if k in frame: - fr[k] = f(frame[k]) - fr['draw'] = frame.get('draw', True) - fr['color'] = frame.get('color', None) - fr['thickness'] = f(frame.get('thickness', None)) - fr['labels'] = frame.get('labels', None) - if 'fontsize' in frame: - fr['fontsize'] = int(frame['fontsize']) - elif isinstance(frame, bool): - fr['draw'] = frame - - # convert the Sage graphics object to a JSON object that can be rendered - scene = {'opts': opts, 'obj': graphics3d_to_jsonable(g)} - - # Store that object in the database, rather than sending it directly as an output message. - # We do this since obj can easily be quite large/complicated, and managing it as part of the - # document is too slow and doesn't scale. - blob = json.dumps(scene, separators=(',', ':')) - uuid = self._conn.send_blob(blob) - - # flush output (so any text appears before 3d graphics, in case they are interleaved) - self._flush_stdio() - - # send message pointing to the 3d 'file', which will get downloaded from database - self._send_output(id=self._id, - file={ - 'filename': unicode8("%s.sage3d" % uuid), - 'uuid': uuid - }, - done=done) - - def d3_graph(self, g, **kwds): - from .graphics import graph_to_d3_jsonable - self._send_output(id=self._id, - d3={ - "viewer": "graph", - "data": graph_to_d3_jsonable(g, **kwds) - }) - - def file(self, - filename, - show=True, - done=False, - download=False, - once=False, - events=None, - raw=False, - text=None): - """ - Display or provide a link to the given file. Raises a RuntimeError if this - is not possible, e.g, if the file is too large. - - If show=True (the default), the browser will show the file, - or provide a clickable link to it if there is no way to show it. - If text is also given that will be used instead of the path to the file. - - If show=False, this function returns an object T such that - T.url (or str(t)) is a string of the form "/blobs/filename?uuid=the_uuid" - that can be used to access the file even if the file is immediately - deleted after calling this function (the file is stored in a database). - Also, T.ttl is the time to live (in seconds) of the object. A ttl of - 0 means the object is permanently available. - - raw=False (the default): - If you use the URL - /blobs/filename?uuid=the_uuid&download - then the server will include a header that tells the browser to - download the file to disk instead of displaying it. Only relatively - small files can be made available this way. However, they remain - available (for a day) even *after* the file is deleted. - NOTE: It is safe to delete the file immediately after this - function (salvus.file) returns. - - raw=True: - Instead, the URL is to the raw file, which is served directly - from the project: - /project-id/raw/path/to/filename - This will only work if the file is not deleted; however, arbitrarily - large files can be streamed this way. This is useful for animations. - - This function creates an output message {file:...}; if the user saves - a worksheet containing this message, then any referenced blobs are made - permanent in the database. - - The uuid is based on the Sha-1 hash of the file content (it is computed using the - function sage_server.uuidsha1). Any two files with the same content have the - same Sha1 hash. - - The file does NOT have to be in the HOME directory. - """ - filename = unicode8(filename) - if raw: - info = self.project_info() - path = os.path.abspath(filename) - home = os.environ['HOME'] + '/' - - if not path.startswith(home): - # Attempt to use the $HOME/.smc/root symlink instead: - path = os.path.join(os.environ['HOME'], '.smc', 'root', - path.lstrip('/')) - - if path.startswith(home): - path = path[len(home):] - else: - raise ValueError( - "can only send raw files in your home directory -- path='%s'" - % path) - url = os.path.join('/', info['base_url'].strip('/'), - info['project_id'], 'raw', path.lstrip('/')) - if show: - self._flush_stdio() - self._send_output(id=self._id, - once=once, - file={ - 'filename': filename, - 'url': url, - 'show': show, - 'text': text - }, - events=events, - done=done) - return - else: - return TemporaryURL(url=url, ttl=0) - - file_uuid = self._conn.send_file(filename) - - mesg = None - while mesg is None: - self.message_queue.recv() - for i, (typ, m) in enumerate(self.message_queue.queue): - if typ == 'json' and m.get('event') == 'save_blob' and m.get( - 'sha1') == file_uuid: - mesg = m - del self.message_queue[i] - break - - if 'error' in mesg: - raise RuntimeError("error saving blob -- %s" % mesg['error']) - - self._flush_stdio() - self._send_output(id=self._id, - once=once, - file={ - 'filename': filename, - 'uuid': file_uuid, - 'show': show, - 'text': text - }, - events=events, - done=done) - if not show: - info = self.project_info() - url = "%s/blobs/%s?uuid=%s" % (info['base_url'], filename, - file_uuid) - if download: - url += '?download' - return TemporaryURL(url=url, ttl=mesg.get('ttl', 0)) - - def python_future_feature(self, feature=None, enable=None): - """ - Allow users to enable, disable, and query the features in the python __future__ module. - """ - if feature is None: - if enable is not None: - raise ValueError( - "enable may not be specified when feature is None") - return sorted(Salvus._py_features.keys()) - - attr = getattr(future, feature, None) - if (feature not in future.all_feature_names) or ( - attr is None) or not isinstance(attr, future._Feature): - raise RuntimeError("future feature %.50r is not defined" % - (feature, )) - - if enable is None: - return feature in Salvus._py_features - - if enable: - Salvus._py_features[feature] = attr - else: - try: - del Salvus._py_features[feature] - except KeyError: - pass - - def default_mode(self, mode=None): - """ - Set the default mode for cell evaluation. This is equivalent - to putting %mode at the top of any cell that does not start - with %. Use salvus.default_mode() to return the current mode. - Use salvus.default_mode("") to have no default mode. - - This is implemented using salvus.cell_prefix. - """ - if mode is None: - return Salvus._default_mode - Salvus._default_mode = mode - if mode == "sage": - self.cell_prefix("") - else: - self.cell_prefix("%" + mode) - - def cell_prefix(self, prefix=None): - """ - Make it so that the given prefix code is textually - prepending to the input before evaluating any cell, unless - the first character of the cell is a %. - - To append code at the end, use cell_postfix. - - INPUT: - - - ``prefix`` -- None (to return prefix) or a string ("" to disable) - - EXAMPLES: - - Make it so every cell is timed: - - salvus.cell_prefix('%time') - - Make it so cells are typeset using latex, and latex comments are allowed even - as the first line. - - salvus.cell_prefix('%latex') - - %sage salvus.cell_prefix('') - - Evaluate each cell using GP (Pari) and display the time it took: - - salvus.cell_prefix('%time\n%gp') - - %sage salvus.cell_prefix('') # back to normal - """ - if prefix is None: - return Salvus._prefix - else: - Salvus._prefix = prefix - - def cell_postfix(self, postfix=None): - """ - Make it so that the given code is textually - appended to the input before evaluating a cell. - To prepend code at the beginning, use cell_prefix. - - INPUT: - - - ``postfix`` -- None (to return postfix) or a string ("" to disable) - - EXAMPLES: - - Print memory usage after evaluating each cell: - - salvus.cell_postfix('print("%s MB used"%int(get_memory_usage()))') - - Return to normal - - salvus.set_cell_postfix('') - - """ - if postfix is None: - return Salvus._postfix - else: - Salvus._postfix = postfix - - def execute(self, code, namespace=None, preparse=True, locals=None): - - ascii_warn = False - code_error = False - if sys.getdefaultencoding() == 'ascii': - for c in code: - if ord(c) >= 128: - ascii_warn = True - break - - if namespace is None: - namespace = self.namespace - - # clear pylab figure (takes a few microseconds) - if pylab is not None: - pylab.clf() - - compile_flags = reduce(operator.or_, - (feature.compiler_flag - for feature in Salvus._py_features.values()), - 0) - - #code = sage_parsing.strip_leading_prompts(code) # broken -- wrong on "def foo(x):\n print(x)" - blocks = sage_parsing.divide_into_blocks(code) - - try: - import sage.repl - # CRITICAL -- we do NOT import sage.repl.interpreter!!!!!!! - # That would waste several seconds importing ipython and much more, which is just dumb. - # The only reason this is needed below is if the user has run preparser(False), which - # would cause sage.repl.interpreter to be imported at that point (as preparser is - # lazy imported.) - sage_repl_interpreter = sage.repl.interpreter - except: - pass # expected behavior usually, since sage.repl.interpreter usually not imported (only used by command line...) - - import sage.misc.session - for start, stop, block in blocks: - # if import sage.repl.interpreter fails, sag_repl_interpreter is unreferenced - try: - do_pp = getattr(sage_repl_interpreter, '_do_preparse', True) - except: - do_pp = True - if preparse and do_pp: - block = sage_parsing.preparse_code(block) - sys.stdout.reset() - sys.stderr.reset() - try: - b = block.rstrip() - # get rid of comments at the end of the line -- issue #1835 - #from ushlex import shlex - #s = shlex(b) - #s.commenters = '#' - #s.quotes = '"\'' - #b = ''.join(s) - # e.g. now a line like 'x = test? # bar' becomes 'x=test?' - if b.endswith('??'): - p = sage_parsing.introspect(b, - namespace=namespace, - preparse=False) - self.code(source=p['result'], mode="python") - elif b.endswith('?'): - p = sage_parsing.introspect(b, - namespace=namespace, - preparse=False) - self.code(source=p['result'], mode="text/x-rst") - else: - reload_attached_files_if_mod_smc() - if execute.count < 2: - execute.count += 1 - if execute.count == 2: - # this fixup has to happen after first block has executed (os.chdir etc) - # but before user assigns any variable in worksheet - # sage.misc.session.init() is not called until first call of show_identifiers - # BUGFIX: be careful to *NOT* assign to _!! see https://github.com/sagemathinc/cocalc/issues/1107 - block2 = "sage.misc.session.state_at_init = dict(globals());sage.misc.session._dummy=sage.misc.session.show_identifiers();\n" - exec(compile(block2, '', 'single'), namespace, - locals) - b2a = """ -if 'SAGE_STARTUP_FILE' in os.environ and os.path.isfile(os.environ['SAGE_STARTUP_FILE']): - try: - load(os.environ['SAGE_STARTUP_FILE']) - except: - sys.stdout.flush() - sys.stderr.write('\\nException loading startup file: {}\\n'.format(os.environ['SAGE_STARTUP_FILE'])) - sys.stderr.flush() - raise -""" - exec(compile(b2a, '', 'exec'), namespace, locals) - features = sage_parsing.get_future_features( - block, 'single') - if features: - compile_flags = reduce( - operator.or_, (feature.compiler_flag - for feature in features.values()), - compile_flags) - exec( - compile(block + '\n', - '', - 'single', - flags=compile_flags), namespace, locals) - if features: - Salvus._py_features.update(features) - sys.stdout.flush() - sys.stderr.flush() - except: - if ascii_warn: - sys.stderr.write( - '\n\n*** WARNING: Code contains non-ascii characters ***\n' - ) - for c in '\u201c\u201d': - if c in code: - sys.stderr.write( - '*** Maybe the character < %s > should be replaced by < " > ? ***\n' - % c) - break - sys.stderr.write('\n\n') - - if six.PY2: - from exceptions import SyntaxError, TypeError - # py3: all standard errors are available by default via "builtin", not available here for some reason ... - if six.PY3: - from builtins import SyntaxError, TypeError - - exc_type, _, _ = sys.exc_info() - if exc_type in [SyntaxError, TypeError]: - from .sage_parsing import strip_string_literals - code0, _, _ = strip_string_literals(code) - implicit_mul = RE_POSSIBLE_IMPLICIT_MUL.findall(code0) - if len(implicit_mul) > 0: - implicit_mul_list = ', '.join( - str(_) for _ in implicit_mul) - # we know there is a SyntaxError and there could be an implicit multiplication - sys.stderr.write( - '\n\n*** WARNING: Code contains possible implicit multiplication ***\n' - ) - sys.stderr.write( - '*** Check if any of [ %s ] need a "*" sign for multiplication, e.g. 5x should be 5*x ! ***\n\n' - % implicit_mul_list) - - sys.stdout.flush() - sys.stderr.write('Error in lines %s-%s\n' % - (start + 1, stop + 1)) - traceback.print_exc() - sys.stderr.flush() - break - - def execute_with_code_decorators(self, - code_decorators, - code, - preparse=True, - namespace=None, - locals=None): - """ - salvus.execute_with_code_decorators is used when evaluating - code blocks that are set to any non-default code_decorator. - """ - import sage # used below as a code decorator - if is_string(code_decorators): - code_decorators = [code_decorators] - - if preparse: - code_decorators = list( - map(sage_parsing.preparse_code, code_decorators)) - - code_decorators = [ - eval(code_decorator, self.namespace) - for code_decorator in code_decorators - ] - - # The code itself may want to know exactly what code decorators are in effect. - # For example, r.eval can do extra things when being used as a decorator. - self.code_decorators = code_decorators - - for i, code_decorator in enumerate(code_decorators): - # eval is for backward compatibility - if not hasattr(code_decorator, 'eval') and hasattr( - code_decorator, 'before'): - code_decorators[i] = code_decorator.before(code) - - for code_decorator in reversed(code_decorators): - # eval is for backward compatibility - if hasattr(code_decorator, 'eval'): - print(code_decorator.eval( - code, locals=self.namespace)) # removed , end=' ' - code = '' - elif code_decorator is sage: - # special case -- the sage module (i.e., %sage) should do nothing. - pass - else: - code = code_decorator(code) - if code is None: - code = '' - - if code != '' and is_string(code): - self.execute(code, - preparse=preparse, - namespace=namespace, - locals=locals) - - for code_decorator in code_decorators: - if not hasattr(code_decorator, 'eval') and hasattr( - code_decorator, 'after'): - code_decorator.after(code) - - def html(self, html, done=False, once=None): - """ - Display html in the output stream. - - EXAMPLE: - - salvus.html("Hi") - """ - self._flush_stdio() - self._send_output(html=unicode8(html), - id=self._id, - done=done, - once=once) - - def md(self, md, done=False, once=None): - """ - Display markdown in the output stream. - - EXAMPLE: - - salvus.md("**Hi**") - """ - self._flush_stdio() - self._send_output(md=unicode8(md), id=self._id, done=done, once=once) - - def pdf(self, filename, **kwds): - sage_salvus.show_pdf(filename, **kwds) - - def tex(self, obj, display=False, done=False, once=None, **kwds): - """ - Display obj nicely using TeX rendering. - - INPUT: - - - obj -- latex string or object that is automatically be converted to TeX - - display -- (default: False); if True, typeset as display math (so centered, etc.) - """ - self._flush_stdio() - tex = obj if is_string(obj) else self.namespace['latex'](obj, **kwds) - self._send_output(tex={ - 'tex': tex, - 'display': display - }, - id=self._id, - done=done, - once=once) - return self - - def start_executing(self): - self._send_output(done=False, id=self._id) - - def clear(self, done=False): - self._send_output(clear=True, id=self._id, done=done) - - def delete_last_output(self, done=False): - self._send_output(delete_last=True, id=self._id, done=done) - - def stdout(self, output, done=False, once=None): - """ - Send the string output (or unicode8(output) if output is not a - string) to the standard output stream of the compute cell. - - INPUT: - - - output -- string or object - - """ - stdout = output if is_string(output) else unicode8(output) - self._send_output(stdout=stdout, done=done, id=self._id, once=once) - return self - - def stderr(self, output, done=False, once=None): - """ - Send the string output (or unicode8(output) if output is not a - string) to the standard error stream of the compute cell. - - INPUT: - - - output -- string or object - - """ - stderr = output if is_string(output) else unicode8(output) - self._send_output(stderr=stderr, done=done, id=self._id, once=once) - return self - - def code( - self, - source, # actual source code - mode=None, # the syntax highlight codemirror mode - filename=None, # path of file it is contained in (if applicable) - lineno=-1, # line number where source starts (0-based) - done=False, - once=None): - """ - Send a code message, which is to be rendered as code by the client, with - appropriate syntax highlighting, maybe a link to open the source file, etc. - """ - source = source if is_string(source) else unicode8(source) - code = { - 'source': source, - 'filename': filename, - 'lineno': int(lineno), - 'mode': mode - } - self._send_output(code=code, done=done, id=self._id, once=once) - return self - - def _execute_interact(self, id, vals): - if id not in sage_salvus.interacts: - print("(Evaluate this cell to use this interact.)") - #raise RuntimeError("Error: No interact with id %s"%id) - else: - sage_salvus.interacts[id](vals) - - def interact(self, f, done=False, once=None, **kwds): - I = sage_salvus.InteractCell(f, **kwds) - self._flush_stdio() - self._send_output(interact=I.jsonable(), - id=self._id, - done=done, - once=once) - return sage_salvus.InteractFunction(I) - - def javascript(self, - code, - once=False, - coffeescript=False, - done=False, - obj=None): - """ - Execute the given Javascript code as part of the output - stream. This same code will be executed (at exactly this - point in the output stream) every time the worksheet is - rendered. - - See the docs for the top-level javascript function for more details. - - INPUT: - - - code -- a string - - once -- boolean (default: FAlse); if True the Javascript is - only executed once, not every time the cell is loaded. This - is what you would use if you call salvus.stdout, etc. Use - once=False, e.g., if you are using javascript to make a DOM - element draggable (say). WARNING: If once=True, then the - javascript is likely to get executed before other output to - a given cell is even rendered. - - coffeescript -- boolean (default: False); if True, the input - code is first converted from CoffeeScript to Javascript. - - At least the following Javascript objects are defined in the - scope in which the code is evaluated:: - - - cell -- jQuery wrapper around the current compute cell - - salvus.stdout, salvus.stderr, salvus.html, salvus.tex -- all - allow you to write additional output to the cell - - worksheet - jQuery wrapper around the current worksheet DOM object - - obj -- the optional obj argument, which is passed via JSON serialization - """ - if obj is None: - obj = {} - self._send_output(javascript={ - 'code': code, - 'coffeescript': coffeescript - }, - id=self._id, - done=done, - obj=obj, - once=once) - - def coffeescript(self, *args, **kwds): - """ - This is the same as salvus.javascript, but with coffeescript=True. - - See the docs for the top-level javascript function for more details. - """ - kwds['coffeescript'] = True - self.javascript(*args, **kwds) - - def raw_input(self, - prompt='', - default='', - placeholder='', - input_width=None, - label_width=None, - done=False, - type=None): # done is ignored here - self._flush_stdio() - m = {'prompt': unicode8(prompt)} - if input_width is not None: - m['input_width'] = unicode8(input_width) - if label_width is not None: - m['label_width'] = unicode8(label_width) - if default: - m['value'] = unicode8(default) - if placeholder: - m['placeholder'] = unicode8(placeholder) - self._send_output(raw_input=m, id=self._id) - typ, mesg = self.message_queue.next_mesg() - log("handling raw input message ", truncate_text(unicode8(mesg), 400)) - if typ == 'json' and mesg['event'] == 'sage_raw_input': - # everything worked out perfectly - self.delete_last_output() - m['value'] = mesg['value'] # as unicode! - m['submitted'] = True - self._send_output(raw_input=m, id=self._id) - value = mesg['value'] - if type is not None: - if type == 'sage': - value = sage_salvus.sage_eval(value) - else: - try: - value = type(value) - except TypeError: - # Some things in Sage are clueless about unicode for some reason... - # Let's at least try, in case the unicode can convert to a string. - value = type(str(value)) - return value - else: - raise KeyboardInterrupt( - "raw_input interrupted by another action: event='%s' (expected 'sage_raw_input')" - % mesg['event']) - - def _check_component(self, component): - if component not in ['input', 'output']: - raise ValueError("component must be 'input' or 'output'") - - def hide(self, component): - """ - Hide the given component ('input' or 'output') of the cell. - """ - self._check_component(component) - self._send_output(self._id, hide=component) - - def show(self, component): - """ - Show the given component ('input' or 'output') of the cell. - """ - self._check_component(component) - self._send_output(self._id, show=component) - - def notify(self, **kwds): - """ - Display a graphical notification using the alert_message Javascript function. - - INPUTS: - - - `type: "default"` - Type of the notice. "default", "warning", "info", "success", or "error". - - `title: ""` - The notice's title. - - `message: ""` - The notice's text. - - `timeout: ?` - Delay in seconds before the notice is automatically removed. - - EXAMPLE: - - salvus.notify(type="warning", title="This warning", message="This is a quick message.", timeout=3) - """ - obj = {} - for k, v in kwds.items(): - if k == 'text': # backward compat - k = 'message' - elif k == 'type' and v == 'notice': # backward compat - v = 'default' - obj[k] = sage_salvus.jsonable(v) - if k == 'delay': # backward compat - obj['timeout'] = v / 1000.0 # units are in seconds now. - - self.javascript("alert_message(obj)", once=True, obj=obj) - - def execute_javascript(self, code, coffeescript=False, obj=None): - """ - Tell the browser to execute javascript. Basically the same as - salvus.javascript with once=True (the default), except this - isn't tied to a particular cell. There is a worksheet object - defined in the scope of the evaluation. - - See the docs for the top-level javascript function for more details. - """ - self._conn.send_json( - message.execute_javascript(code, - coffeescript=coffeescript, - obj=json.dumps(obj, - separators=(',', ':')))) - - def execute_coffeescript(self, *args, **kwds): - """ - This is the same as salvus.execute_javascript, but with coffeescript=True. - - See the docs for the top-level javascript function for more details. - """ - kwds['coffeescript'] = True - self.execute_javascript(*args, **kwds) - - def _cython(self, filename, **opts): - """ - Return module obtained by compiling the Cython code in the - given file. - - INPUT: - - - filename -- name of a Cython file - - all other options are passed to sage.misc.cython.cython unchanged, - except for use_cache which defaults to True (instead of False) - - OUTPUT: - - - a module - """ - if 'use_cache' not in opts: - opts['use_cache'] = True - import sage.misc.cython - modname, path = sage.misc.cython.cython(filename, **opts) - try: - sys.path.insert(0, path) - module = __import__(modname) - finally: - del sys.path[0] - return module - - def _import_code(self, content, **opts): - while True: - py_file_base = uuid().replace('-', '_') - if not os.path.exists(py_file_base + '.py'): - break - try: - open(py_file_base + '.py', 'w').write(content) - try: - sys.path.insert(0, os.path.abspath('.')) - mod = __import__(py_file_base) - finally: - del sys.path[0] - finally: - os.unlink(py_file_base + '.py') - os.unlink(py_file_base + '.pyc') - return mod - - def _sage(self, filename, **opts): - import sage.misc.preparser - content = "from sage.all import *\n" + sage.misc.preparser.preparse_file( - open(filename).read()) - return self._import_code(content, **opts) - - def _spy(self, filename, **opts): - import sage.misc.preparser - content = "from sage.all import Integer, RealNumber, PolynomialRing\n" + sage.misc.preparser.preparse_file( - open(filename).read()) - return self._import_code(content, **opts) - - def _py(self, filename, **opts): - return __import__(filename) - - def require(self, filename, **opts): - if not os.path.exists(filename): - raise ValueError("file '%s' must exist" % filename) - base, ext = os.path.splitext(filename) - if ext == '.pyx' or ext == '.spyx': - return self._cython(filename, **opts) - if ext == ".sage": - return self._sage(filename, **opts) - if ext == ".spy": - return self._spy(filename, **opts) - if ext == ".py": - return self._py(filename, **opts) - raise NotImplementedError("require file of type %s not implemented" % - ext) - - def typeset_mode(self, on=True): - sage_salvus.typeset_mode(on) - - def project_info(self): - """ - Return a dictionary with information about the project in which this code is running. - - EXAMPLES:: - - sage: salvus.project_info() - {"stdout":"{u'project_id': u'...', u'location': {u'username': u'teaAuZ9M', u'path': u'.', u'host': u'localhost', u'port': 22}, u'base_url': u'/...'}\n"} - """ - return INFO - - -if six.PY2: - Salvus.pdf.__func__.__doc__ = sage_salvus.show_pdf.__doc__ - Salvus.raw_input.__func__.__doc__ = sage_salvus.raw_input.__doc__ - Salvus.clear.__func__.__doc__ = sage_salvus.clear.__doc__ - Salvus.delete_last_output.__func__.__doc__ = sage_salvus.delete_last_output.__doc__ -else: - Salvus.pdf.__doc__ = sage_salvus.show_pdf.__doc__ - Salvus.raw_input.__doc__ = sage_salvus.raw_input.__doc__ - Salvus.clear.__doc__ = sage_salvus.clear.__doc__ - Salvus.delete_last_output.__doc__ = sage_salvus.delete_last_output.__doc__ - - -def execute(conn, id, code, data, cell_id, preparse, message_queue): - - salvus = Salvus(conn=conn, - id=id, - data=data, - message_queue=message_queue, - cell_id=cell_id) - - #salvus.start_executing() # with our new mainly client-side execution this isn't needed; not doing this makes evaluation roundtrip around 100ms instead of 200ms too, which is a major win. - - try: - # initialize the salvus output streams - streams = (sys.stdout, sys.stderr) - sys.stdout = BufferedOutputStream(salvus.stdout) - sys.stderr = BufferedOutputStream(salvus.stderr) - try: - # initialize more salvus functionality - sage_salvus.set_salvus(salvus) - namespace['sage_salvus'] = sage_salvus - except: - traceback.print_exc() - - if salvus._prefix: - if not code.startswith("%"): - code = salvus._prefix + '\n' + code - - if salvus._postfix: - code += '\n' + salvus._postfix - - salvus.execute(code, namespace=namespace, preparse=preparse) - - finally: - # there must be exactly one done message, unless salvus._done is False. - if sys.stderr._buf: - if sys.stdout._buf: - sys.stdout.flush() - sys.stderr.flush(done=salvus._done) - else: - sys.stdout.flush(done=salvus._done) - (sys.stdout, sys.stderr) = streams - - -# execute.count goes from 0 to 2 -# used for show_identifiers() -execute.count = 0 - - -def drop_privileges(id, home, transient, username): - gid = id - uid = id - if transient: - os.chown(home, uid, gid) - os.setgid(gid) - os.setuid(uid) - os.environ['DOT_SAGE'] = home - mpl = os.environ['MPLCONFIGDIR'] - os.environ['MPLCONFIGDIR'] = home + mpl[5:] - os.environ['HOME'] = home - os.environ['IPYTHON_DIR'] = home - os.environ['USERNAME'] = username - os.environ['USER'] = username - os.chdir(home) - - # Monkey patch the Sage library and anything else that does not - # deal well with changing user. This sucks, but it is work that - # simply must be done because we're not importing the library from - # scratch (which would take a long time). - import sage.misc.misc - sage.misc.misc.DOT_SAGE = home + '/.sage/' - - -class MessageQueue(list): - - def __init__(self, conn): - self.queue = [] - self.conn = conn - - def __repr__(self): - return "Sage Server Message Queue" - - def __getitem__(self, i): - return self.queue[i] - - def __delitem__(self, i): - del self.queue[i] - - def next_mesg(self): - """ - Remove oldest message from the queue and return it. - If the queue is empty, wait for a message to arrive - and return it (does not place it in the queue). - """ - if self.queue: - return self.queue.pop() - else: - return self.conn.recv() - - def recv(self): - """ - Wait until one message is received and enqueue it. - Also returns the mesg. - """ - mesg = self.conn.recv() - self.queue.insert(0, mesg) - return mesg - - -def session(conn): - """ - This is run by the child process that is forked off on each new - connection. It drops privileges, then handles the complete - compute session. - - INPUT: - - - ``conn`` -- the TCP connection - """ - mq = MessageQueue(conn) - - pid = os.getpid() - - # seed the random number generator(s) - import sage.all - sage.all.set_random_seed() - import random - random.seed(sage.all.initial_seed()) - - cnt = 0 - while True: - try: - typ, mesg = mq.next_mesg() - - #print('INFO:child%s: received message "%s"'%(pid, mesg)) - log("handling message ", truncate_text(unicode8(mesg), 400)) - event = mesg['event'] - if event == 'terminate_session': - return - elif event == 'execute_code': - try: - execute(conn=conn, - id=mesg['id'], - code=mesg['code'], - data=mesg.get('data', None), - cell_id=mesg.get('cell_id', None), - preparse=mesg.get('preparse', True), - message_queue=mq) - except Exception as err: - log("ERROR -- exception raised '%s' when executing '%s'" % - (err, mesg['code'])) - elif event == 'introspect': - try: - # check for introspect from jupyter cell - prefix = Salvus._default_mode - if 'top' in mesg: - top = mesg['top'] - log('introspect cell top line %s' % top) - if top.startswith("%"): - prefix = top[1:] - try: - # see if prefix is the name of a jupyter kernel function - kc = eval(prefix + "(get_kernel_client=True)", - namespace, locals()) - kn = eval(prefix + "(get_kernel_name=True)", namespace, - locals()) - log("jupyter introspect prefix %s kernel %s" % - (prefix, kn)) # e.g. "p2", "python2" - jupyter_introspect(conn=conn, - id=mesg['id'], - line=mesg['line'], - preparse=mesg.get('preparse', True), - kc=kc) - except: - import traceback - exc_type, exc_value, exc_traceback = sys.exc_info() - lines = traceback.format_exception( - exc_type, exc_value, exc_traceback) - log(lines) - introspect(conn=conn, - id=mesg['id'], - line=mesg['line'], - preparse=mesg.get('preparse', True)) - except: - pass - else: - raise RuntimeError("invalid message '%s'" % mesg) - except: - # When hub connection dies, loop goes crazy. - # Unfortunately, just catching SIGINT doesn't seem to - # work, and leads to random exits during a - # session. Howeer, when connection dies, 10000 iterations - # happen almost instantly. Ugly, but it works. - cnt += 1 - if cnt > 10000: - sys.exit(0) - else: - pass - - -def jupyter_introspect(conn, id, line, preparse, kc): - import jupyter_client - from queue import Empty - - try: - salvus = Salvus(conn=conn, id=id) - msg_id = kc.complete(line) - shell = kc.shell_channel - iopub = kc.iopub_channel - - # handle iopub responses - while True: - try: - msg = iopub.get_msg(timeout=1) - msg_type = msg['msg_type'] - content = msg['content'] - - except Empty: - # shouldn't happen - log("jupyter iopub channel empty") - break - - if msg['parent_header'].get('msg_id') != msg_id: - continue - - log("jupyter iopub recv %s %s" % (msg_type, str(content))) - - if msg_type == 'status' and content['execution_state'] == 'idle': - break - - # handle shell responses - while True: - try: - msg = shell.get_msg(timeout=10) - msg_type = msg['msg_type'] - content = msg['content'] - - except: - # shouldn't happen - log("jupyter shell channel empty") - break - - if msg['parent_header'].get('msg_id') != msg_id: - continue - - log("jupyter shell recv %s %s" % (msg_type, str(content))) - - if msg_type == 'complete_reply' and content['status'] == 'ok': - # jupyter kernel returns matches like "xyz.append" and smc wants just "append" - matches = content['matches'] - offset = content['cursor_end'] - content['cursor_start'] - completions = [s[offset:] for s in matches] - mesg = message.introspect_completions(id=id, - completions=completions, - target=line[-offset:]) - conn.send_json(mesg) - break - except: - log("jupyter completion exception: %s" % sys.exc_info()[0]) - - -def introspect(conn, id, line, preparse): - salvus = Salvus( - conn=conn, id=id - ) # so salvus.[tab] works -- note that Salvus(...) modifies namespace. - z = sage_parsing.introspect(line, namespace=namespace, preparse=preparse) - if z['get_completions']: - mesg = message.introspect_completions(id=id, - completions=z['result'], - target=z['target']) - elif z['get_help']: - mesg = message.introspect_docstring(id=id, - docstring=z['result'], - target=z['expr']) - elif z['get_source']: - mesg = message.introspect_source_code(id=id, - source_code=z['result'], - target=z['expr']) - conn.send_json(mesg) - - -def handle_session_term(signum, frame): - while True: - try: - pid, exit_status = os.waitpid(-1, os.WNOHANG) - except: - return - if not pid: return - - -secret_token = None - -if 'COCALC_SECRET_TOKEN' in os.environ: - secret_token_path = os.environ['COCALC_SECRET_TOKEN'] -else: - secret_token_path = os.path.join(os.environ['SMC'], 'secret_token') - - -def unlock_conn(conn): - global secret_token - if secret_token is None: - try: - secret_token = open(secret_token_path).read().strip() - except: - conn.send(six.b('n')) - conn.send( - six. - b("Unable to accept connection, since Sage server doesn't yet know the secret token; unable to read from '%s'" - % secret_token_path)) - conn.close() - - n = len(secret_token) - token = six.b('') - while len(token) < n: - token += conn.recv(n) - if token != secret_token[:len(token)]: - break # definitely not right -- don't try anymore - if token != six.b(secret_token): - log("token='%s'; secret_token='%s'" % (token, secret_token)) - conn.send(six.b('n')) # no -- invalid login - conn.send(six.b("Invalid secret token.")) - conn.close() - return False - else: - conn.send(six.b('y')) # yes -- valid login - return True - - -def serve_connection(conn): - global PID - PID = os.getpid() - # First the client *must* send the secret shared token. If they - # don't, we return (and the connection will have been destroyed by - # unlock_conn). - log("Serving a connection") - log("Waiting for client to unlock the connection...") - # TODO -- put in a timeout (?) - if not unlock_conn(conn): - log("Client failed to unlock connection. Dumping them.") - return - log("Connection unlocked.") - - try: - conn = ConnectionJSON(conn) - typ, mesg = conn.recv() - log("Received message %s" % mesg) - except Exception as err: - log("Error receiving message: %s (connection terminated)" % str(err)) - raise - - if mesg['event'] == 'send_signal': - if mesg['pid'] == 0: - log("invalid signal mesg (pid=0)") - else: - log("Sending a signal") - os.kill(mesg['pid'], mesg['signal']) - return - if mesg['event'] != 'start_session': - log("Received an unknown message event = %s; terminating session." % - mesg['event']) - return - - log("Starting a session") - desc = message.session_description(os.getpid()) - log("child sending session description back: %s" % desc) - conn.send_json(desc) - session(conn=conn) - - -def serve(port, host, extra_imports=False): - #log.info('opening connection on port %s', port) - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - - # check for children that have finished every few seconds, so - # we don't end up with zombies. - s.settimeout(5) - - s.bind((host, port)) - log('Sage server %s:%s' % (host, port)) - - # Enabling the following signal completely breaks subprocess pexpect in many cases, which is - # obviously totally unacceptable. - #signal.signal(signal.SIGCHLD, handle_session_term) - - def init_library(): - tm = time.time() - log("pre-importing the sage library...") - - # FOR testing purposes. - ##log("fake 40 second pause to slow things down for testing....") - ##time.sleep(40) - ##log("done with pause") - - # Actually import sage now. This must happen after the interact - # import because of library interacts. - log("import sage...") - import sage.all - log("imported sage.") - - # Monkey patching interact using the new and improved Salvus - # implementation of interact. - sage.all.interact = sage_salvus.interact - - # Monkey patch the html command. - try: - # need the following for sage_server to start with sage-8.0 - # or `import sage.interacts.library` will fail (not really important below, as we don't do that). - import sage.repl.user_globals - sage.repl.user_globals.set_globals(globals()) - log("initialized user_globals") - except RuntimeError: - # may happen with sage version < 8.0 - log("user_globals.set_globals failed, continuing", sys.exc_info()) - - sage.all.html = sage.misc.html.html = sage_salvus.html - - # CRITICAL: look, we are just going to not do this, and have sage.interacts.library - # be broken. It's **really slow** to do this, and I don't think sage.interacts.library - # ever ended up going anywhere! People use wiki.sagemath.org/interact instead... - #import sage.interacts.library - #sage.interacts.library.html = sage_salvus.html - - # Set a useful figsize default; the matplotlib one is not notebook friendly. - import sage.plot.graphics - sage.plot.graphics.Graphics.SHOW_OPTIONS['figsize'] = [8, 4] - - # Monkey patch latex.eval, so that %latex works in worksheets - sage.misc.latex.latex.eval = sage_salvus.latex0 - - # Plot, integrate, etc., -- so startup time of worksheets is minimal. - cmds = [ - 'from sage.all import *', 'from sage.calculus.predefined import x', - 'import pylab' - ] - if extra_imports: - cmds.extend([ - 'import scipy', 'import sympy', - "plot(sin).save('%s/a.png'%os.environ['SMC'], figsize=2)", - 'integrate(sin(x**2),x)' - ]) - tm0 = time.time() - for cmd in cmds: - log(cmd) - exec(cmd, namespace) - - global pylab - pylab = namespace['pylab'] # used for clearing - - log('imported sage library and other components in %s seconds' % - (time.time() - tm)) - - for k, v in sage_salvus.interact_functions.items(): - namespace[k] = v - # See above -- not doing this, since it is REALLY SLOW to import. - # This does mean that some old code that tries to use interact might break (?). - #namespace[k] = sagenb.notebook.interact.__dict__[k] = v - - namespace['_salvus_parsing'] = sage_parsing - - for name in [ - 'anaconda', 'asy', 'attach', 'auto', 'capture', 'cell', - 'clear', 'coffeescript', 'cython', 'default_mode', - 'delete_last_output', 'dynamic', 'exercise', 'fork', 'fortran', - 'go', 'help', 'hide', 'hideall', 'input', 'java', 'javascript', - 'julia', 'jupyter', 'license', 'load', 'md', 'mediawiki', - 'modes', 'octave', 'pandoc', 'perl', 'plot3d_using_matplotlib', - 'prun', 'python_future_feature', 'py3print_mode', 'python', - 'python3', 'r', 'raw_input', 'reset', 'restore', 'ruby', - 'runfile', 'sage_eval', 'scala', 'scala211', 'script', - 'search_doc', 'search_src', 'sh', 'show', 'show_identifiers', - 'singular_kernel', 'time', 'timeit', 'typeset_mode', 'var', - 'wiki' - ]: - namespace[name] = getattr(sage_salvus, name) - - namespace['sage_server'] = sys.modules[ - __name__] # http://stackoverflow.com/questions/1676835/python-how-do-i-get-a-reference-to-a-module-inside-the-module-itself - - # alias pretty_print_default to typeset_mode, since sagenb has/uses that. - namespace['pretty_print_default'] = namespace['typeset_mode'] - # and monkey patch it - sage.misc.latex.pretty_print_default = namespace[ - 'pretty_print_default'] - - sage_salvus.default_namespace = dict(namespace) - log("setup namespace with extra functions") - - # Sage's pretty_print and view are both ancient and a mess - sage.all.pretty_print = sage.misc.latex.pretty_print = namespace[ - 'pretty_print'] = namespace['view'] = namespace['show'] - - # this way client code can tell it is running as a Sage Worksheet. - namespace['__SAGEWS__'] = True - - log("Initialize sage library.") - init_library() - - t = time.time() - s.listen(128) - i = 0 - - children = {} - log("Starting server listening for connections") - try: - while True: - i += 1 - #print i, time.time()-t, 'cps: ', int(i/(time.time()-t)) - # do not use log.info(...) in the server loop; threads = race conditions that hang server every so often!! - try: - if children: - for pid in list(children.keys()): - if os.waitpid(pid, os.WNOHANG) != (0, 0): - log("subprocess %s terminated, closing connection" - % pid) - conn.close() - del children[pid] - - try: - conn, addr = s.accept() - log("Accepted a connection from", addr) - except: - # this will happen periodically since we did s.settimeout above, so - # that we wait for children above periodically. - continue - except socket.error: - continue - child_pid = os.fork() - if child_pid: # parent - log("forked off child with pid %s to handle this connection" % - child_pid) - children[child_pid] = conn - else: - # child - global PID - PID = os.getpid() - log("child process, will now serve this new connection") - serve_connection(conn) - - # end while - except Exception as err: - log("Error taking connection: ", err) - traceback.print_exc(file=open(LOGFILE, 'a')) - #log.error("error: %s %s", type(err), str(err)) - - finally: - log("closing socket") - #s.shutdown(0) - s.close() - - -def run_server(port, host, pidfile, logfile=None): - global LOGFILE - if logfile: - LOGFILE = logfile - if pidfile: - pid = str(os.getpid()) - print("os.getpid() = %s" % pid) - open(pidfile, 'w').write(pid) - log("run_server: port=%s, host=%s, pidfile='%s', logfile='%s'" % - (port, host, pidfile, LOGFILE)) - try: - serve(port, host) - finally: - if pidfile: - os.unlink(pidfile) - - -if __name__ == "__main__": - import argparse - parser = argparse.ArgumentParser(description="Run Sage server") - parser.add_argument( - "-p", - dest="port", - type=int, - default=0, - help= - "port to listen on (default: 0); 0 = automatically allocated; saved to $SMC/data/sage_server.port" - ) - parser.add_argument( - "-l", - dest='log_level', - type=str, - default='INFO', - help= - "log level (default: INFO) useful options include WARNING and DEBUG") - parser.add_argument("-d", - dest="daemon", - default=False, - action="store_const", - const=True, - help="daemon mode (default: False)") - parser.add_argument( - "--host", - dest="host", - type=str, - default='127.0.0.1', - help="host interface to bind to -- default is 127.0.0.1") - parser.add_argument("--pidfile", - dest="pidfile", - type=str, - default='', - help="store pid in this file") - parser.add_argument( - "--logfile", - dest="logfile", - type=str, - default='', - help="store log in this file (default: '' = don't log to a file)") - parser.add_argument("-c", - dest="client", - default=False, - action="store_const", - const=True, - help="run in test client mode number 1 (command line)") - parser.add_argument("--hostname", - dest="hostname", - type=str, - default='', - help="hostname to connect to in client mode") - parser.add_argument("--portfile", - dest="portfile", - type=str, - default='', - help="write port to this file") - - args = parser.parse_args() - - if args.daemon and not args.pidfile: - print(("%s: must specify pidfile in daemon mode" % sys.argv[0])) - sys.exit(1) - - if args.log_level: - pass - #level = getattr(logging, args.log_level.upper()) - #log.setLevel(level) - - if args.client: - client1( - port=args.port if args.port else int(open(args.portfile).read()), - hostname=args.hostname) - sys.exit(0) - - if not args.port: - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.bind(('', 0)) # pick a free port - args.port = s.getsockname()[1] - del s - - if args.portfile: - open(args.portfile, 'w').write(str(args.port)) - - pidfile = os.path.abspath(args.pidfile) if args.pidfile else '' - logfile = os.path.abspath(args.logfile) if args.logfile else '' - if logfile: - LOGFILE = logfile - open(LOGFILE, 'w') # for now we clear it on restart... - log("setting logfile to %s" % LOGFILE) - - main = lambda: run_server(port=args.port, host=args.host, pidfile=pidfile) - if args.daemon and args.pidfile: - from . import daemon - daemon.daemonize(args.pidfile) - main() - else: - main() diff --git a/src/smc_sagews/smc_sagews/sage_server_command_line.py b/src/smc_sagews/smc_sagews/sage_server_command_line.py deleted file mode 100644 index 47b4f2fe3e..0000000000 --- a/src/smc_sagews/smc_sagews/sage_server_command_line.py +++ /dev/null @@ -1,83 +0,0 @@ -# A very simple interface exposed from setup.py -from __future__ import absolute_import -import os, socket, sys -import time - - -def log(s): - sys.stderr.write('sage_server: %s\n' % s) - sys.stderr.flush() - - -def main(action='', daemon=True): - SMC = os.environ['SMC'] - PATH = os.path.join(SMC, 'sage_server') - if not os.path.exists(PATH): - os.makedirs(PATH) - file = os.path.join(PATH, 'sage_server.') - - pidfile = file + 'pid' - portfile = file + 'port' - logfile = file + 'log' - - if action == '': - if len(sys.argv) <= 1: - action = '' - else: - action = sys.argv[1] - - def start(): - log("starting...") - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.bind(('', 0)) # pick a free port - port = s.getsockname()[1] - del s - log("port=%s" % port) - open(portfile, 'w').write(str(port)) - open(logfile, 'w') # for now we clear it on restart... - log("setting logfile to %s" % logfile) - - t0 = time.time() - from . import sage_server - log("seconds to import sage_server: %s" % (time.time() - t0)) - run_server = lambda: sage_server.run_server(port=port, host='127.0.0.1', pidfile=pidfile, logfile=logfile) - if daemon: - log("daemonizing") - from .daemon import daemonize - daemonize(pidfile) - run_server() - else: - log("starting in foreground") - run_server() - - def stop(): - log("stopping...") - if os.path.exists(pidfile): - try: - pid = int(open(pidfile).read()) - sid = os.getsid(pid) - log("killing sid %s" % sid) - os.killpg(sid, 9) - log("successfully killed") - except Exception as e: - log("failed -- %s" % e) - log("removing '%s'" % pidfile) - os.unlink(pidfile) - else: - log("no pidfile") - - def usage(): - print(("Usage: %s [start|stop|restart]" % sys.argv[0])) - - if action == 'start': - start() - elif action == 'stop': - stop() - elif action == 'restart': - try: - stop() - except: - pass - start() - else: - usage() diff --git a/src/smc_sagews/smc_sagews/test_direct/README.md b/src/smc_sagews/smc_sagews/test_direct/README.md deleted file mode 100644 index 51fbaa5d95..0000000000 --- a/src/smc_sagews/smc_sagews/test_direct/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# Unit test sagews features directly - -(without sage_server) - -## Goals - -- test components of sage worksheets -- avoid overhead and extraneous issues raised when testing through sage_server - -## How to Use These Tests - -### Running the Tests - -``` -cd cocalc/src/smc_sagews/smc_sagews/test_direct - -# test jupyter_client launch of all non-sage kernels -python -m pytest - -# test selected kernels -python -m pytest --kname=anaconda3,singular -``` - -### Test Results - -Test results will be stored in machine-readable file `~/sagews-direct-test-report.json`: - -Example: - -``` -{ - "start": "2017-06-06 14:40:43.066034", - "end": "2017-06-06 14:41:29.976735", - "version": 1, - "name": "sagews_direct.test", - "fields": [ - "name", - "outcome", - "duration" - ], - "results": [ - [ - "start_new_kernel[anaconda3]", - "passed", - 2.9162349700927734 - ], - ... -``` - -and `~/sagews-direct-test-report.prom` for ingestion by Prometheus' node exporter. - diff --git a/src/smc_sagews/smc_sagews/test_direct/conftest.py b/src/smc_sagews/smc_sagews/test_direct/conftest.py deleted file mode 100644 index 7152372ed3..0000000000 --- a/src/smc_sagews/smc_sagews/test_direct/conftest.py +++ /dev/null @@ -1,100 +0,0 @@ -from __future__ import absolute_import - - -def pytest_addoption(parser): - """specify comma-delimited list of kernel names to limit test""" - parser.addoption("--kname", action="store", help="kernel name") - - -def pytest_generate_tests(metafunc): - """default is to test all non-sage kernels listed with `jupyter kernelspec list`""" - option_value = metafunc.config.option.kname - if 'kname' in metafunc.fixturenames and option_value is not None: - knames = option_value.split(',') - metafunc.parametrize("kname", knames) - # nsk = list of available non-sage kernel names - # skip first line of command output, "Available kernels" - else: - v = [ - x.strip() for x in os.popen("jupyter kernelspec list").readlines() - ] - nsk = [x.split()[0] for x in v[1:] if not x.startswith('sage')] - metafunc.parametrize("kname", nsk) - - -# -# Write machine-readable report files into the $HOME directory -# http://doc.pytest.org/en/latest/example/simple.html#post-process-test-reports-failures -# -import os -import json -import pytest -import time -from datetime import datetime - -report_json = os.path.expanduser('~/sagews-direct-test-report.json') -report_prom = os.path.expanduser('~/sagews-direct-test-report.prom') -results = [] -start_time = None - - -@pytest.hookimpl -def pytest_configure(config): - global start_time - start_time = datetime.utcnow() - - -@pytest.hookimpl -def pytest_unconfigure(config): - global start_time - - def append_file(f1, f2): - with open(f1, 'a') as outf1: - with open(f2, 'r') as inf2: - outf1.write(inf2.read()) - - data = { - 'name': 'sagews_direct.test', - 'version': 1, - 'start': str(start_time), - 'end': str(datetime.utcnow()), - 'fields': ['name', 'outcome', 'duration'], - 'results': results, - } - report_json_tmp = report_json + '~' - with open(report_json, 'w') as out: - json.dump(data, out, indent=1) - # this is a plain text prometheus report - # https://prometheus.io/docs/instrumenting/exposition_formats/#text-format-details - # timestamp milliseconds since epoch - ts = int(1000 * time.mktime(start_time.timetuple())) - # first write to temp file ... - report_prom_tmp = report_prom + '~' - with open(report_prom_tmp, 'w') as prom: - for (name, outcome, duration) in results: - labels = 'name="{name}",outcome="{outcome}"'.format(**locals()) - line = 'sagews_direct_test{{{labels}}} {duration} {ts}'.format( - **locals()) - prom.write(line + '\n') - # ... then atomically overwrite the real one - os.rename(report_prom_tmp, report_prom) - - -@pytest.hookimpl(tryfirst=True, hookwrapper=True) -def pytest_runtest_makereport(item, call): - # execute all other hooks to obtain the report object - outcome = yield - rep = outcome.get_result() - - if rep.when != "call": - return - - #import pdb; pdb.set_trace() # uncomment to inspect item and rep objects - # the following `res` should match the `fields` above - # parent: item.parent.name could be interesting, but just () for auto discovery - name = item.name - test_ = 'test_' - if name.startswith(test_): - name = name[len(test_):] - res = [name, rep.outcome, rep.duration] - results.append(res) diff --git a/src/smc_sagews/smc_sagews/test_direct/test_jupyter.py b/src/smc_sagews/smc_sagews/test_direct/test_jupyter.py deleted file mode 100644 index 6123eb71d9..0000000000 --- a/src/smc_sagews/smc_sagews/test_direct/test_jupyter.py +++ /dev/null @@ -1,15 +0,0 @@ -from __future__ import absolute_import -import jupyter_client - - -def test_start_new_kernel(kname): - """launch jupyter kernel using same interface as sagews jupyter bridge""" - try: - km, kc = jupyter_client.manager.start_new_kernel(kernel_name=kname, - startup_timeout=10) - assert km is not None - assert kc is not None - print(("kernel {} started successfully".format(kname))) - km.shutdown_kernel() - except: - assert 0, "kernel {} failed to start".format(kname) diff --git a/src/smc_sagews/smc_sagews/tests/README.md b/src/smc_sagews/smc_sagews/tests/README.md deleted file mode 100644 index 29fc8ecd87..0000000000 --- a/src/smc_sagews/smc_sagews/tests/README.md +++ /dev/null @@ -1,85 +0,0 @@ -# Pytest suite for sage_server - -## Goals - -- verify that sage worksheets return correct results for given cell inputs -- back-end only test, uses message protocol to run tests against running sage_server - -## Non-goals - -- unit tests of functions internal to sagews modules -- UI testing - -## How to Use These Tests - -### Prerequisites - -- pytest must be installed -- file ~/.smc/sage_server/sage_server.log must exist and have the - current port number around line 3, like this: - ``` - 9240 (2016-09-07 04:03:39.455): Sage server 127.0.0.1:43359 - ``` - -### Running the Tests - ->= 9.0 - -[hsy] I'm running `sage -python -m pytest tests/` from within the smc_sagews directory. - ---- - -pre 9.0 - -Use `runtests.sh` to set up environment for doctests, otherwise they will be skipped. - -``` -cd ~/cocalc/src/smc_sagews/smc_sagews/tests -./runtests.sh -``` - -to skip doctests: - -``` -cd ~/cocalc/src/smc_sagews/smc_sagews/tests -python -m pytest -``` - -### Test Results - -The test results will be stored in a machine-readable json file in `$HOME/sagews-test-report.json`: - -``` -cat ~/sagews-test-report.json -``` - -Example: - -``` -{ - "start": "2016-12-15 12:50:09.620189", - "version": 1, - "end": "2016-12-15 12:53:00.064441", - "name": "smc_sagews.test", - "fields": [ - "name", - "outcome", - "duration" - ], - "results": [ - [ - "basic_timing", - "passed", - 1.0065569877624512 - ], - ... - ] - } -``` - -and `$HOME/sagews-test-report.prom` for ingestion by Prometheus' node exporter. - -## Test Layout - -These tests follow the 'inline' test layout documented at pytest docs [Choosing a test layout / import rules](http://doc.pytest.org/en/latest/goodpractices.html#choosing-a-test-layout-import-rules). - diff --git a/src/smc_sagews/smc_sagews/tests/a.html b/src/smc_sagews/smc_sagews/tests/a.html deleted file mode 100644 index 405135f606..0000000000 --- a/src/smc_sagews/smc_sagews/tests/a.html +++ /dev/null @@ -1,8 +0,0 @@ - - - - TESTING 1 - diff --git a/src/smc_sagews/smc_sagews/tests/a.py b/src/smc_sagews/smc_sagews/tests/a.py deleted file mode 100644 index 78a20f56a6..0000000000 --- a/src/smc_sagews/smc_sagews/tests/a.py +++ /dev/null @@ -1,2 +0,0 @@ -def f2(*args, **kwargs): - print("test f2 1") diff --git a/src/smc_sagews/smc_sagews/tests/a.sage b/src/smc_sagews/smc_sagews/tests/a.sage deleted file mode 100644 index 4a2329e60e..0000000000 --- a/src/smc_sagews/smc_sagews/tests/a.sage +++ /dev/null @@ -1,7 +0,0 @@ -def f1(arg, *args, **kwargs): - print 'f1 arg = %r'%arg - for count,v in enumerate(args): - print 'f1 *args',count,v - for k,v in kwargs.items(): - print 'f1 **kwargs',k,v - print "test f1 1" diff --git a/src/smc_sagews/smc_sagews/tests/conftest.py b/src/smc_sagews/smc_sagews/tests/conftest.py deleted file mode 100644 index 9f64be66ea..0000000000 --- a/src/smc_sagews/smc_sagews/tests/conftest.py +++ /dev/null @@ -1,882 +0,0 @@ -from __future__ import absolute_import -import pytest -import os -import re -import socket -import json -import signal -import struct -import hashlib -import time -import six -from datetime import datetime - -# timeout for socket to sage_server in seconds -default_timeout = 20 - -### -# much of the code here is copied from sage_server.py -# cut and paste was done because it takes over 30 sec to import sage_server -# and requires the script to be run from sage -sh -### - - -def unicode8(s): - try: - if six.PY3: - return str(s, 'utf8') - else: - return str(s).encode('utf-8') - except: - try: - return str(s) - except: - return s - - -PID = os.getpid() - - -def log(*args): - mesg = "%s (%s): %s\n" % (PID, datetime.utcnow().strftime( - '%Y-%m-%d %H:%M:%S.%f')[:-3], ' '.join([unicode8(x) for x in args])) - print(mesg) - - -def uuidsha1(data): - sha1sum = hashlib.sha1() - sha1sum.update(data) - s = sha1sum.hexdigest() - t = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx' - r = list(t) - j = 0 - for i in range(len(t)): - if t[i] == 'x': - r[i] = s[j] - j += 1 - elif t[i] == 'y': - # take 8 + low order 3 bits of hex number. - r[i] = hex((int(s[j], 16) & 0x3) | 0x8)[-1] - j += 1 - return ''.join(r) - - -class ConnectionJSON(object): - def __init__(self, conn): - # avoid common mistake -- conn is supposed to be from socket.socket... - assert not isinstance(conn, ConnectionJSON) - self._conn = conn - - def close(self): - self._conn.close() - - def _send(self, s): - if six.PY3 and type(s) == str: - s = s.encode('utf8') - length_header = struct.pack(">L", len(s)) - self._conn.send(length_header + s) - - def send_json(self, m): - m = json.dumps(m) - #log(u"sending message '", truncate_text(m, 256), u"'") - self._send('j' + m) - return len(m) - - def send_blob(self, blob): - s = uuidsha1(blob) - - if six.PY3 and type(blob) == bytes: - # we convert all to bytes first, to avoid unnecessary conversions - self._send(('b' + s).encode('utf8') + blob) - else: - # old sage py2 code - self._send('b' + s + blob) - - return s - - def send_file(self, filename): - #log("sending file '%s'"%filename) - f = open(filename, 'rb') - data = f.read() - f.close() - return self.send_blob(data) - - def _recv(self, n): - # see http://stackoverflow.com/questions/3016369/catching-blocking-sigint-during-system-call - for i in range(20): - try: - r = self._conn.recv(n) - return r - except socket.error as exc: - if isinstance(exc, socket.timeout): - raise - else: - (errno, msg) = exc - if errno != 4: - raise - raise EOFError - - def recv(self): - n = self._recv(4) - if len(n) < 4: - print(("expecting 4 byte header, got", n)) - tries = 0 - while tries < 5: - tries += 1 - n2 = self._recv(4 - len(n)) - n += n2 - if len(n) >= 4: - break - else: - raise EOFError - n = struct.unpack('>L', n)[0] # big endian 32 bits - #print("test got header, expect message of length %s"%n) - s = self._recv(n) - while len(s) < n: - t = self._recv(n - len(s)) - if len(t) == 0: - raise EOFError - s += t - - if six.PY3 and type(s) == bytes: - s = s.decode('utf8') - - mtyp = s[0] - if mtyp == 'j': - try: - return 'json', json.loads(s[1:]) - except Exception as msg: - log("Unable to parse JSON '%s'" % s[1:]) - raise - - elif mtyp == 'b': - return 'blob', s[1:] - raise ValueError("unknown message type '%s'" % s[0]) - - def set_timeout(self, timeout): - "set socket timeout in seconds" - self._conn.settimeout(timeout) - - -def truncate_text(s, max_size): - if len(s) > max_size: - return s[:max_size] + "[...]", True - else: - return s, False - - -class Message(object): - def _new(self, event, props={}): - m = {'event': event} - for key, val in list(props.items()): - if key != 'self': - m[key] = val - return m - - def start_session(self): - return self._new('start_session') - - def session_description(self, pid): - return self._new('session_description', {'pid': pid}) - - def send_signal(self, pid, signal=signal.SIGINT): - return self._new('send_signal', locals()) - - def terminate_session(self, done=True): - return self._new('terminate_session', locals()) - - def execute_code(self, id, code, preparse=True): - return self._new('execute_code', locals()) - - def execute_javascript(self, code, obj=None, coffeescript=False): - return self._new('execute_javascript', locals()) - - def output( - self, - id, - stdout=None, - stderr=None, - code=None, - html=None, - javascript=None, - coffeescript=None, - interact=None, - md=None, - tex=None, - d3=None, - file=None, - raw_input=None, - obj=None, - once=None, - hide=None, - show=None, - events=None, - clear=None, - delete_last=None, - done=False # CRITICAL: done must be specified for multi-response; this is assumed by sage_session.coffee; otherwise response assumed single. - ): - m = self._new('output') - m['id'] = id - t = truncate_text_warn - did_truncate = False - import sage_server # we do this so that the user can customize the MAX's below. - if code is not None: - code['source'], did_truncate, tmsg = t(code['source'], - sage_server.MAX_CODE_SIZE, - 'MAX_CODE_SIZE') - m['code'] = code - if stderr is not None and len(stderr) > 0: - m['stderr'], did_truncate, tmsg = t(stderr, - sage_server.MAX_STDERR_SIZE, - 'MAX_STDERR_SIZE') - if stdout is not None and len(stdout) > 0: - m['stdout'], did_truncate, tmsg = t(stdout, - sage_server.MAX_STDOUT_SIZE, - 'MAX_STDOUT_SIZE') - if html is not None and len(html) > 0: - m['html'], did_truncate, tmsg = t(html, sage_server.MAX_HTML_SIZE, - 'MAX_HTML_SIZE') - if md is not None and len(md) > 0: - m['md'], did_truncate, tmsg = t(md, sage_server.MAX_MD_SIZE, - 'MAX_MD_SIZE') - if tex is not None and len(tex) > 0: - tex['tex'], did_truncate, tmsg = t(tex['tex'], - sage_server.MAX_TEX_SIZE, - 'MAX_TEX_SIZE') - m['tex'] = tex - if javascript is not None: m['javascript'] = javascript - if coffeescript is not None: m['coffeescript'] = coffeescript - if interact is not None: m['interact'] = interact - if d3 is not None: m['d3'] = d3 - if obj is not None: m['obj'] = json.dumps(obj) - if file is not None: m['file'] = file # = {'filename':..., 'uuid':...} - if raw_input is not None: m['raw_input'] = raw_input - if done is not None: m['done'] = done - if once is not None: m['once'] = once - if hide is not None: m['hide'] = hide - if show is not None: m['show'] = show - if events is not None: m['events'] = events - if clear is not None: m['clear'] = clear - if delete_last is not None: m['delete_last'] = delete_last - if did_truncate: - if 'stderr' in m: - m['stderr'] += '\n' + tmsg - else: - m['stderr'] = '\n' + tmsg - return m - - def introspect_completions(self, id, completions, target): - m = self._new('introspect_completions', locals()) - m['id'] = id - return m - - def introspect_docstring(self, id, docstring, target): - m = self._new('introspect_docstring', locals()) - m['id'] = id - return m - - def introspect_source_code(self, id, source_code, target): - m = self._new('introspect_source_code', locals()) - m['id'] = id - return m - - # NOTE: these functions are NOT in sage_server.py - def save_blob(self, sha1): - return self._new('save_blob', {'sha1': sha1}) - - def introspect(self, id, line, top): - return self._new('introspect', {'id': id, 'line': line, 'top': top}) - - -message = Message() - -### -# end of copy region -### - - -def set_salvus_path(self, id): - r""" - create json message to set path and file at start of virtual worksheet - """ - m = self._new('execute_code', locals()) - - -# hard code SMC for now so we don't have to run with sage wrapper -SMC = os.path.join(os.environ["HOME"], ".smc") -default_log_file = os.path.join(SMC, "sage_server", "sage_server.log") -default_pid_file = os.path.join(SMC, "sage_server", "sage_server.pid") - - -def get_sage_server_info(log_file=default_log_file): - for loop_count in range(3): - # log file ~/.smc/sage_server/sage_server.log - # sample sage_server startup line in first lines of log: - # 3136 (2016-08-18 15:02:49.372): Sage server 127.0.0.1:44483 - try: - with open(log_file, "r") as inf: - for lno in range(5): - line = inf.readline().strip() - m = re.search( - "Sage server (?P[\w.]+):(?P\d+)$", line) - if m: - host = m.group('host') - port = int(m.group('port')) - #return host, int(port) - break - else: - raise ValueError('Server info not found in log_file', - log_file) - break - except IOError: - print("starting new sage_server") - os.system(start_cmd()) - time.sleep(5.0) - else: - pytest.fail( - "Unable to open log file %s\nThere is probably no sage server running. You either have to open a sage worksheet or run smc-sage-server start" - % log_file) - print(("got host %s port %s" % (host, port))) - return host, int(port) - - -secret_token = None -secret_token_path = os.path.join(os.environ['SMC'], 'secret_token') - -if 'COCALC_SECRET_TOKEN' in os.environ: - secret_token_path = os.environ['COCALC_SECRET_TOKEN'] -else: - secret_token_path = os.path.join(os.environ['SMC'], 'secret_token') - - -def client_unlock_connection(sock): - secret_token = open(secret_token_path).read().strip() - sock.sendall(secret_token.encode()) - - -def path_info(): - file = __file__ - full_path = os.path.abspath(file) - head, tail = os.path.split(full_path) - #file = head + "/testing.sagews" - return head, file - - -def recv_til_done(conn, test_id): - r""" - Discard json messages from server for current test_id until 'done' is True - or limit is reached. Used in finalizer for single cell tests. - """ - for loop_count in range(5): - typ, mesg = conn.recv() - assert typ == 'json' - assert mesg['id'] == test_id - assert 'done' in mesg - if mesg['done']: - break - else: - pytest.fail("too many responses for message id %s" % test_id) - - -def my_sage_startup(): - """ - name of pytest SAGE_STARTUP_FILE - used in other test files so we export it - """ - return "a-init.sage" - - -def start_cmd(action='start'): - """ - launch sage-server with env setting for startup file - - - `` action `` -- string "start" | "restart" - - """ - pssf = os.path.join(os.path.dirname(__file__), my_sage_startup()) - cmd = "export SAGE_STARTUP_FILE={};smc-sage-server {}".format(pssf, action) - return cmd - - -### -# Start of fixtures -### - - -@pytest.fixture(autouse=True, scope="session") -def sage_server_setup(pid_file=default_pid_file, log_file=default_log_file): - r""" - make sure sage_server pid file exists and process running at given pid - """ - os.system(start_cmd('restart')) - for loop_count in range(20): - time.sleep(0.5) - if not os.path.exists(log_file): - continue - lmsg = "Starting server listening for connections" - if lmsg in open(log_file).read(): - break - else: - pytest.fail("Unable to start sage_server and setup log file") - return - - -@pytest.fixture() -def test_id(request): - r""" - Return increasing sequence of integers starting at 1. This number is used as - test id as well as message 'id' value so sage_server log can be matched - with pytest output. - """ - test_id.id += 1 - return test_id.id - - -test_id.id = 1 - - -# see http://doc.pytest.org/en/latest/tmpdir.html#the-tmpdir-factory-fixture -@pytest.fixture(scope='session') -def image_file(tmpdir_factory): - def make_img(): - import matplotlib - matplotlib.use('Agg') - import matplotlib.pyplot as plt - my_circle = plt.Circle((0.5, 0.5), 0.2) - fig, ax = plt.subplots() - ax.add_artist(my_circle) - return fig - - fn = tmpdir_factory.mktemp('data').join('my_circle.png') - make_img().savefig(str(fn)) - return fn - - -@pytest.fixture(scope='session') -def data_path(tmpdir_factory): - path = tmpdir_factory.mktemp("data") - path.ensure_dir() - return path - - -@pytest.fixture() -def execdoc(request, sagews, test_id): - r""" - Fixture function execdoc. Depends on two other fixtures, sagews and test_id. - - EXAMPLES: - - :: - - def test_assg(execdoc): - execdoc("random?") - """ - def execfn(code, pattern='Docstring'): - m = message.execute_code(code=code, id=test_id) - sagews.send_json(m) - typ, mesg = sagews.recv() - assert typ == 'json' - assert mesg['id'] == test_id - assert 'code' in mesg - assert 'source' in mesg['code'] - assert re.sub('\s+', '', pattern) in re.sub('\s+', '', - mesg['code']['source']) - - def fin(): - recv_til_done(sagews, test_id) - - request.addfinalizer(fin) - return execfn - - -@pytest.fixture() -def exec2(request, sagews, test_id): - r""" - Fixture function exec2. Depends on two other fixtures, sagews and test_id. - If output & patterns are omitted, the cell is not expected to produce a - stdout result. All arguments after 'code' are optional. - - If argument `timeout` is provided, the default socket timeout - for connection to sage_server will be overridden to the value of `timeout` in seconds. - - - `` code `` -- string of code to run - - - `` output `` -- string or list of strings of output to be matched up to leading & trailing whitespace - - - `` pattern `` -- regex to match with expected stdout output - - - `` html_pattern `` -- regex to match with expected html output - - - `` timeout `` -- socket timeout in seconds - - - `` errout `` -- stderr substring to be matched. stderr may come as several messages - - EXAMPLES: - - :: - - def test_assg(exec2): - code = "x = 42\nx\n" - output = "42\n" - exec2(code, output) - - :: - - def test_set_file_env(exec2): - code = "os.chdir(salvus.data[\'path\']);__file__=salvus.data[\'file\']" - exec2(code) - - :: - - def test_sh(exec2): - exec2("sh('date +%Y-%m-%d')", pattern = '^\d{4}-\d{2}-\d{2}$') - - .. NOTE:: - - If `output` is a list of strings, `pattern` and `html_pattern` are ignored - - """ - def execfn(code, - output=None, - pattern=None, - html_pattern=None, - timeout=default_timeout, - errout=None): - m = message.execute_code(code=code, id=test_id) - m['preparse'] = True - - if timeout is not None: - print(('overriding socket timeout to {}'.format(timeout))) - sagews.set_timeout(timeout) - - # send block of code to be executed - sagews.send_json(m) - - # check stdout - if isinstance(output, list): - for o in output: - typ, mesg = sagews.recv() - assert typ == 'json' - assert mesg['id'] == test_id - assert 'stdout' in mesg - assert o.strip() in (mesg['stdout']).strip() - elif output or pattern: - typ, mesg = sagews.recv() - assert typ == 'json' - assert mesg['id'] == test_id - assert 'stdout' in mesg - mout = mesg['stdout'] - if output is not None: - assert output.strip() in mout - elif pattern is not None: - assert re.search(pattern, mout) is not None - elif html_pattern: - typ, mesg = sagews.recv() - assert typ == 'json' - assert mesg['id'] == test_id - assert 'html' in mesg - assert re.search(html_pattern, mesg['html']) is not None - elif errout: - mout = "" - while True: - typ, mesg = sagews.recv() - assert typ == 'json' - assert mesg['id'] == test_id - assert 'stderr' in mesg - mout += mesg['stderr'] - if errout.strip() in mout: - break - - def fin(): - recv_til_done(sagews, test_id) - - request.addfinalizer(fin) - return execfn - - -@pytest.fixture() -def execbuf(request, sagews, test_id): - r""" - Fixture function execbuf. - Inner function will execute code, then append messages received - from sage_server. - As messages are appended, the result is checked for either - an exact match, if `output` string is specified, or - pattern match, if `pattern` string is given. - Test fails if non-`stdout` message is received before - match or receive times out. - """ - def execfn(code, output=None, pattern=None): - m = message.execute_code(code=code, id=test_id) - m['preparse'] = True - # send block of code to be executed - sagews.send_json(m) - outbuf = '' - while True: - typ, mesg = sagews.recv() - assert typ == 'json' - assert mesg['id'] == test_id - assert 'stdout' in mesg - outbuf += mesg['stdout'] - if output is not None: - if output in outbuf: - break - elif pattern is not None: - if re.search(pattern, outbuf) is not None: - break - - def fin(): - recv_til_done(sagews, test_id) - - request.addfinalizer(fin) - return execfn - - -@pytest.fixture() -def execinteract(request, sagews, test_id): - def execfn(code): - m = message.execute_code(code=code, id=test_id) - m['preparse'] = True - sagews.send_json(m) - typ, mesg = sagews.recv() - assert typ == 'json' - assert mesg['id'] == test_id - assert 'interact' in mesg - - def fin(): - recv_til_done(sagews, test_id) - - request.addfinalizer(fin) - return execfn - - -@pytest.fixture() -def execblob(request, sagews, test_id): - def execblobfn(code, - want_html=True, - want_javascript=False, - file_type='png', - ignore_stdout=False): - r""" - fixture when test generates an image - - INPUT: - - - ``file_type`` -- string or list of strings, e.g. ["svg", "png"] - - """ - - SHA_LEN = 36 - - # format and send the plot command - m = message.execute_code(code=code, id=test_id) - sagews.send_json(m) - - # expect several responses before "done", but order may vary - want_blob = True - want_name = True - while any([want_blob, want_name, want_html, want_javascript]): - typ, mesg = sagews.recv() - if typ == 'blob': - assert want_blob - want_blob = False - # when a blob is sent, the first 36 bytes are the sha1 uuid - print(("blob len %s" % len(mesg))) - file_uuid = mesg[:SHA_LEN].decode() - assert file_uuid == uuidsha1(mesg[SHA_LEN:]) - - # sage_server expects an ack with the right uuid - m = message.save_blob(sha1=file_uuid) - sagews.send_json(m) - else: - assert typ == 'json' - if 'html' in mesg: - assert want_html - want_html = False - print('got html') - elif 'javascript' in mesg: - assert want_javascript - want_javascript = False - print('got javascript') - elif ignore_stdout and 'stdout' in mesg: - pass - else: - assert want_name - want_name = False - assert 'file' in mesg - print('got file name') - if isinstance(file_type, str): - assert file_type in mesg['file']['filename'] - elif isinstance(file_type, list): - assert any([(f0 in mesg['file']['filename']) for f0 in file_type]), \ - "missing one of file types {} in response from sage_server".format(file_type) - else: - assert 0 - - # final response is json "done" message - typ, mesg = sagews.recv() - assert typ == 'json' - assert mesg['done'] == True - - return execblobfn - - -@pytest.fixture() -def execintrospect(request, sagews, test_id): - def execfn(line, completions, target, top=None): - if top is None: - top = line - m = message.introspect(test_id, line=line, top=top) - m['preparse'] = True - sagews.send_json(m) - typ, mesg = sagews.recv() - assert typ == 'json' - assert mesg['id'] == test_id - assert mesg['event'] == "introspect_completions" - assert mesg['completions'] == completions - assert mesg['target'] == target - - return execfn - - -@pytest.fixture(scope="class") -def sagews(request): - r""" - Module-scoped fixture for tests that don't leave - extra threads running. - """ - # setup connection to sage_server TCP listener - host, port = get_sage_server_info() - print(("host %s port %s" % (host, port))) - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect((host, port)) - sock.settimeout(default_timeout) - print("connected to socket") - - # unlock - client_unlock_connection(sock) - print("socket unlocked") - conn = ConnectionJSON(sock) - c_ack = conn._recv(1).decode() - assert c_ack == 'y', "expect ack for token, got %s" % c_ack - - # open connection with sage_server and run tests - msg = message.start_session() - msg['type'] = 'sage' - conn.send_json(msg) - print("start_session sent") - typ, mesg = conn.recv() - assert typ == 'json' - pid = mesg['pid'] - print(("sage_server PID = %s" % pid)) - - # teardown needed - terminate session nicely - # use yield instead of request.addfinalizer in newer versions of pytest - def fin(): - print("\nExiting Sage client.") - conn.send_json(message.terminate_session()) - # wait several seconds for client to die - for loop_count in range(8): - try: - os.kill(pid, 0) - except OSError: - # client is dead - break - time.sleep(0.5) - else: - print(("sending sigterm to %s" % pid)) - try: - os.kill(pid, signal.SIGTERM) - except OSError: - pass - - request.addfinalizer(fin) - return conn - - -@pytest.fixture(scope="class") -def own_sage_server(request): - assert os.geteuid() != 0, "Do not run as root, will kill all sage_servers." - print("starting sage_server class fixture") - os.system(start_cmd()) - time.sleep(0.5) - - def fin(): - print("killing all sage_server processes") - os.system("pkill -f sage_server_command_line") - - request.addfinalizer(fin) - - -@pytest.fixture(scope="class") -def test_ro_data_dir(request): - """ - Return the directory containing the test file. - Used for tests which have read-only data files in the test dir. - """ - return os.path.dirname(request.module.__file__) - - -# -# Write machine-readable report files into the $HOME directory -# http://doc.pytest.org/en/latest/example/simple.html#post-process-test-reports-failures -# -import os -report_json = os.path.expanduser('~/sagews-test-report.json') -report_prom = os.path.expanduser('~/sagews-test-report.prom') -results = [] -start_time = None - - -@pytest.hookimpl -def pytest_configure(config): - global start_time - start_time = datetime.utcnow() - - -@pytest.hookimpl -def pytest_unconfigure(config): - global start_time - data = { - 'name': 'smc_sagews.test', - 'version': 1, - 'start': str(start_time), - 'end': str(datetime.utcnow()), - 'fields': ['name', 'outcome', 'duration'], - 'results': results, - } - with open(report_json, 'w') as out: - json.dump(data, out, indent=1) - # this is a plain text prometheus report - # https://prometheus.io/docs/instrumenting/exposition_formats/#text-format-details - # timestamp milliseconds since epoch - ts = int(1000 * time.mktime(start_time.timetuple())) - # first write to temp file ... - report_prom_tmp = report_prom + '~' - with open(report_prom_tmp, 'w') as prom: - for (name, outcome, duration) in results: - labels = 'name="{name}",outcome="{outcome}"'.format(**locals()) - line = 'sagews_test{{{labels}}} {duration} {ts}'.format(**locals()) - prom.write(line + '\n') - # ... then atomically overwrite the real one - os.rename(report_prom_tmp, report_prom) - - -@pytest.hookimpl(tryfirst=True, hookwrapper=True) -def pytest_runtest_makereport(item, call): - # execute all other hooks to obtain the report object - outcome = yield - rep = outcome.get_result() - - if rep.when != "call": - return - - #import pdb; pdb.set_trace() # uncomment to inspect item and rep objects - # the following `res` should match the `fields` above - # parent: item.parent.name could be interesting, but just () for auto discovery - name = item.name - test_ = 'test_' - if name.startswith(test_): - name = name[len(test_):] - res = [name, rep.outcome, rep.duration] - results.append(res) diff --git a/src/smc_sagews/smc_sagews/tests/runtests.sh b/src/smc_sagews/smc_sagews/tests/runtests.sh deleted file mode 100755 index 148136fa9f..0000000000 --- a/src/smc_sagews/smc_sagews/tests/runtests.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -export SAGE_ROOT=${SAGE_ROOT:-${EXT:-/ext}/sage/sage} -. $SAGE_ROOT/src/bin/sage-env -python -m pytest $@ diff --git a/src/smc_sagews/smc_sagews/tests/sage_init_files/define_var.sage b/src/smc_sagews/smc_sagews/tests/sage_init_files/define_var.sage deleted file mode 100755 index f5882d14ca..0000000000 --- a/src/smc_sagews/smc_sagews/tests/sage_init_files/define_var.sage +++ /dev/null @@ -1,2 +0,0 @@ -xyzzy = 42 - diff --git a/src/smc_sagews/smc_sagews/tests/sage_init_files/runtime_err.sage b/src/smc_sagews/smc_sagews/tests/sage_init_files/runtime_err.sage deleted file mode 100644 index b6f8c5f2da..0000000000 --- a/src/smc_sagews/smc_sagews/tests/sage_init_files/runtime_err.sage +++ /dev/null @@ -1 +0,0 @@ -1/0 diff --git a/src/smc_sagews/smc_sagews/tests/sage_init_files/syntax_err.sage b/src/smc_sagews/smc_sagews/tests/sage_init_files/syntax_err.sage deleted file mode 100644 index c856c006c4..0000000000 --- a/src/smc_sagews/smc_sagews/tests/sage_init_files/syntax_err.sage +++ /dev/null @@ -1 +0,0 @@ -3+ diff --git a/src/smc_sagews/smc_sagews/tests/test_00_timing.py b/src/smc_sagews/smc_sagews/tests/test_00_timing.py deleted file mode 100644 index 691c132a73..0000000000 --- a/src/smc_sagews/smc_sagews/tests/test_00_timing.py +++ /dev/null @@ -1,151 +0,0 @@ -# test_sagews_timing.py -# tests of sage worksheet that measure test duration - -# at present I don't see a pytest api feature for this -# other than --duration flag which is experimental and -# for profiling only -from __future__ import absolute_import -import pytest -import socket -import conftest -import os -import time -import signal - - -class TestSageTiming: - r""" - These tests are to validate the test framework. They do not - run sage_server. - """ - def test_basic_timing(self): - start = time.time() - result = os.system('sleep 1') - assert result == 0 - tick = time.time() - elapsed = tick - start - assert 1.0 == pytest.approx(elapsed, abs=0.2) - - def test_load_sage(self): - start = time.time() - # maybe put first load into fixture - result = os.system("echo '2+2' | sage -python") - assert result == 0 - tick = time.time() - elapsed = tick - start - print(("elapsed 1: %s" % elapsed)) - # second load after things are cached - start = time.time() - result = os.system("echo '2+2' | sage -python") - assert result == 0 - tick = time.time() - elapsed = tick - start - print(("elapsed 2: %s" % elapsed)) - assert elapsed < 4.0 - - def test_import_sage_server(self): - start = time.time() - # sage 9.0: previously, this was setting the path to import from the global /cocalc/... location - # now, it's using the code here - code = ';'.join([ - "import sys", - "sys.path.insert(0, '.')", - "import sage_server" - ]) - result = os.system("echo \"{}\" | sage -python".format(code)) - assert result == 0 - tick = time.time() - elapsed = tick - start - print(("elapsed %s" % elapsed)) - assert elapsed < 20.0 - - -class TestStartSageServer: - def test_2plus2_timing(self, test_id): - import sys - - # if sage_server is running, stop it - os.system("smc-sage-server stop") - - # start the clock - start = time.time() - - # start a new sage_server process - os.system(conftest.start_cmd()) - print(("sage_server start time %s sec" % (time.time() - start))) - # add pause here because sometimes the log file isn't ready immediately - time.sleep(0.5) - - # setup connection to sage_server TCP listener - host, port = conftest.get_sage_server_info() - print(("host %s port %s" % (host, port))) - - # multiple tries at connecting because there's a delay between - # writing the port number and listening on the socket for connections - for attempt in range(10): - attempt += 1 - print(("attempt %s" % attempt)) - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect((host, port)) - break - except: - print((sys.exc_info()[0])) - pass - time.sleep(0.5) - else: - pytest.fail("Could not connect to sage_server at port %s" % port) - print("connected to socket") - - # unlock - conftest.client_unlock_connection(sock) - print("socket unlocked") - conn = conftest.ConnectionJSON(sock) - c_ack = conn._recv(1) - assert c_ack == b'y', "expect ack for token, got %s" % c_ack - - # start session - msg = conftest.message.start_session() - msg['type'] = 'sage' - conn.send_json(msg) - print("start_session sent") - typ, mesg = conn.recv() - assert typ == 'json' - pid = mesg['pid'] - print(("sage_server PID = %s" % pid)) - - code = "2+2\n" - output = "4\n" - - m = conftest.message.execute_code(code=code, id=test_id) - m['preparse'] = True - - # send block of code to be executed - conn.send_json(m) - - # check stdout - typ, mesg = conn.recv() - assert typ == 'json' - assert mesg['id'] == test_id - assert mesg['stdout'] == output - elapsed = time.time() - start - - # teardown connection - conn.send_json(conftest.message.terminate_session()) - print("\nExiting Sage client.") - # wait 3 sec for process to die, then kill it - for loop_count in range(6): - try: - os.kill(pid, 0) - except OSError: - pass - time.sleep(0.5) - else: - print(("sending sigterm to %s" % pid)) - os.kill(pid, signal.SIGTERM) - - # check timing - print(("elapsed 2+2 %s" % elapsed)) - assert elapsed < 25.0 - - return diff --git a/src/smc_sagews/smc_sagews/tests/test_doctests.py b/src/smc_sagews/smc_sagews/tests/test_doctests.py deleted file mode 100644 index fc08a4747d..0000000000 --- a/src/smc_sagews/smc_sagews/tests/test_doctests.py +++ /dev/null @@ -1,89 +0,0 @@ -# run doctests for selected sagemath source files in sagews server -from __future__ import absolute_import - -import socket -import conftest -import os -import re - -from textwrap import dedent - -import pytest -import future -import doctest - -# doctests need sagemath env settings -# skip this entire module if not set -dtvar = 'SAGE_LOCAL' -if not dtvar in os.environ: - pytest.skip("skipping doctests, {} not defined".format(dtvar), - allow_module_level=True) - - -@pytest.mark.parametrize( - "src_file", - [('/ext/sage/sage/local/lib/python2.7/site-packages/sage/symbolic/units.py' - ), - ('/ext/sage/sage/local/lib/python2.7/site-packages/sage/misc/flatten.py'), - ('/ext/sage/sage/local/lib/python2.7/site-packages/sage/misc/banner.py')]) -class TestDT: - def test_dt_file(self, test_id, sagews, src_file): - print(("src_file=", src_file)) - import sys - - from sage.doctest.sources import FileDocTestSource - from sage.doctest.control import DocTestDefaults - - FDS = FileDocTestSource(src_file, DocTestDefaults()) - doctests, extras = FDS.create_doctests(globals()) - id = test_id - excount = 0 - dtn = 0 - print(("{} doctests".format(len(doctests)))) - for dt in doctests: - print(("doctest number", dtn)) - dtn += 1 - exs = dt.examples - excount += len(exs) - for ex in exs: - c = ex.sage_source - print(("code", c)) - w = ex.want - print(("want", w)) - use_pattern = False - # handle ellipsis in wanted output - if '...' in w: - use_pattern = True - # special case for bad "want" value at end of banner() - wf = w.find('"help()" for help') - if wf > 0: - w = w[:wf] + '...' - m = conftest.message.execute_code(code=c, id=id) - sagews.send_json(m) - - if len(w) > 0: - typ, mesg = sagews.recv() - assert typ == 'json' - assert mesg['id'] == id - if 'stdout' in mesg: - #assert 'stdout' in mesg - output = mesg['stdout'] - print(("outp", output)) - elif 'stderr' in mesg: - output = mesg['stderr'] - # bypass err line number reporting in CoCalc - if w.startswith('Traceback'): - otf = output.find('Traceback') - if otf > 0: - output = output[otf:] - print(("outp", output)) - else: - assert 0 - if use_pattern: - assert doctest._ellipsis_match(w, output) - else: - assert output.strip() == w.strip() - conftest.recv_til_done(sagews, id) - id += 1 - print(("{} examples".format(excount))) - conftest.test_id.id = id diff --git a/src/smc_sagews/smc_sagews/tests/test_env.py b/src/smc_sagews/smc_sagews/tests/test_env.py deleted file mode 100644 index 75142e173b..0000000000 --- a/src/smc_sagews/smc_sagews/tests/test_env.py +++ /dev/null @@ -1,78 +0,0 @@ -# -*- coding: utf-8 -*- -# test_env.py -# tests of sage worksheet environment options -from __future__ import absolute_import -import conftest -import os -import errno - - -def remove_no_exc(fname): - try: - os.remove(fname) - except OSError as e: - if e.errno != errno.ENOENT: - raise - - -# NOTE: Tests in this file will not work when run individually. -# Because they require processing different versions of SAGE_STARTUP_FILE, -# each test sets up the next one before it ends. -class TestSetEnv: - def test_set_startup(self, exec2, test_ro_data_dir): - """ - verify that SAGE_STARTUP_FILE is set by pytest to a-init.sage - """ - pssf = os.path.join(test_ro_data_dir, conftest.my_sage_startup()) - exec2("print(os.environ['SAGE_STARTUP_FILE'])", pssf) - remove_no_exc(pssf) - - -class TestNoStartupFile: - def test_init_sage(self, exec2, test_ro_data_dir): - exec2("pi", pattern="pi") - pssf = os.path.join(test_ro_data_dir, conftest.my_sage_startup()) - pssf2 = os.path.join(test_ro_data_dir, - "sage_init_files/define_var.sage") - os.symlink(pssf2, pssf) - - -class TestGoodStartupFile: - def test_ident_set_file_env(self, exec2): - """emulate initial code block sent from UI, needed for first show_identifiers""" - code = "os.chdir(salvus.data[\'path\']);__file__=salvus.data[\'file\']" - exec2(code) - - def test_init_sage(self, exec2, test_ro_data_dir): - """check for variable defined in startup file""" - exec2("show_identifiers()", "xyzzy") - pssf = os.path.join(test_ro_data_dir, conftest.my_sage_startup()) - remove_no_exc(pssf) - pssf3 = os.path.join(test_ro_data_dir, - "sage_init_files/runtime_err.sage") - os.symlink(pssf3, pssf) - - -class TestRuntimeErrStartupFile: - def test_ident_set_file_env(self, exec2): - code = "os.chdir(salvus.data[\'path\']);__file__=salvus.data[\'file\']" - exec2(code) - - def test_init_sage(self, exec2, test_ro_data_dir): - exec2("None", errout="division by zero") - pssf = os.path.join(test_ro_data_dir, conftest.my_sage_startup()) - remove_no_exc(pssf) - pssf4 = os.path.join(test_ro_data_dir, - "sage_init_files/syntax_err.sage") - os.symlink(pssf4, pssf) - - -class TestSyntaxErrStartupFile: - def test_ident_set_file_env(self, exec2): - code = "os.chdir(salvus.data[\'path\']);__file__=salvus.data[\'file\']" - exec2(code) - - def test_init_sage(self, exec2, test_ro_data_dir): - exec2("None", errout="invalid syntax") - pssf = os.path.join(test_ro_data_dir, conftest.my_sage_startup()) - remove_no_exc(pssf) diff --git a/src/smc_sagews/smc_sagews/tests/test_graphics.py b/src/smc_sagews/smc_sagews/tests/test_graphics.py deleted file mode 100644 index 7a2e843e02..0000000000 --- a/src/smc_sagews/smc_sagews/tests/test_graphics.py +++ /dev/null @@ -1,136 +0,0 @@ -# test_graphics.py -# tests of sage worksheet that return more than stdout, e.g. svg files -from __future__ import absolute_import - -import conftest -import time - -from textwrap import dedent - -import pytest - -# TODO(hal) refactor this later -SHA_LEN = 36 - - -class TestTachyon: - def test_t_show0(self, exec2): - code = dedent(r"""t = Tachyon(xres=400,yres=400, camera_center=(2,0,0)) - t.light((4,3,2), 0.2, (1,1,1)) - t.sphere((0,0,0), 0.5, 't0')""") - exec2(code, []) - - def test_t_show1(self, execblob): - execblob( - "t.show()", want_html=False, file_type='png', ignore_stdout=True) - - def test_show_t(self, execblob): - execblob( - "show(t)", want_html=False, file_type='png', ignore_stdout=True) - - def test_t(self, execblob): - execblob("t", want_html=False, file_type='png', ignore_stdout=True) - - -class TestThreeJS: - # https://github.com/sagemathinc/cocalc/issues/2450 - def test_2450(self, execblob): - code = """ - t, theta = var('t, theta', domain='real') - x(t) = cosh(t) - z(t) = t - formula = (x(t)*cos(theta), x(t)*sin(theta), z(t)) - parameters = ((t, -3, 3), (theta, -pi, pi)) - surface = ParametrizedSurface3D(formula, parameters) - p = surface.plot(aspect_ratio=1, color='yellow') - show(p, viewer='threejs', online=True) - """ - execblob( - dedent(code), - want_html=False, - ignore_stdout=True, - file_type='sage3d') - - -class TestGraphics: - def test_plot(self, execblob): - execblob("plot(cos(x),x,0,pi)", want_html=False, file_type='svg') - - -class TestOctavePlot: - def test_octave_plot(self, execblob): - # assume octave kernel not running at start of test - execblob( - "%octave\nx = -10:0.1:10;plot (x, sin (x));", - file_type='png', - ignore_stdout=True) - - -class TestRPlot: - def test_r_smallplot(self, execblob): - execblob("%r\nwith(mtcars,plot(wt,mpg))", file_type=['svg','png']) - - def test_r_bigplot(self, execblob): - "lots of points, do not overrun blob size limit" - code = """%r -N <- 100000 -xx <- rnorm(N, 5) + 3 -yy <- rnorm(N, 3) - 1 -plot(xx, yy, cex=.1)""" - execblob("%r\nwith(mtcars,plot(wt,mpg))", file_type=['svg','png']) - - -class TestShowGraphs: - def test_issue594(self, test_id, sagews): - code = """G = Graph(sparse=True) -G.allow_multiple_edges(True) -G.add_edge(1,2) -G.add_edge(2,3) -G.add_edge(3,1) -for i in range(2): - print ("BEFORE PLOT %s"%i) - G.show() - print ("AFTER PLOT %s"%i)""" - m = conftest.message.execute_code(code=code, id=test_id) - sagews.send_json(m) - # expect 12 messages from worksheet client, including final done:true - json_wanted = 6 - jstep = 0 - blob_wanted = 2 - while json_wanted > 0 and blob_wanted > 0: - typ, mesg = sagews.recv() - assert typ == 'json' or typ == 'blob' - if typ == 'json': - assert mesg['id'] == test_id - json_wanted -= 1 - jstep += 1 - if jstep == 1: - assert 'stdout' in mesg - assert 'BEFORE PLOT 0' in mesg['stdout'] - continue - elif jstep == 2: - assert 'file' in mesg - continue - elif jstep == 3: - assert 'stdout' in mesg - assert 'AFTER PLOT 0' in mesg['stdout'] - continue - elif jstep == 4: - assert 'stdout' in mesg - assert 'BEFORE PLOT 1' in mesg['stdout'] - continue - elif jstep == 5: - assert 'file' in mesg - continue - elif jstep == 6: - assert 'stdout' in mesg - assert 'AFTER PLOT 1' in mesg['stdout'] - continue - else: - blob_wanted -= 1 - file_uuid = mesg[:SHA_LEN].decode() - assert file_uuid == conftest.uuidsha1(mesg[SHA_LEN:]) - m = conftest.message.save_blob(sha1=file_uuid) - sagews.send_json(m) - continue - conftest.recv_til_done(sagews, test_id) diff --git a/src/smc_sagews/smc_sagews/tests/test_sagews.py b/src/smc_sagews/smc_sagews/tests/test_sagews.py deleted file mode 100644 index 821df693f4..0000000000 --- a/src/smc_sagews/smc_sagews/tests/test_sagews.py +++ /dev/null @@ -1,479 +0,0 @@ -# test_sagews.py -# basic tests of sage worksheet using TCP protocol with sage_server -from __future__ import absolute_import -import conftest -import os -import re - -from textwrap import dedent - -import pytest - - -@pytest.mark.skip(reason="waiting until #1835 is fixed") -class TestLex: - def test_lex_1(self, execdoc): - execdoc("x = random? # bar") - - def test_lex_2(self, execdoc): - execdoc("x = random? # plot?", pattern='random') - - def test_lex_3(self, exec2): - exec2("x = 1 # plot?\nx", "1\n") - - def test_lex_4(self, exec2): - exec2('x="random?" # plot?\nx', "'random?'\n") - - def test_lex_5(self, exec2): - code = dedent(r''' - x = """ - salvus? - """;pi''') - exec2(code, "pi\n") - - -class TestSageVersion: - def test_sage_vsn(self, exec2): - code = "sage.misc.banner.banner()" - patn = "version 8.[89]" - exec2(code, pattern=patn) - - -class TestDecorators: - def test_simple_dec(self, exec2): - code = dedent(r""" - def d2(f): return lambda x: f(x)+'-'+f(x) - @d2 - def s(str): return str.upper() - s('spam')""") - exec2(code, "'SPAM-SPAM'\n") - - def test_multiple_dec(self, exec2): - code = dedent(r""" - def dummy(f): return f - @dummy - @dummy - def f(x): return 2*x+1 - f(2)""") - exec2(code, "5\n") - - -class TestSageCommands: - def test_reset(self, exec2): - "issue 2646 do not clear salvus fns with sage reset" - code = dedent(r""" - a = EllipticCurve('123a') - save(a, 'load-save-test.sobj') - reset() - b = load('load-save-test.sobj') - b == EllipticCurve('123a')""") - exec2(code, "True\n") - - -class TestLinearAlgebra: - def test_solve_right(self, exec2): - code = dedent(r""" - A=matrix([[1,2,6],[1,2,0],[1,-2,3]]) - b=vector([1,-1,1]) - A.solve_right(b)""") - exec2(code, "(-1/2, -1/4, 1/3)") - - def test_kernel(self, exec2): - code = dedent(r""" - A=matrix([[1,2,3],[1,2,3],[1,2,3]]) - kernel(A)""") - pat = "\[ 1 0 -1\]\n\[ 0 1 -1\]" - exec2(code, pattern=pat) - - def test_charpoly(self, exec2): - code = dedent(r""" - A=matrix([[1,2,3],[1,2,3],[1,2,3]]) - A.charpoly()""") - exec2(code, "x^3 - 6*x^2\n") - - def test_eigenvalues(self, exec2): - code = dedent(r""" - A=matrix([[1,2,3],[1,2,3],[1,2,3]]) - A=matrix([[1,2,3],[1,2,3],[1,2,3]]) - A.eigenvalues()""") - exec2(code, "[6, 0, 0]\n") - - -class TestBasic: - def test_connection_type(self, sagews): - print(("type %s" % type(sagews))) - assert isinstance(sagews, conftest.ConnectionJSON) - return - - def test_set_file_env(self, exec2): - code = "os.chdir(salvus.data[\'path\']);__file__=salvus.data[\'file\']" - exec2(code) - - def test_sage_assignment(self, exec2): - code = "x = 42\nx\n" - output = "42\n" - exec2(code, output) - - def test_issue70(self, exec2): - code = dedent(r""" - for i in range(1): - pass - 'x' - """) - output = dedent(r""" - 'x' - """).lstrip() - exec2(code, output) - - def test_issue819(self, exec2): - code = dedent(r""" - def never_called(a): - print('should not execute 1', a) - # comment - # comment at indent 0 - print('should not execute 2', a) - 22 - """) - output = "22\n" - exec2(code, output) - - def test_search_doc(self, exec2): - code = "search_doc('laurent')" - html = "https://www.google.com/search\?q=site%3Adoc.sagemath.org\+laurent\&oq=site%3Adoc.sagemath.org" - exec2(code, html_pattern=html) - - def test_show_doc(self, test_id, sagews): - # issue 476 - code = "show?" - patn = dedent(""" - import smc_sagews.graphics - smc_sagews.graphics.graph_to_d3_jsonable?""") - m = conftest.message.execute_code(code=code, id=test_id) - sagews.send_json(m) - # ignore stderr message about deprecation warning - for ix in [0, 1]: - typ, mesg = sagews.recv() - assert typ == 'json' - assert mesg['id'] == test_id - if 'stderr' in mesg: - continue - assert 'code' in mesg - assert 'source' in mesg['code'] - assert re.sub('\s+', '', patn) in re.sub('\s+', '', - mesg['code']['source']) - conftest.recv_til_done(sagews, test_id) - break - - -class TestPythonFutureFeatures: - def test_pyfutfeats_0(self, exec2): - exec2("python_future_feature()", "[]\n") - - def test_pyfutfeats_1(self, exec2): - exec2("python_future_feature('division')", "False\n") - - def test_pyfutfeats_2(self, exec2): - exec2("python_future_feature('division', True)") - - def test_pyfutfeats_3(self, exec2): - exec2("python_future_feature()", "['division']\n") - - def test_pyfutfeats_4(self, exec2): - exec2("python_future_feature('division')", "True\n") - - def test_pyfutfeats_5(self, exec2): - exec2("print(8r / 5r)", "1.6\n") - - def test_pyfutfeats_6(self, exec2): - exec2("python_future_feature('division', False)") - - def test_pyfutfeats_7(self, exec2): - exec2("python_future_feature()", "[]\n") - - def test_pyfutfeats_8(self, exec2): - exec2("python_future_feature('division')", "False\n") - - def test_pyfutfeats_9(self, exec2): - exec2("print(8r / 5r)", "1\n") - - -class TestPythonFutureImport: - def test_pyfutimp_0(self, exec2): - code = dedent(r""" - for feature in python_future_feature(): - python_future_feature(feature, False) - """) - exec2(code) - - def test_pyfutimp_1(self, exec2): - exec2("print(8r / 5r)", "1\n") - - def test_pyfutimp_2(self, exec2): - code = dedent(r""" - from __future__ import division - print(8r / 5r) - """) - output = "1.6\n" - exec2(code, output) - - def test_pyfutimp_3(self, exec2): - exec2("print(8r / 5r)", "1.6\n") - - def test_pyfutimp_4(self, exec2): - exec2("python_future_feature('division', False)") - - def test_pyfutimp_5(self, exec2): - exec2("print(8r / 5r)", "1\n") - - -class TestPy3printMode: - def test_py3print_mode0(self, exec2): - exec2("py3print_mode()", "False\n") - - def test_py3print_mode1(self, exec2): - exec2("py3print_mode(True)") - - def test_py3print_mode2(self, exec2): - exec2("py3print_mode()", "True\n") - - def test_py3print_mode3(self, exec2): - code = dedent(r""" - py3print_mode(True) - print('hello', end=' Q') - """) - output = "hello Q" - exec2(code, output) - - def test_py3print_mode4(self, exec2): - exec2("py3print_mode(False)") - - def test_py3print_mode5(self, exec2): - exec2("print '42'", "42\n") - - -class TestUnderscore: - # https://github.com/sagemathinc/cocalc/issues/1107 - def test_sage_underscore_1(self, exec2): - exec2("2/5", "2/5\n") - - def test_sage_underscore_2(self, exec2): - exec2("_", "2/5\n") - - # https://github.com/sagemathinc/cocalc/issues/2124 - def test_sage_underscore_3(self, exec2): - exec2("typeset_mode(True)\n_", html_pattern=r'\\frac\{2\}\{5\}') - - def test_sage_underscore_4(self, exec2): - exec2("3*7", html_pattern="21\$") - - def test_sage_underscore_5(self, exec2): - exec2("typeset_mode(False)\n_", "21\n") - - -class TestModeComments: - # https://github.com/sagemathinc/cocalc/issues/978 - def test_mode_comments_1(self, exec2): - exec2(dedent(""" - def f(s): - print "s='%s'"%s""")) - - def test_mode_comments_2(self, exec2): - exec2( - dedent(""" - %f - 123 - # foo - 456"""), - dedent(""" - s='123 - # foo - 456' - """).lstrip()) - - -class TestBlockParser: - def test_block_parser(self, execbuf): - """ - .. NOTE:: - - This function supplies a list of expected outputs to `exec2`. - """ - execbuf( - dedent(""" - pi.n().round() - [x for x in [1,2,3] if x<3] - for z in ['a','b']: - z - else: - z"""), "3\n[1, 2]\n'a'\n'b'\n'b'\n") - - -class TestIntrospect: - # test names end with SMC issue number - def test_sage_autocomplete_1188(self, execintrospect): - execintrospect('2016.fa', ["ctor", "ctorial"], "fa") - - def test_sage_autocomplete_295_setup(self, exec2): - exec2("aaa=Rings()._super_categories_for_classes;len(aaa[0].axioms())", - "6\n") - - def test_sage_autocomplete_295a(self, execintrospect): - execintrospect('for a in aa', ["a"], "aa") - - def test_sage_autocomplete_295b(self, execintrospect): - execintrospect('3 * aa', ["a"], "aa") - - def test_sage_autocomplete_701_setup(self, exec2): - exec2( - dedent(""" - class Xyz: - numerical_attribute = 42 - x1 = Xyz() - x1.numerical_attribute.next_prime()"""), "43\n") - - def test_sage_autocomplete_701a(self, execintrospect): - execintrospect('3 / x1.nu', ["merical_attribute"], "nu") - - def test_sage_autocomplete_701b(self, execintrospect): - execintrospect('aa', ["a"], "aa") - - def test_sage_autocomplete_701c(self, execintrospect): - execintrospect('[aa', ["a"], "aa") - - def test_sage_autocomplete_701d(self, execintrospect): - execintrospect('( aa', ["a"], "aa") - - def test_sage_autocomplete_734a(self, execintrospect): - f = '*_factors' - execintrospect(f, ["cunningham_prime_factors", "prime_factors"], f) - - def test_sage_autocomplete_734b(self, execintrospect): - f = '*le_pr*' - execintrospect(f, ["next_probable_prime"], f) - - def test_sage_autocomplete_734c(self, execintrospect): - execintrospect('list.re*e', ["remove", "reverse"], 're*e') - - def test_sage_autocomplete_1225a(self, execintrospect): - execintrospect('z = 12.5 * units.len', ["gth"], 'len') - - def test_sage_autocomplete_1225b_setup(self, exec2): - exec2( - dedent(""" - class TC: - def __init__(self, xval): - self.x = xval - y = TC(49) - """)) - - def test_sage_autocomplete_1225b(self, execintrospect): - execintrospect('z = 12 * y.', ["x"], '') - - def test_sage_autocomplete_1252a(self, execintrospect): - execintrospect('2*sqr', ["t"], 'sqr') - - def test_sage_autocomplete_1252b(self, execintrospect): - execintrospect('2+sqr', ["t"], 'sqr') - - -class TestAttach: - def test_define_paf(self, exec2): - exec2( - dedent(r""" - def paf(): - print("attached files: %d"%len(attached_files())) - print("\n".join(attached_files())) - paf()"""), "attached files: 0\n\n") - - def test_attach_sage_1(self, exec2, test_ro_data_dir): - fn = os.path.join(test_ro_data_dir, 'a.sage') - exec2( - "%attach {}\npaf()".format(fn), - pattern="attached files: 1\n.*/a.sage\n") - - def test_attach_sage_2(self, exec2): - exec2("f1('foo')", "f1 arg = 'foo'\ntest f1 1\n") - - def test_attach_py_1(self, exec2, test_ro_data_dir): - fn = os.path.join(test_ro_data_dir, 'a.py') - exec2( - "%attach {}\npaf()".format(fn), - pattern="attached files: 2\n.*/a.py\n.*/a.sage\n") - - def test_attach_py_2(self, exec2): - exec2("f2('foo')", "test f2 1\n") - - def test_attach_html_1(self, execblob, test_ro_data_dir): - fn = os.path.join(test_ro_data_dir, 'a.html') - execblob( - "%attach {}".format(fn), - want_html=False, - want_javascript=True, - file_type='html') - - def test_attach_html_2(self, exec2): - exec2( - "paf()", - pattern="attached files: 3\n.*/a.html\n.*/a.py\n.*/a.sage\n") - - def test_detach_1(self, exec2): - exec2("detach(attached_files())") - - def test_detach_2(self, exec2): - exec2("paf()", "attached files: 0\n\n") - - -class TestSearchSrc: - def test_search_src_simple(self, execinteract): - execinteract('search_src("convolution")') - - def test_search_src_max_chars(self, execinteract): - execinteract('search_src("full cremonadatabase", max_chars = 1000)') - - -class TestIdentifiers: - """ - see SMC issue #63 - """ - - def test_ident_set_file_env(self, exec2): - """emulate initial code block sent from UI, needed for first show_identifiers""" - code = "os.chdir(salvus.data[\'path\']);__file__=salvus.data[\'file\']" - exec2(code) - - def test_show_identifiers_initial(self, exec2): - exec2("show_identifiers()", "[]\n") - - def test_show_identifiers_vars(self, exec2): - code = dedent(r""" - k = ['a','b','c'] - A = {'a':'foo','b':'bar','c':'baz'} - z = 99 - sorted(show_identifiers())""") - exec2(code, "['A', 'k', 'z']\n") - - def test_save_and_reset(self, exec2, data_path): - code = dedent(r""" - save_session('%s') - reset() - show_identifiers()""") % data_path.join('session').strpath - exec2(code, "[]\n") - - def test_load_session1(self, exec2, data_path): - code = dedent(r""" - pretty_print = 8 - view = 9 - load_session('%s') - sorted(show_identifiers())""") % data_path.join('session').strpath - output = "['A', 'k', 'pretty_print', 'view', 'z']\n" - exec2(code, output) - - def test_load_session2(self, exec2): - exec2("pretty_print,view", "(8, 9)\n") - - def test_redefine_sage(self, exec2): - code = dedent(r""" - reset() - sage=1 - show_identifiers()""") - exec2(code, "['sage']\n") diff --git a/src/smc_sagews/smc_sagews/tests/test_sagews_modes.py b/src/smc_sagews/smc_sagews/tests/test_sagews_modes.py deleted file mode 100644 index 54ffd9a4f3..0000000000 --- a/src/smc_sagews/smc_sagews/tests/test_sagews_modes.py +++ /dev/null @@ -1,344 +0,0 @@ -# test_sagews_modes.py -# tests of sage worksheet modes -from __future__ import absolute_import -import pytest -import conftest -import re -import os -from textwrap import dedent - - -class TestLatexMode: - def test_latex(self, execblob): - execblob( - "%latex\nhello", - want_html=False, - file_type='png', - ignore_stdout=True) - - -class TestP3Mode: - def test_p3a(self, exec2): - exec2("p3 = jupyter('python3')") - - def test_p3b(self, exec2): - exec2("%p3\nimport sys\nprint(sys.version)", pattern=r"^3\.[56]\.\d+ ") - - -class TestSingularMode: - def test_singular_version(self, exec2): - exec2('%singular_kernel\nsystem("version");', pattern=r"^41\d\d\b") - - def test_singular_factor_polynomial(self, exec2): - code = dedent(''' - %singular_kernel - ring R1 = 0,(x,y),dp; - poly f = 9x16 - 18x13y2 - 9x12y3 + 9x10y4 - 18x11y2 + 36x8y4 + 18x7y5 - 18x5y6 + 9x6y4 - 18x3y6 - 9x2y7 + 9y8; - factorize(f);''').strip() - exec2( - code, - '[1]:\n _[1]=9\n _[2]=x6-2x3y2-x2y3+y4\n _[3]=-x5+y2\n[2]:\n 1,1,2\n' - ) - - -import platform - -# this comparison doesn't work in sage env -# because the distro version is spoofed -# platform.linux_distribution()[1] == "18.04", - -@pytest.mark.skipif( - True, - reason="scala jupyter kernel broken in 18.04") -class TestScalaMode: - def test_scala_list(self, exec2): - print(("linux version {}".format(platform.linux_distribution()[1]))) - exec2( - "%scala\nList(1,2,3)", - html_pattern="res0.*List.*Int.*List.*1.*2.*3", - timeout=80) - - -@pytest.mark.skipif( - True, - reason="scala jupyter kernel broken in 18.04") -class TestScala211Mode: - # example from ScalaTour-1.6, p. 31, Pattern Matching - # http://www.scala-lang.org/docu/files/ScalaTour-1.6.pdf - def test_scala211_pat1(self, exec2): - code = dedent(''' - %scala211 - object MatchTest1 extends App { - def matchTest(x: Int): String = x match { - case 1 => "one" - case 2 => "two" - case _ => "many" - } - println(matchTest(3)) - } - ''').strip() - exec2(code, html_pattern="defined.*object.*MatchTest1", timeout=80) - - def test_scala211_pat2(self, exec2): - exec2("%scala211\nMatchTest1.main(Array())", pattern="many") - - def test_scala_version(self, exec2): - exec2( - "%scala211\nutil.Properties.versionString", html_pattern="2.11.11") - - -class TestAnaconda5: - def test_anaconda_version(self, exec2): - exec2( - "%anaconda\nimport sys\nprint(sys.version)", - pattern=r"^3\.6\.\d+ ") - - def test_anaconda_kernel_name(self, exec2): - exec2("anaconda.jupyter_kernel.kernel_name", "anaconda5") - -class TestPython3Mode: - def test_p3_max(self, exec2): - exec2("%python3\nmax([],default=9)", "9", timeout=30) - - def test_p3_kernel_name(self, exec2): - exec2("python3.jupyter_kernel.kernel_name", "python3") - - def test_p3_version(self, exec2): - exec2( - "%python3\nimport sys\nprint(sys.version)", pattern=r"^3\.6\.\d+ ") - - def test_capture_p3_01(self, exec2): - exec2( - "%capture(stdout='output')\n%python3\nimport numpy as np\nnp.arange(9).reshape(3,3).trace()" - ) - - def test_capture_p3_02(self, exec2): - exec2("print(output)", "12\n") - - def test_p3_latex(self, exec2): - code = r"""%python3 -from IPython.display import Math -Math(r'F(k) = \int_{-\infty}^{\infty} f(x) e^{2\pi i k} dx')""" - htmp = r"""F\(k\) = \\int_\{-\\infty\}\^\{\\infty\} f\(x\) e\^\{2\\pi i k\} dx""" - exec2(code, html_pattern=htmp) - - def test_p3_pandas(self, exec2): - code = dedent(''' - %python3 - import pandas as pd - from io import StringIO - - df_csv = r"""Item,Category,Quantity,Weight - Pack,Pack,1,33.0 - Tent,Shelter,1,80.0 - Sleeping Pad,Sleep,0,27.0 - Sleeping Bag,Sleep,1,20.0 - Shoes,Clothing,1,12.0 - Hat,Clothing,1,2.5""" - mydata = pd.read_csv(StringIO(df_csv)) - mydata.shape''').strip() - exec2(code, "(6, 4)") - - def test_p3_autocomplete(self, execintrospect): - execintrospect('myd', ["ata"], 'myd', '%python3') - - -class TestPython3DefaultMode: - def test_set_python3_mode(self, exec2): - exec2("%default_mode python3") - - def test_python3_assignment(self, exec2): - exec2("xx=[2,5,99]\nsum(xx)", "106") - - def test_capture_p3d_01(self, exec2): - exec2("%capture(stdout='output')\nmax(xx)") - - def test_capture_p3d_02(self, exec2): - exec2("%sage\nprint(output)", "99\n") - - -class TestShMode: - def test_start_sh(self, exec2): - code = "%sh\ndate +%Y-%m-%d" - patn = r'\d{4}-\d{2}-\d{2}' - exec2(code, pattern=patn) - - # examples from sh mode docstring in sage_salvus.py - # note jupyter kernel text ouput is displayed as html - def test_single_line(self, exec2): - exec2("%sh uptime\n", pattern="\d\.\d") - - def test_multiline(self, exec2): - exec2("%sh\nFOO=hello\necho $FOO", pattern="hello") - - def test_direct_call(self, exec2): - exec2("sh('date +%Y-%m-%d')", pattern=r'\d{4}-\d{2}-\d{2}') - - def test_capture_sh_01(self, exec2): - exec2("%capture(stdout='output')\n%sh uptime") - - def test_capture_sh_02(self, exec2): - exec2("output", pattern="up.*user.*load average") - - def test_remember_settings_01(self, exec2): - exec2("%sh FOO='testing123'") - - def test_remember_settings_02(self, exec2): - exec2("%sh echo $FOO", pattern=r"^testing123\s+") - - def test_sh_display(self, execblob, image_file): - execblob("%sh display < " + str(image_file), want_html=False) - - def test_sh_autocomplete_01(self, exec2): - exec2("%sh TESTVAR29=xyz") - - def test_sh_autocomplete_02(self, execintrospect): - execintrospect('echo $TESTV', ["AR29"], '$TESTV', '%sh') - - def test_bad_command(self, exec2): - exec2("%sh xyz", pattern="command not found") - - -class TestShDefaultMode: - def test_start_sh_default(self, exec2): - exec2("%default_mode sh") - - def test_multiline_default(self, exec2): - exec2("FOO=hello\necho $FOO", pattern="^hello") - - def test_date(self, exec2): - exec2("date +%Y-%m-%d", pattern=r'^\d{4}-\d{2}-\d{2}') - - def test_capture_sh_01_default(self, exec2): - exec2("%capture(stdout='output')\nuptime") - - def test_capture_sh_02_default(self, exec2): - exec2("%sage\noutput", pattern="up.*user.*load average") - - def test_remember_settings_01_default(self, exec2): - exec2("FOO='testing123'") - - def test_remember_settings_02_default(self, exec2): - exec2("echo $FOO", pattern=r"^testing123\s+") - - def test_sh_display_default(self, execblob, image_file): - execblob("display < " + str(image_file), want_html=False) - - def test_sh_autocomplete_01_default(self, exec2): - exec2("TESTVAR29=xyz") - - def test_sh_autocomplete_02_default(self, execintrospect): - execintrospect('echo $TESTV', ["AR29"], '$TESTV') - - -class TestRMode: - def test_r_assignment(self, exec2): - exec2("%r\nxx <- c(4,7,13)\nmean(xx)", html_pattern="^8$") - - def test_r_version(self, exec2): - exec2("%r\nR.version.string", html_pattern=r"\d+\.\d+\.\d+") - - def test_capture_r_01(self, exec2): - exec2("%capture(stdout='output')\n%r\nsum(xx)") - - def test_capture_r_02(self, exec2): - exec2("print(output)", "24\n") - - -class TestRDefaultMode: - def test_set_r_mode(self, exec2): - exec2("%default_mode r") - - def test_rdefault_assignment(self, exec2): - exec2("xx <- c(4,7,13)\nmean(xx)", html_pattern="^8$") - - def test_default_capture_r_01(self, exec2): - exec2("%capture(stdout='output')\nsum(xx)") - - def test_default_capture_r_02(self, exec2): - exec2("%sage\nprint(output)", "24\n") - - -class TestRWD: - "issue 240" - - def test_wd0(self, exec2, data_path): - dp = data_path.strpath - code = "os.chdir('%s')" % dp - exec2(code) - - def test_wd(self, exec2, data_path): - dp = data_path.strpath - exec2("%r\ngetwd()", html_pattern=dp) - - -class TestOctaveMode: - def test_start_octave(self, exec2): - exec2("%octave") - - def test_octave_calc(self, exec2): - code = "%octave\nformat short\nbesselh(0,2)" - outp = r"ans = 0.22389\s+\+\s+0.51038i" - exec2(code, pattern=outp) - - def test_octave_fibonacci(self, exec2): - code = dedent('''%octave - fib = ones (1, 10); - for i = 3:10 - fib(i) = fib(i-1) + fib(i-2); - printf('%d,', fib(i)) - endfor - ''') - outp = '2,3,5,8,13,21,34,55,' - exec2(code, pattern=outp) - - def test_octave_insync(self, exec2): - # this just confirms, that input/output is still in sync after the for loop above - exec2('%octave\n1+1', pattern='ans = 2') - - -class TestOctaveDefaultMode: - def test_octave_capture1(self, exec2): - exec2("%default_mode octave") - - def test_octave_capture2(self, exec2): - exec2("%capture(stdout='output')\nx = [1,2]") - - def test_octave_capture3(self, exec2): - exec2("%sage\nprint(output)", pattern=" 1 2") - - def test_octave_version(self, exec2): - exec2("version()", pattern="4.2.2") - - -class TestAnaconda2019Mode: - def test_start_a2019(self, exec2): - exec2('a2019 = jupyter("anaconda2019")') - - def test_issue_862(self, exec2): - exec2('%a2019\nx=1\nprint("x = %s" % x)\nx', 'x = 1\n') - - def test_a2019_error(self, exec2): - exec2('%a2019\nxyz*', html_pattern='span style.*color') - - -class TestAnaconda5Mode: - def test_start_a5(self, exec2): - exec2('a5 = jupyter("anaconda5")') - - def test_issue_862(self, exec2): - exec2('%a5\nx=1\nprint("x = %s" % x)\nx', 'x = 1\n') - - def test_a5_error(self, exec2): - exec2('%a5\nxyz*', html_pattern='span style.*color') - - -class TestJuliaMode: - def test_julia_quadratic(self, exec2): - exec2( - '%julia\nquadratic(a, sqr_term, b) = (-b + sqr_term) / 2a\nquadratic(2.0, -2.0, -12.0)', - '2.5', - timeout=40) - - def test_julia_version(self, exec2): - exec2("%julia\nVERSION", pattern=r'^v"1\.2\.\d+"', timeout=40) diff --git a/src/smc_sagews/smc_sagews/tests/test_unicode.py b/src/smc_sagews/smc_sagews/tests/test_unicode.py deleted file mode 100644 index 53f33bbc81..0000000000 --- a/src/smc_sagews/smc_sagews/tests/test_unicode.py +++ /dev/null @@ -1,131 +0,0 @@ -# -*- coding: utf-8 -*- -# test_sagews.py -# basic tests of sage worksheet using TCP protocol with sage_server -from __future__ import absolute_import -import socket -import conftest -import os -import re - - -class TestBadContinuation: - r""" - String with badly-formed utf8 would hang worksheet process #1866 - """ - - def test_bad_utf8(self, exec2): - code = r"""print('u"\xe1"')""" - outp = "�" - exec2(code, outp) - - -class TestUnicode: - r""" - To pass unicode in a simulated input cell, quote it. - That will send the same message to sage_server - as a real input cell without the outer quotes. - """ - - def test_unicode_1(self, exec2): - r""" - test for cell with input u"äöüß" - """ - ustr = 'u"äöüß"' - x = r'\xe4\xf6\xfc\xdf' - exec2(ustr, x) - - def test_unicode_2(self, exec2): - r""" - Test for cell with input u"ááá". - Input u"ááá" in an actual cell causes latin1 encoding to appear - enclosed by u"...", inside a unicode string in the message to sage_server. - (So there are two u's in the displayed message in the log.) - Code part of logged input message to sage_server: - u'code': u'u"\xe1\xe1\xe1"\n' - Stdout part of logged output message from sage_server: - "stdout": "u\'\\\\xe1\\\\xe1\\\\xe1\'\\n" - """ - ustr = 'u"ááá"' - x = r'\xe1\xe1\xe1' - exec2(ustr, x) - - def test_unicode_3(self, exec2): - r""" - Test for cell with input "ááá". - Input "ááá" in an actual cell causes utf8 encoding to appear - inside a unicode string in the message to sage_server. - Code part of logged input message to sage_server: - u'code': u'"\xe1\xe1\xe1"\n' - Stdout part of logged output message from sage_server: - "stdout": "\'\\\\xc3\\\\xa1\\\\xc3\\\\xa1\\\\xc3\\\\xa1\'\\n" - """ - ustr = '"ááá"' - x = r'\xc3\xa1\xc3\xa1\xc3\xa1' - exec2(ustr, x) - - def test_unicode_4(self, exec2): - r""" - test for cell with input "öäß" - """ - ustr = '"öäß"' - x = r'\xc3\xb6\xc3\xa4\xc3\x9f' - exec2(ustr, x) - - -class TestOutputReplace: - def test_1865(self, exec2): - code = 'for x in [u"ááá", "ááá"]: print(x)' - xout = 'ááá\nááá\n' - exec2(code, xout) - - -class TestErr: - def test_non_ascii(self, test_id, sagews): - # assign to hbar to trigger non-ascii warning - code = "ħ = 9" - m = conftest.message.execute_code(code=code, id=test_id) - sagews.send_json(m) - # expect 2 messages from worksheet client - # 1 stderr Error in lines 1-1 - typ, mesg = sagews.recv() - assert typ == 'json' - assert mesg['id'] == test_id - assert 'stderr' in mesg - assert 'Error in lines 1-1' in mesg['stderr'] - assert 'should be replaced by < " >' not in mesg['stderr'] - conftest.recv_til_done(sagews, test_id) - - def test_bad_quote(self, test_id, sagews): - # assign x to U+201C (could use U+201D) to trigger bad quote warning - code = "“" - m = conftest.message.execute_code(code=code, id=test_id) - sagews.send_json(m) - # expect 2 messages from worksheet client - # 1 stderr Error in lines 1-1 - typ, mesg = sagews.recv() - assert typ == 'json' - assert mesg['id'] == test_id - assert 'stderr' in mesg - assert 'non-ascii' in mesg['stderr'] - if 'should be replaced by < " >' not in mesg['stderr']: - typ, mesg = sagews.recv() - assert typ == 'json' - assert mesg['id'] == test_id - assert 'stderr' in mesg - assert 'should be replaced by < " >' in mesg['stderr'] - conftest.recv_til_done(sagews, test_id) - - def test_bad_mult(self, test_id, sagews): - # warn about possible missing '*' with patterns like 3x^2 and 5(1+x) - code = ("x=1\ny=3x^2x") - m = conftest.message.execute_code(code=code, id=test_id) - sagews.send_json(m) - # expect 2 messages from worksheet client - # 1 stderr - typ, mesg = sagews.recv() - assert typ == 'json' - assert mesg['id'] == test_id - assert 'stderr' in mesg - assert 'implicit multiplication' in mesg['stderr'] - # 2 done - conftest.recv_til_done(sagews, test_id) From 7a0fb4375ff220a09c27c89adf75597748f3b730 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 8 Oct 2025 11:06:56 -0700 Subject: [PATCH 784/798] delete all the old HTML templates (that were used for the jquery+coffeescript approach, long ago) --- src/packages/frontend/client/project.ts | 20 - src/packages/frontend/editor-templates.ts | 1819 --------------------- 2 files changed, 1839 deletions(-) delete mode 100644 src/packages/frontend/editor-templates.ts diff --git a/src/packages/frontend/client/project.ts b/src/packages/frontend/client/project.ts index 7550a796eb..ee58e58a5d 100644 --- a/src/packages/frontend/client/project.ts +++ b/src/packages/frontend/client/project.ts @@ -582,26 +582,6 @@ export class ProjectClient { TOUCH_THROTTLE, ); - // Print sagews to pdf - // The printed version of the file will be created in the same directory - // as path, but with extension replaced by ".pdf". - // Only used for sagews. - print_to_pdf = async ({ - project_id, - path, - options, - timeout, - }: { - project_id: string; - path: string; - timeout?: number; // client timeout -- some things can take a long time to print! - options?: any; // optional options that get passed to the specific backend for this file type - }): Promise => { - return await this.client.conat_client - .projectApi({ project_id }) - .editor.printSageWS({ path, timeout, options }); - }; - create = async (opts: { title: string; description: string; diff --git a/src/packages/frontend/editor-templates.ts b/src/packages/frontend/editor-templates.ts deleted file mode 100644 index 447a90ed61..0000000000 --- a/src/packages/frontend/editor-templates.ts +++ /dev/null @@ -1,1819 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2024 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -const console = ` -
-
- - - -
-
- -
- -    - -
- - - - - - - -
- -
-    -
- WARNING: Large burst of output! (May try to interrupt.) -
- -
- - - - esc - return - submit - ctrl-b - ctrl-c - tab - - - - - -
- - - -
- - - -
- -
-
-
-
-
-
-
-
-
- -`; - -const editor = ` -
- -
-
-
- -
-
-
-
- - - - - -
- - -
-
-
- - -
- - - - - Save - - - - - - Run - - - - - - - - - - - - - - - - Copy To Your Project... - - - - - - - - - - - - - - - - - - - - load… - - - - sync… - - - - - -
- -
-
-
- - - - - - - - -
-
- - - - - - - - - - - $ - - - $$ - - - - - - - - - - - - - - - - - - - - - - - - -
    -     - - - - -
      -       - - - - -
        -         -         - - - - - - - - - - - - - - -
        -
        -
        - -
        - -
        -
        -
        -
        -
        -
        -
        - -
        - -
        -
        -
        -
        - -
        -
        -
        -
        - -
        - -
        -
        -
        -
        - -
        -
        -
        -
        -
        - -
        - - -
        - -
           
        -
        - -
        - -
           
        -
        - - -
        -
        - -
        -
        - - -
        - -
        -
        - - - - - - - - - - - Build preview - - -
        -
        -
        -
        -
        -
        - - -
        -
        - - - - - - - - - - - -
        -
        - -
        -
        - -
        -
        - - - TimeTravel - - - Changes - Changes - - - - - - - Open File - Revert live version to this   - - - - - Load All History - - Export - -
        -    to   -
        -
        -
        -   - -
        -
        -
        -
        -
        - -
        - WARNING: History viewer for this file type not implemented, so - showing underlying raw file instead. -
        - -
        -
        - - - - - - - - - - - - - -
        - - -
        - - - Save - - - - - Run - - - - Stop - - - - Tab - - - - - Reload - - - - -
        -

        - Opening... -

        -
        -
        - - - - -
        - - - - - - - - - - - - $ - - - $$ - - - - - - - — - Ω - - - - - - - - - - - - - - - -
          -           - - - - -
            -             - - - - -
              -               -               - - - - - - - - - - - - - -
              - -
              - -
              - -
              - -
              -
              - -
              -
              - -
              -
              - -
              -
              - … -
              - -
              -
              -
              - -
              - - - - -
              -
              -
              -
              -
              - -`; - -const sagews_interact = ` -
              - - -
              -
              - - -
              -
              -
              - -
              -
              - - - - Submit -

              -               -
              -
              -
              - - - -
              -
              -
              - -
              -
              - -
              -
              -
              - - -
              -
              -
              - -
              -
              -
              -
              -
              -
              - - -
              -
              -
              - -
              -
              -    -
              -
              -
              - - -
              -
              -
              - -
              -
              - -
              -
              -
              - - -
              -
              -
              - -
              -
              -
              - - -
              -
              -
              -
              - - -
              -
              -
              -
              -
              -
              -
              -
              -
              -
              -
              -
              -
              -
              -
              - - - -
              -
              -
              -
              -
              -
              -
              -
              -
              -
              -
              -
              -
              -
              -
              - - -
              -
              -
              - -
              -
              - - -
              -
              -
              - -
              -`; - -const sagews_3d = ` -
              - - - Loading 3D scene... - - - - Evaluate to see 3d plot. - - - - canvas - -
              -`; - -const sagews_d3 = ` -
              - - -
              -`; - -export const TEMPLATES_HTML = - console + editor + sagews_interact + sagews_3d + sagews_d3; From 054d7c9991922f3e36d327d9fd52983bff35f2af Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 8 Oct 2025 11:12:24 -0700 Subject: [PATCH 785/798] do not install ancient threejs -- no longer need it --- src/packages/frontend/package.json | 1 - src/packages/pnpm-lock.yaml | 8 -------- 2 files changed, 9 deletions(-) diff --git a/src/packages/frontend/package.json b/src/packages/frontend/package.json index 457630602c..6097691ba4 100644 --- a/src/packages/frontend/package.json +++ b/src/packages/frontend/package.json @@ -140,7 +140,6 @@ "sanitize-html": "^2.17.0", "slate": "^0.103.0", "superb": "^3.0.0", - "three-ancient": "npm:three@=0.78.0", "underscore": "^1.12.1", "universal-cookie": "^4.0.4", "use-async-effect": "^2.2.7", diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index e2f5603bbf..092444eca4 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -699,9 +699,6 @@ importers: superb: specifier: ^3.0.0 version: 3.0.0 - three-ancient: - specifier: npm:three@=0.78.0 - version: three@0.78.0 tsd: specifier: ^0.22.0 version: 0.22.0 @@ -11588,9 +11585,6 @@ packages: peerDependencies: tslib: ^2 - three@0.78.0: - resolution: {integrity: sha512-JIu4XJrBiwN68qb9Vz/0vF1MjhXM5WueBWpRid+wI2gAKlmnseKCNI3srWbBCV7te1bk9X+sw4ASMSMl5Ng98w==} - throttle-debounce@5.0.2: resolution: {integrity: sha512-B71/4oyj61iNH0KeCamLuE2rmKuTO5byTOSVwECM5FA7TiAiAW+UqTKZ9ERueC4qvgSttUhdmq1mXC3kJqGX7A==} engines: {node: '>=12.22'} @@ -24602,8 +24596,6 @@ snapshots: dependencies: tslib: 2.8.1 - three@0.78.0: {} - throttle-debounce@5.0.2: {} through2@0.6.5: From b4b35c15d7d5cc91372e29a90e7b3f69f195b936 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 8 Oct 2025 11:15:13 -0700 Subject: [PATCH 786/798] remove +New sagews --- .../project/new/file-type-selector.tsx | 56 +------------------ 1 file changed, 1 insertion(+), 55 deletions(-) diff --git a/src/packages/frontend/project/new/file-type-selector.tsx b/src/packages/frontend/project/new/file-type-selector.tsx index 253813b558..3c0c9dc304 100644 --- a/src/packages/frontend/project/new/file-type-selector.tsx +++ b/src/packages/frontend/project/new/file-type-selector.tsx @@ -3,7 +3,7 @@ * License: MS-RSL – see LICENSE.md for details */ -import { Col, Flex, Modal, Row, Tag } from "antd"; +import { Col, Flex, Row, Tag } from "antd"; import { Gutter } from "antd/es/grid/row"; import type { ReactNode } from "react"; import { FormattedMessage, useIntl } from "react-intl"; @@ -368,59 +368,6 @@ export function FileTypeSelector({ ); } - function renderSageWS() { - if (!availableFeatures.sage) return; - - function handleClick(ext) { - Modal.confirm({ - width: 500, - icon: , - title: intl.formatMessage({ - id: "project.new.file-type-selector.sagews.modal.title", - defaultMessage: - "SageMath Worksheets are *DEPRECATED* and MAY NOT WORK AT ALL", - }), - content: intl.formatMessage({ - id: "project.new.file-type-selector.sagews.modal.content", - defaultMessage: - "Instead, create a Jupyter Notebook and use a SageMath Kernel. You can also convert existing SageMath Worksheets to Jupyter Notebooks by opening the worksheet and clicking 'Jupyter'.", - }), - okText: intl.formatMessage({ - id: "project.new.file-type-selector.sagews.modal.ok", - defaultMessage: "Create SageMath Worksheet Anyways", - }), - onOk: (close) => { - create_file(ext); - close(); - }, - closable: true, - }); - } - - return ( - - - - - - ); - } - function addAiDocGenerate(btn: React.JSX.Element, ext: Ext) { if (isFlyout) { return ( @@ -613,7 +560,6 @@ export function FileTypeSelector({ /> - {renderSageWS()} ); From 530f4b0d735d1d0a78befca69afe00a001158c5d Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 8 Oct 2025 11:37:05 -0700 Subject: [PATCH 787/798] start working on sagews converter; delete old code --- .../frame-editors/frame-tree/title-bar.tsx | 13 ++- .../frame-editors/sagews-editor/actions.ts | 54 +-------- .../sagews-editor/cell-worksheet.tsx | 68 ------------ .../sagews-editor/code-executor.ts | 105 ------------------ .../frame-editors/sagews-editor/convert.tsx | 15 +++ .../sagews-editor/document-worksheet.tsx | 61 ---------- .../frame-editors/sagews-editor/editor.ts | 84 ++------------ .../sagews-editor/hidden-input-cell.tsx | 16 --- .../sagews-editor/hidden-output-cell.tsx | 16 --- .../sagews-editor/input-cell.tsx | 32 ------ .../sagews-editor/output-cell.tsx | 18 --- .../frame-editors/sagews-editor/print.tsx | 50 --------- 12 files changed, 36 insertions(+), 496 deletions(-) delete mode 100644 src/packages/frontend/frame-editors/sagews-editor/cell-worksheet.tsx delete mode 100644 src/packages/frontend/frame-editors/sagews-editor/code-executor.ts create mode 100644 src/packages/frontend/frame-editors/sagews-editor/convert.tsx delete mode 100644 src/packages/frontend/frame-editors/sagews-editor/document-worksheet.tsx delete mode 100644 src/packages/frontend/frame-editors/sagews-editor/hidden-input-cell.tsx delete mode 100644 src/packages/frontend/frame-editors/sagews-editor/hidden-output-cell.tsx delete mode 100644 src/packages/frontend/frame-editors/sagews-editor/input-cell.tsx delete mode 100644 src/packages/frontend/frame-editors/sagews-editor/output-cell.tsx delete mode 100644 src/packages/frontend/frame-editors/sagews-editor/print.tsx diff --git a/src/packages/frontend/frame-editors/frame-tree/title-bar.tsx b/src/packages/frontend/frame-editors/frame-tree/title-bar.tsx index 811cc86548..bc320359d2 100644 --- a/src/packages/frontend/frame-editors/frame-tree/title-bar.tsx +++ b/src/packages/frontend/frame-editors/frame-tree/title-bar.tsx @@ -16,7 +16,6 @@ import type { MenuProps } from "antd/lib"; import { List } from "immutable"; import { useMemo, useRef } from "react"; import { useIntl } from "react-intl"; - import { CSS, redux, @@ -71,6 +70,11 @@ import TitleBarTour from "./title-bar-tour"; import { ConnectionStatus, EditorDescription, EditorSpec } from "./types"; import { TITLE_BAR_BORDER } from "./style"; +const ALWAYS_HIDE_AI = new Set(['sagews']); +function alwaysHideAi(ext:string) { + return ALWAYS_HIDE_AI.has(ext); +} + // Certain special frame editors (e.g., for latex) have extra // actions that are not defined in the base code editor actions. // In all cases, we check these are actually defined before calling @@ -589,7 +593,8 @@ export function FrameTitleBar(props: FrameTitleBarProps) { function renderAssistant(noLabel, where: "main" | "popover"): Rendered { if ( !manageCommands.isVisible("chatgpt") || - !redux.getStore("projects").hasLanguageModelEnabled(props.project_id) + !redux.getStore("projects").hasLanguageModelEnabled(props.project_id) || + alwaysHideAi(filename_extension(props.path)) ) { return; } @@ -726,8 +731,8 @@ export function FrameTitleBar(props: FrameTitleBarProps) { label === APPLICATION_MENU ? manageCommands.applicationMenuTitle() : isIntlMessage(label) - ? intl.formatMessage(label) - : label + ? intl.formatMessage(label) + : label } items={v} /> diff --git a/src/packages/frontend/frame-editors/sagews-editor/actions.ts b/src/packages/frontend/frame-editors/sagews-editor/actions.ts index aabc3f760d..f4d5164721 100644 --- a/src/packages/frontend/frame-editors/sagews-editor/actions.ts +++ b/src/packages/frontend/frame-editors/sagews-editor/actions.ts @@ -6,71 +6,23 @@ /* Sage Worksheet Editor Actions */ -import { Map } from "immutable"; - import { Actions, CodeEditorState } from "../code-editor/actions"; -//import { print_html } from "../frame-tree/print"; import { FrameTree } from "../frame-tree/types"; import { Store } from "../../app-framework"; -import { CellObject } from "./types"; - -import { code_executor, CodeExecutor } from "./code-executor"; - -interface SageWorksheetEditorState extends CodeEditorState { - /* cells: { - [key: string]: CellObject; - }; - */ - cells: any; -} +interface SageWorksheetEditorState extends CodeEditorState {} export class SageWorksheetActions extends Actions { - protected doctype: string = "syncdb"; - protected primary_keys: string[] = ["type", "id"]; - protected string_cols: string[] = ["input"]; public store: Store; _init2(): void { - this.setState({ cells: {} }); - this._syncstring.on("change", (keys) => { - keys.forEach((value) => { - const id = value.get("id"); - if (id) { - let cells = this.store.get("cells"); - cells = cells.set(id, this._get_cell(id)); - this.setState({ cells: cells }); - } - }); + console.log("change", keys); }); } - set_cell(cell: CellObject): void { - (cell as any).type = "cell"; - this._syncstring.set(cell); - } - - private _get_cell(id: string): Map { - return this._syncstring.get_one({ id: id, type: "cell" }); - } - _raw_default_frame_tree(): FrameTree { - return { type: "cells" }; - } - - print(id: string): void { - console.warn("TODO -- print", id); - } - - _code_executor( - code: string, - data?: object, - cell_id?: string, - preparse?: boolean - ): CodeExecutor { - // todo: if cell_id is given, ensure is valid. - return code_executor({ path: this.path, code, data, cell_id, preparse }); + return { type: "convert" }; } } diff --git a/src/packages/frontend/frame-editors/sagews-editor/cell-worksheet.tsx b/src/packages/frontend/frame-editors/sagews-editor/cell-worksheet.tsx deleted file mode 100644 index 69725a6807..0000000000 --- a/src/packages/frontend/frame-editors/sagews-editor/cell-worksheet.tsx +++ /dev/null @@ -1,68 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { Map } from "immutable"; -import { Component, Rendered, rtypes, rclass } from "../../app-framework"; - -import { input_is_hidden, output_is_hidden } from "./flags"; - -import { InputCell } from "./input-cell"; -import { OutputCell } from "./output-cell"; -import { HiddenInputCell } from "./hidden-input-cell"; -import { HiddenOutputCell } from "./hidden-output-cell"; - -interface Props { - // reduxProps: - cells: Map>; -} - -class CellWorksheet extends Component { - static reduxProps({ name }) { - return { - [name]: { - cells: rtypes.immutable.Map, - }, - }; - } - - render_input_cell(cell: Map): Rendered { - if (input_is_hidden(cell.get("flags"))) { - return ; - } else { - return ; - } - } - - render_output_cell(cell: Map): Rendered { - if (output_is_hidden(cell.get("flags"))) { - return ; - } else { - return ( - - ); - } - } - - render_cells(): Rendered[] { - const v: Rendered[] = []; - // TODO: sort by position. - this.props.cells.forEach((cell, id) => { - v.push( -
              -
              {this.render_input_cell(cell)}
              -
              {this.render_output_cell(cell)}
              -
              , - ); - }); - return v; - } - - render(): Rendered { - return
              {this.render_cells()}
              ; - } -} - -const tmp0 = rclass(CellWorksheet); -export { tmp0 as CellWorksheet }; diff --git a/src/packages/frontend/frame-editors/sagews-editor/code-executor.ts b/src/packages/frontend/frame-editors/sagews-editor/code-executor.ts deleted file mode 100644 index 2b7ea1fce0..0000000000 --- a/src/packages/frontend/frame-editors/sagews-editor/code-executor.ts +++ /dev/null @@ -1,105 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { EventEmitter } from "events"; - -interface ExecutionRequest { - path: string; - code: string; - data?: object; - cell_id?: string; - preparse?: boolean; -} - -type States = "init" | "running" | "interrupted" | "done" | "closed"; - -export interface CodeExecutor { - start(): Promise; - interrupt(): Promise; - close(): void; -} - -const MOCK = true; - -export function code_executor(req: ExecutionRequest): CodeExecutor { - if (MOCK) { - return new CodeExecutorMock(req); - } else { - return new CodeExecutorProject(req); - } -} - -/* -Emits: - - 'output', mesg -- as each output message arrives - - 'state', state -- for each state change -*/ - -abstract class CodeExecutorAbstract - extends EventEmitter - implements CodeExecutor -{ - protected state: States = "init"; - protected request: ExecutionRequest; - - constructor(request: ExecutionRequest) { - super(); - this.request = request; - } - - protected _set_state(state: States): void { - this.state = state; - this.emit("state", state); - } - - // start code running - abstract start(): Promise; - - // interrupt running code - abstract interrupt(): Promise; - - // call to close and free any used space - abstract close(): void; -} - -class CodeExecutorProject extends CodeExecutorAbstract { - // start code running - async start(): Promise {} - - // interrupt running code - async interrupt(): Promise {} - - // call to close and free any used space - close(): void { - this._set_state("closed"); - } -} - -class CodeExecutorMock extends CodeExecutorAbstract { - // start code running - async start(): Promise { - console.log("start", this.request); - switch (this.request.code) { - case "2+2": - this.emit("output", { stdout: "4" }); - break; - default: - this.emit("output", { - stderr: `Unknown mock code "${this.request.code}"`, - }); - } - this._set_state("done"); - } - - // interrupt running code - async interrupt(): Promise { - this._set_state("done"); - } - - // call to close and free any used space - close(): void { - this._set_state("closed"); - } -} diff --git a/src/packages/frontend/frame-editors/sagews-editor/convert.tsx b/src/packages/frontend/frame-editors/sagews-editor/convert.tsx new file mode 100644 index 0000000000..950c6cbd8e --- /dev/null +++ b/src/packages/frontend/frame-editors/sagews-editor/convert.tsx @@ -0,0 +1,15 @@ +import { Button } from "antd"; + +export default function Convert({}) { + return ( +
              +
              +

              Sage Worksheets are Deprecated

              +
              + +
              +
              + ); +} diff --git a/src/packages/frontend/frame-editors/sagews-editor/document-worksheet.tsx b/src/packages/frontend/frame-editors/sagews-editor/document-worksheet.tsx deleted file mode 100644 index 31bcc5b22c..0000000000 --- a/src/packages/frontend/frame-editors/sagews-editor/document-worksheet.tsx +++ /dev/null @@ -1,61 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { Map } from "immutable"; -import { Component, Rendered, rtypes, rclass } from "../../app-framework"; -import { input_is_hidden, output_is_hidden } from "./flags"; - -interface Props { - // reduxProps: - cells: Map>; -} - -function cells_to_value(cells: Map>): string { - let value: string = ""; // todo: sort matters... - cells.forEach((cell, _) => { - if (input_is_hidden(cell.get("flags"))) { - value += "hidden"; - } else { - value += cell.get("input"); - } - value += "\n---\n"; - if (output_is_hidden(cell.get("flags"))) { - value += "hidden"; - } else { - value += JSON.stringify(cell.get("output", Map()).toJS()); - } - value += "\n===\n"; - return; - }); - return value; -} - -class DocumentWorksheet extends Component { - static reduxProps({ name }) { - return { - [name]: { - cells: rtypes.immutable.Map, - }, - }; - } - - render_doc(): Rendered { - return ( -