new wshrpc mechanism (#112)

lots of changes. new wshrpc implementation. unify websocket, web,
blockcontroller, domain sockets, and terminal inputs to all use the new
rpc system.

lots of moving files around to deal with circular dependencies

use new wshrpc as a client in wsh cmd

frontend/app/store/global.ts

@@ -5,6 +5,7 @@ import { LayoutTreeAction, LayoutTreeActionType, LayoutTreeInsertNodeAction, new
 import { getLayoutStateAtomForTab } from "@/faraday/lib/layoutAtom";
 import { layoutTreeStateReducer } from "@/faraday/lib/layoutState";
 
+import { handleIncomingRpcMessage } from "@/app/store/wshrpc";
 import * as layoututil from "@/util/layoututil";
 import { produce } from "immer";
 import * as jotai from "jotai";
@@ -27,8 +28,8 @@ let globalClientId: string = null;
 if (typeof window !== "undefined") {
     // this if statement allows us to use the code in nodejs as well
     const urlParams = new URLSearchParams(window.location.search);
-    globalWindowId = urlParams.get("windowid") || "74eba2d0-22fc-4221-82ad-d028dd496342";
-    globalClientId = urlParams.get("clientid") || "f4bc1713-a364-41b3-a5c4-b000ba10d622";
+    globalWindowId = urlParams.get("windowid");
+    globalClientId = urlParams.get("clientid");
 }
 const windowIdAtom = jotai.atom(null) as jotai.PrimitiveAtom<string>;
 const clientIdAtom = jotai.atom(null) as jotai.PrimitiveAtom<string>;
@@ -223,6 +224,11 @@ function handleWSEventMessage(msg: WSEventType) {
         }
         return;
     }
+    if (msg.eventtype == "rpc") {
+        const rpcMsg: RpcMessage = msg.data;
+        handleIncomingRpcMessage(rpcMsg);
+        return;
+    }
     if (msg.eventtype == "layoutaction") {
         const layoutAction: WSLayoutActionData = msg.data;
         if (layoutAction.actiontype == LayoutTreeActionType.InsertNode) {

frontend/app/store/services.ts

@@ -13,14 +13,9 @@ class BlockServiceType {
     SaveTerminalState(arg2: string, arg3: string, arg4: string, arg5: number): Promise<void> {
         return WOS.callBackendService("block", "SaveTerminalState", Array.from(arguments))
     }
-
-    // send command to block
-    SendCommand(cmd: string, arg3: BlockCommand): Promise<void> {
-        return WOS.callBackendService("block", "SendCommand", Array.from(arguments))
-    }
 }
 
-export const BlockService = new BlockServiceType()
+export const BlockService = new BlockServiceType();
 
 // clientservice.ClientService (client)
 class ClientServiceType {
@@ -44,7 +39,7 @@ class ClientServiceType {
     }
 }
 
-export const ClientService = new ClientServiceType()
+export const ClientService = new ClientServiceType();
 
 // fileservice.FileService (file)
 class FileServiceType {
@@ -71,7 +66,7 @@ class FileServiceType {
     }
 }
 
-export const FileService = new FileServiceType()
+export const FileService = new FileServiceType();
 
 // objectservice.ObjectService (object)
 class ObjectServiceType {
@@ -126,7 +121,7 @@ class ObjectServiceType {
     }
 }
 
-export const ObjectService = new ObjectServiceType()
+export const ObjectService = new ObjectServiceType();
 
 // windowservice.WindowService (window)
 class WindowServiceType {
@@ -150,5 +145,5 @@ class WindowServiceType {
     }
 }
 
-export const WindowService = new WindowServiceType()
+export const WindowService = new WindowServiceType();
 

frontend/app/store/wos.ts

@@ -3,8 +3,10 @@
 
 // WaveObjectStore
 
+import { sendRpcCommand } from "@/app/store/wshrpc";
 import * as jotai from "jotai";
 import * as React from "react";
+import { v4 as uuidv4 } from "uuid";
 import { atoms, getBackendHostPort, globalStore } from "./global";
 import * as services from "./services";
 
@@ -103,6 +105,50 @@ function callBackendService(service: string, method: string, args: any[], noUICo
     return prtn;
 }
 
+function callWshServerRpc(
+    command: string,
+    data: any,
+    meta: WshServerCommandMeta,
+    opts: WshRpcCommandOpts
+): Promise<any> {
+    let msg: RpcMessage = {
+        command: command,
+        data: data,
+    };
+    if (!opts?.noresponse) {
+        msg.reqid = uuidv4();
+    }
+    if (opts?.timeout) {
+        msg.timeout = opts.timeout;
+    }
+    if (meta.commandtype != "call") {
+        throw new Error("unimplemented wshserver commandtype " + meta.commandtype);
+    }
+    const rpcGen = sendRpcCommand(msg);
+    if (rpcGen == null) {
+        return null;
+    }
+    let resolveFn: (value: any) => void;
+    let rejectFn: (reason?: any) => void;
+    const prtn = new Promise((resolve, reject) => {
+        resolveFn = resolve;
+        rejectFn = reject;
+    });
+    const respMsg = rpcGen.next(true); // pass true to force termination of rpc after 1 response (not streaing)
+    respMsg.then((msg: IteratorResult<RpcMessage, void>) => {
+        if (msg.value == null) {
+            resolveFn(null);
+        }
+        let respMsg: RpcMessage = msg.value as RpcMessage;
+        if (respMsg.error != null) {
+            rejectFn(new Error(respMsg.error));
+            return;
+        }
+        resolveFn(respMsg.data);
+    });
+    return prtn;
+}
+
 const waveObjectValueCache = new Map<string, WaveObjectValue<any>>();
 
 function clearWaveObjectCache() {
@@ -320,6 +366,7 @@ function setObjectValue<T extends WaveObj>(value: T, setFn?: jotai.Setter, pushT
 
 export {
     callBackendService,
+    callWshServerRpc,
     cleanWaveObjectCache,
     clearWaveObjectCache,
     getObjectValue,

frontend/app/store/ws.ts

@@ -3,19 +3,9 @@
 
 import * as jotai from "jotai";
 import { sprintf } from "sprintf-js";
-import { v4 as uuidv4 } from "uuid";
 
 const MaxWebSocketSendSize = 1024 * 1024; // 1MB
 
-type RpcEntry = {
-    reqId: string;
-    startTs: number;
-    method: string;
-    resolve: (any) => void;
-    reject: (any) => void;
-    promise: Promise<any>;
-};
-
 type JotaiStore = {
     get: <Value>(atom: jotai.Atom<Value>) => Value;
     set: <Value>(atom: jotai.WritableAtom<Value, [Value], void>, value: Value) => void;
@@ -35,7 +25,6 @@ class WSControl {
     authKey: string;
     baseHostPort: string;
     lastReconnectTime: number = 0;
-    rpcMap: Map<string, RpcEntry> = new Map(); // reqId -> RpcEntry
     jotaiStore: JotaiStore;
 
     constructor(
@@ -169,10 +158,6 @@ class WSControl {
             this.reconnectTimes = 0;
             return;
         }
-        if (eventData.type == "rpcresp") {
-            this.handleRpcResp(eventData);
-            return;
-        }
         if (this.messageCallback) {
             try {
                 this.messageCallback(eventData);
@@ -189,60 +174,20 @@ class WSControl {
         this.wsConn.send(JSON.stringify({ type: "ping", stime: Date.now() }));
     }
 
-    handleRpcResp(data: any) {
-        let reqId = data.reqid;
-        let rpcEntry = this.rpcMap.get(reqId);
-        if (rpcEntry == null) {
-            console.log("rpcresp for unknown reqid", reqId);
-            return;
-        }
-        this.rpcMap.delete(reqId);
-        console.log("rpcresp", rpcEntry.method, Math.round(performance.now() - rpcEntry.startTs) + "ms");
-        if (data.error != null) {
-            rpcEntry.reject(data.error);
-        } else {
-            rpcEntry.resolve(data.data);
-        }
-    }
-
-    doRpc(method: string, params: any[]): Promise<any> {
-        if (!this.isOpen()) {
-            return Promise.reject("not connected");
-        }
-        let reqId = uuidv4();
-        let req = { type: "rpc", method: method, params: params, reqid: reqId };
-        let rpcEntry: RpcEntry = {
-            method: method,
-            startTs: performance.now(),
-            reqId: reqId,
-            resolve: null,
-            reject: null,
-            promise: null,
-        };
-        let rpcPromise = new Promise((resolve, reject) => {
-            rpcEntry.resolve = resolve;
-            rpcEntry.reject = reject;
-        });
-        rpcEntry.promise = rpcPromise;
-        this.rpcMap.set(reqId, rpcEntry);
-        this.wsConn.send(JSON.stringify(req));
-        return rpcPromise;
-    }
-
-    sendMessage(data: any) {
+    sendMessage(data: WSCommandType) {
         if (!this.isOpen()) {
             return;
         }
         let msg = JSON.stringify(data);
         const byteSize = new Blob([msg]).size;
         if (byteSize > MaxWebSocketSendSize) {
-            console.log("ws message too large", byteSize, data.type, msg.substring(0, 100));
+            console.log("ws message too large", byteSize, data.wscommand, msg.substring(0, 100));
             return;
         }
         this.wsConn.send(msg);
     }
 
-    pushMessage(data: any) {
+    pushMessage(data: WSCommandType) {
         if (!this.isOpen()) {
             this.msgQueue.push(data);
             return;

frontend/app/store/wshrpc.ts

@@ -0,0 +1,88 @@
+// Copyright 2024, Command Line Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+import { globalWS } from "./global";
+
+type RpcEntry = {
+    reqId: string;
+    startTs: number;
+    command: string;
+    msgFn: (msg: RpcMessage) => void;
+};
+
+let openRpcs = new Map<string, RpcEntry>();
+
+async function* rpcResponseGenerator(
+    command: string,
+    reqid: string,
+    timeout: number
+): AsyncGenerator<RpcMessage, void, boolean> {
+    const msgQueue: RpcMessage[] = [];
+    let signalFn: () => void;
+    let signalPromise = new Promise<void>((resolve) => (signalFn = resolve));
+    let timeoutId: NodeJS.Timeout = null;
+    if (timeout > 0) {
+        timeoutId = setTimeout(() => {
+            msgQueue.push({ resid: reqid, error: "EC-TIME: timeout waiting for response" });
+            signalFn();
+        }, timeout);
+    }
+    const msgFn = (msg: RpcMessage) => {
+        msgQueue.push(msg);
+        signalFn();
+        // reset signal promise
+        signalPromise = new Promise<void>((resolve) => (signalFn = resolve));
+    };
+    openRpcs.set(reqid, {
+        reqId: reqid,
+        startTs: Date.now(),
+        command: command,
+        msgFn: msgFn,
+    });
+    try {
+        while (true) {
+            while (msgQueue.length > 0) {
+                const msg = msgQueue.shift()!;
+                const shouldTerminate = yield msg;
+                if (shouldTerminate || !msg.cont) {
+                    return;
+                }
+            }
+            await signalPromise;
+        }
+    } finally {
+        openRpcs.delete(reqid);
+        if (timeoutId != null) {
+            clearTimeout(timeoutId);
+        }
+    }
+}
+
+function sendRpcCommand(msg: RpcMessage): AsyncGenerator<RpcMessage, void, boolean> {
+    let wsMsg: WSRpcCommand = { wscommand: "rpc", message: msg };
+    globalWS.pushMessage(wsMsg);
+    if (msg.reqid == null) {
+        return null;
+    }
+    return rpcResponseGenerator(msg.command, msg.reqid, msg.timeout);
+}
+
+function handleIncomingRpcMessage(msg: RpcMessage) {
+    const isRequest = msg.command != null || msg.reqid != null;
+    if (isRequest) {
+        console.log("rpc request not supported", msg);
+        return;
+    }
+    if (msg.resid == null) {
+        console.log("rpc response missing resid", msg);
+        return;
+    }
+    const entry = openRpcs.get(msg.resid);
+    if (entry == null) {
+        console.log("rpc response generator not found", msg);
+        return;
+    }
+    entry.msgFn(msg);
+}
+
+export { handleIncomingRpcMessage, sendRpcCommand };

frontend/app/store/wshserver.ts

@@ -0,0 +1,72 @@
+// Copyright 2024, Command Line Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+// generated by cmd/generate/main-generate.go
+
+import * as WOS from "./wos";
+
+// WshServerCommandToDeclMap
+class WshServerType {
+    // command "controller:input" [call]
+	BlockInputCommand(data: CommandBlockInputData, opts?: WshRpcCommandOpts): Promise<void> {
+        const meta: WshServerCommandMeta = {commandtype: "call"};
+        return WOS.callWshServerRpc("controller:input", data, meta, opts);
+    }
+
+    // command "controller:restart" [call]
+	BlockRestartCommand(data: CommandBlockRestartData, opts?: WshRpcCommandOpts): Promise<void> {
+        const meta: WshServerCommandMeta = {commandtype: "call"};
+        return WOS.callWshServerRpc("controller:restart", data, meta, opts);
+    }
+
+    // command "createblock" [call]
+	CreateBlockCommand(data: CommandCreateBlockData, opts?: WshRpcCommandOpts): Promise<ORef> {
+        const meta: WshServerCommandMeta = {commandtype: "call"};
+        return WOS.callWshServerRpc("createblock", data, meta, opts);
+    }
+
+    // command "file:append" [call]
+	AppendFileCommand(data: CommandAppendFileData, opts?: WshRpcCommandOpts): Promise<void> {
+        const meta: WshServerCommandMeta = {commandtype: "call"};
+        return WOS.callWshServerRpc("file:append", data, meta, opts);
+    }
+
+    // command "file:appendijson" [call]
+	AppendIJsonCommand(data: CommandAppendIJsonData, opts?: WshRpcCommandOpts): Promise<void> {
+        const meta: WshServerCommandMeta = {commandtype: "call"};
+        return WOS.callWshServerRpc("file:appendijson", data, meta, opts);
+    }
+
+    // command "getmeta" [call]
+	GetMetaCommand(data: CommandGetMetaData, opts?: WshRpcCommandOpts): Promise<MetaType> {
+        const meta: WshServerCommandMeta = {commandtype: "call"};
+        return WOS.callWshServerRpc("getmeta", data, meta, opts);
+    }
+
+    // command "message" [call]
+	MessageCommand(data: CommandMessageData, opts?: WshRpcCommandOpts): Promise<void> {
+        const meta: WshServerCommandMeta = {commandtype: "call"};
+        return WOS.callWshServerRpc("message", data, meta, opts);
+    }
+
+    // command "resolveids" [call]
+	ResolveIdsCommand(data: CommandResolveIdsData, opts?: WshRpcCommandOpts): Promise<CommandResolveIdsRtnData> {
+        const meta: WshServerCommandMeta = {commandtype: "call"};
+        return WOS.callWshServerRpc("resolveids", data, meta, opts);
+    }
+
+    // command "setmeta" [call]
+	SetMetaCommand(data: CommandSetMetaData, opts?: WshRpcCommandOpts): Promise<void> {
+        const meta: WshServerCommandMeta = {commandtype: "call"};
+        return WOS.callWshServerRpc("setmeta", data, meta, opts);
+    }
+
+    // command "setview" [call]
+	BlockSetViewCommand(data: CommandBlockSetViewData, opts?: WshRpcCommandOpts): Promise<void> {
+        const meta: WshServerCommandMeta = {commandtype: "call"};
+        return WOS.callWshServerRpc("setview", data, meta, opts);
+    }
+
+}
+
+export const WshServer = new WshServerType();