Restart command (#253)

* working on cmd restart logic

* button to restart command

* bind Cmd-R to restart selected command, and Cmd-Shift-R to restart last command.  Browser Refresh is now Option-R.  also fix 'clear' command to not delete running commands (like archive).  some small changes to keyboard utility code to always set 'alt' and 'meta' appropriately.  use 'cmd' and 'option' for crossplatform bindings

* focus restarted line

* update termopts, use current winsize to set termopts for new command

* add cmd.restartts to track restart time

* display restarted time in line w/ tooltip with original time

* add restartts to line:show

src/app/line/linecomps.tsx

@@ -360,6 +360,12 @@ class LineCmd extends React.Component<
         GlobalCommandRunner.lineDelete(line.lineid, true);
     }
 
+    @boundMethod
+    clickRestart() {
+        let { line } = this.props;
+        GlobalCommandRunner.lineRestart(line.lineid, true);
+    }
+
     @boundMethod
     clickMinimize() {
         mobx.action(() => {
@@ -467,12 +473,21 @@ class LineCmd extends React.Component<
     renderMeta1(cmd: Cmd) {
         let { line } = this.props;
         let termOpts = cmd.getTermOpts();
-        let formattedTime = lineutil.getLineDateTimeStr(line.ts);
+        let formattedTime: string = "";
+        let restartTs = cmd.getRestartTs();
+        let timeTitle: string = null;
+        if (restartTs != null && restartTs > 0) {
+            formattedTime = "restarted @ " + lineutil.getLineDateTimeStr(restartTs);
+            timeTitle = "original start time " + lineutil.getLineDateTimeStr(line.ts);
+        }
+        else {
+            formattedTime = lineutil.getLineDateTimeStr(line.ts);
+        }
         let renderer = line.renderer;
         return (
             <div key="meta1" className="meta meta-line1">
                 <SmallLineAvatar line={line} cmd={cmd} />
-                <div className="ts">{formattedTime}</div>
+                <div title={timeTitle} className="ts">{formattedTime}</div>
                 <div>&nbsp;</div>
                 <If condition={!isBlank(renderer) && renderer != "terminal"}>
                     <div className="renderer">
@@ -665,6 +680,9 @@ class LineCmd extends React.Component<
                         {this.renderMeta1(cmd)}
                         <If condition={!hidePrompt}>{this.renderCmdText(cmd)}</If>
                     </div>
+                    <div key="restart" title="Restart Command" className="line-icon" onClick={this.clickRestart}>
+                        <i className="fa-sharp fa-regular fa-arrows-rotate"/>
+                    </div>
                     <div key="delete" title="Delete Line (&#x2318;D)" className="line-icon" onClick={this.clickDelete}>
                         <i className="fa-sharp fa-regular fa-trash" />
                     </div>

src/electron/emain.ts

@@ -36,7 +36,7 @@ ensureDir(waveHome);
 // these are either "darwin/amd64" or "darwin/arm64"
 // normalize darwin/x64 to darwin/amd64 for GOARCH compatibility
 let unamePlatform = process.platform;
-let unameArch = process.arch;
+let unameArch: string = process.arch;
 if (unameArch == "x64") {
     unameArch = "amd64";
 }
@@ -189,15 +189,21 @@ let menuTemplate = [
             { role: "quit" },
         ],
     },
-    {
-        label: "File",
-        submenu: [{ role: "close" }],
-    },
     {
         role: "editMenu",
     },
     {
         role: "viewMenu",
+        submenu: [
+            { role: "reload", accelerator: "Option+R" },
+            { role: "toggleDevTools" },
+            { type: "separator" },
+            { role: "resetZoom" },
+            { role: "zoomIn" },
+            { role: "zoomOut" },
+            { type: "separator" },
+            { role: "togglefullscreen" },
+        ],
     },
     {
         role: "windowMenu",
@@ -284,14 +290,11 @@ function createMainWindow(clientData) {
             } else {
                 win.webContents.toggleDevTools();
             }
-
             return;
         }
         if (checkKeyPressed(waveEvent, "Cmd:r")) {
-            if (input.shift) {
+            e.preventDefault();
-                e.preventDefault();
+            win.webContents.send("r-cmd", mods);
-                win.reload();
-            }
             return;
         }
         if (checkKeyPressed(waveEvent, "Cmd:l")) {

src/electron/preload.js

@@ -19,6 +19,7 @@ contextBridge.exposeInMainWorld("api", {
     onHCmd: (callback) => ipcRenderer.on("h-cmd", callback),
     onWCmd: (callback) => ipcRenderer.on("w-cmd", callback),
     onPCmd: (callback) => ipcRenderer.on("p-cmd", callback),
+    onRCmd: (callback) => ipcRenderer.on("r-cmd", callback),
     onMetaArrowUp: (callback) => ipcRenderer.on("meta-arrowup", callback),
     onMetaArrowDown: (callback) => ipcRenderer.on("meta-arrowdown", callback),
     onMetaPageUp: (callback) => ipcRenderer.on("meta-pageup", callback),

src/model/model.ts

@@ -200,6 +200,7 @@ type ElectronApi = {
     onLCmd: (callback: (mods: KeyModsType) => void) => void;
     onHCmd: (callback: (mods: KeyModsType) => void) => void;
     onPCmd: (callback: (mods: KeyModsType) => void) => void;
+    onRCmd: (callback: (mods: KeyModsType) => void) => void;
     onWCmd: (callback: (mods: KeyModsType) => void) => void;
     onMenuItemAbout: (callback: () => void) => void;
     onMetaArrowUp: (callback: () => void) => void;
@@ -249,6 +250,10 @@ class Cmd {
         })();
     }
 
+    getRestartTs(): number {
+        return this.data.get().restartts;
+    }
+
     getAsWebCmd(lineid: string): WebCmd {
         let cmd = this.data.get();
         let remote = GlobalModel.getRemote(this.remote.remoteid);
@@ -3403,6 +3408,7 @@ class Model {
         getApi().onHCmd(this.onHCmd.bind(this));
         getApi().onPCmd(this.onPCmd.bind(this));
         getApi().onWCmd(this.onWCmd.bind(this));
+        getApi().onRCmd(this.onRCmd.bind(this));
         getApi().onMenuItemAbout(this.onMenuItemAbout.bind(this));
         getApi().onMetaArrowUp(this.onMetaArrowUp.bind(this));
         getApi().onMetaArrowDown(this.onMetaArrowDown.bind(this));
@@ -3674,6 +3680,9 @@ class Model {
     }
 
     onWCmd(e: any, mods: KeyModsType) {
+        if (this.activeMainView.get() != "session") {
+            return;
+        }
         let activeScreen = this.getActiveScreen();
         if (activeScreen == null) {
             return;
@@ -3690,6 +3699,27 @@ class Model {
         });
     }
 
+    onRCmd(e: any, mods: KeyModsType) {
+        if (this.activeMainView.get() != "session") {
+            return;
+        }
+        let activeScreen = this.getActiveScreen();
+        if (activeScreen == null) {
+            return;
+        }
+        if (mods.shift) {
+            // restart last line
+            GlobalCommandRunner.lineRestart("E", true);
+        } else {
+            // restart selected line
+            let selectedLine = activeScreen.selectedLine.get();
+            if (selectedLine == null || selectedLine == 0) {
+                return;
+            }
+            GlobalCommandRunner.lineRestart(String(selectedLine), true);
+        }
+    }
+
     clearModals(): boolean {
         let didSomething = false;
         mobx.action(() => {
@@ -4151,12 +4181,28 @@ class Model {
         return session.getActiveScreen();
     }
 
+    handleCmdRestart(cmd: CmdDataType) {
+        if (cmd == null || !cmd.restarted) {
+            return;
+        }
+        let screen = this.screenMap.get(cmd.screenid);
+        if (screen == null) {
+            return;
+        }
+        let termWrap = screen.getTermWrap(cmd.lineid);
+        if (termWrap == null) {
+            return;
+        }
+        termWrap.reload(0);
+    }
+
     addLineCmd(line: LineType, cmd: CmdDataType, interactive: boolean) {
         let slines = this.getScreenLinesById(line.screenid);
         if (slines == null) {
             return;
         }
         slines.addLineCmd(line, cmd, interactive);
+        this.handleCmdRestart(cmd);
     }
 
     updateCmd(cmd: CmdDataType) {
@@ -4164,6 +4210,7 @@ class Model {
         if (slines != null) {
             slines.updateCmd(cmd);
         }
+        this.handleCmdRestart(cmd);
     }
 
     isInfoUpdate(update: UpdateMessage): boolean {
@@ -4601,6 +4648,10 @@ class CommandRunner {
         return GlobalModel.submitCommand("line", "delete", [lineArg], { nohist: "1" }, interactive);
     }
 
+    lineRestart(lineArg: string, interactive: boolean): Promise<CommandRtnType> {
+        return GlobalModel.submitCommand("line", "restart", [lineArg], { nohist: "1" }, interactive);
+    }
+
     lineSet(lineArg: string, opts: { renderer?: string }): Promise<CommandRtnType> {
         let kwargs = { nohist: "1" };
         if ("renderer" in opts) {

src/types/types.ts

@@ -245,12 +245,14 @@ type CmdDataType = {
     status: string;
     cmdpid: number;
     remotepid: number;
+    restartts: number;
     donets: number;
     exitcode: number;
     durationms: number;
     runout: any[];
     rtnstate: boolean;
     remove?: boolean;
+    restarted?: boolean;
 };
 
 type PtyDataUpdateType = {

src/util/keyutil.ts

@@ -7,6 +7,8 @@ type KeyPressDecl = {
         Option?: boolean;
         Shift?: boolean;
         Ctrl?: boolean;
+        Alt?: boolean;
+        Meta?: boolean;
     };
     key: string;
 };
@@ -30,6 +32,10 @@ function parseKeyDescription(keyDescription: string): KeyPressDecl {
             rtn.mods.Ctrl = true;
         } else if (key == "Option") {
             rtn.mods.Option = true;
+        } else if (key == "Alt") {
+            rtn.mods.Alt = true;
+        } else if (key == "Meta") {
+            rtn.mods.Meta = true;
         } else {
             rtn.key = key;
             if (key.length == 1) {
@@ -49,7 +55,7 @@ function parseKeyDescription(keyDescription: string): KeyPressDecl {
 
 function checkKeyPressed(event: WaveKeyboardEvent, description: string): boolean {
     let keyPress = parseKeyDescription(description);
-    if (keyPress.mods.Option && !event.alt) {
+    if (keyPress.mods.Option && !event.option) {
         return false;
     }
     if (keyPress.mods.Cmd && !event.cmd) {
@@ -61,6 +67,12 @@ function checkKeyPressed(event: WaveKeyboardEvent, description: string): boolean
     if (keyPress.mods.Ctrl && !event.control) {
         return false;
     }
+    if (keyPress.mods.Alt && !event.alt) {
+        return false;
+    }
+    if (keyPress.mods.Meta && !event.meta) {
+        return false;
+    }
     let eventKey = event.key;
     let descKey = keyPress.key;
     if (eventKey.length == 1 && /[A-Z]/.test(eventKey.charAt(0))) {
@@ -78,7 +90,8 @@ function checkKeyPressed(event: WaveKeyboardEvent, description: string): boolean
     return true;
 }
 
-type ModKeyStrs = "Cmd" | "Option" | "Shift" | "Ctrl";
+// Cmd and Option are portable between Mac and Linux/Windows
+type ModKeyStrs = "Cmd" | "Option" | "Shift" | "Ctrl" | "Alt" | "Meta";
 
 interface WaveKeyboardEvent {
     type: string;
@@ -105,7 +118,16 @@ interface WaveKeyboardEvent {
     /**
      * Equivalent to KeyboardEvent.metaKey.
      */
+    meta: boolean;
+    /**
+     * cmd is special, on mac it is meta, on windows it is alt
+     */
     cmd: boolean;
+    /**
+     * option is special, on mac it is alt, on windows it is meta
+     */
+    option: boolean;
+
     repeat: boolean;
     /**
      * Equivalent to KeyboardEvent.location.
@@ -117,12 +139,10 @@ function adaptFromReactOrNativeKeyEvent(event: React.KeyboardEvent | KeyboardEve
     let rtn: WaveKeyboardEvent = {} as WaveKeyboardEvent;
     rtn.control = event.ctrlKey;
     rtn.shift = event.shiftKey;
-    if (PLATFORM == PlatformMacOS) {
+    rtn.cmd = (PLATFORM == PlatformMacOS ? event.metaKey : event.altKey);
-        rtn.cmd = event.metaKey;
+    rtn.option = (PLATFORM == PlatformMacOS ? event.altKey : event.metaKey);
-        rtn.alt = event.altKey;
+    rtn.meta = event.metaKey;
-    } else {
+    rtn.alt = event.altKey;
-        rtn.cmd = event.altKey;
-    }
     rtn.code = event.code;
     rtn.key = event.key;
     rtn.location = event.location;
@@ -135,12 +155,10 @@ function adaptFromElectronKeyEvent(event: any): WaveKeyboardEvent {
     let rtn: WaveKeyboardEvent = {} as WaveKeyboardEvent;
     rtn.type = event.type;
     rtn.control = event.control;
-    if (PLATFORM == PlatformMacOS) {
+    rtn.cmd = (PLATFORM == PlatformMacOS ? event.meta : event.alt)
-        rtn.cmd = event.meta;
+    rtn.option = (PLATFORM == PlatformMacOS ? event.alt : event.meta);
-        rtn.alt = event.alt;
+    rtn.meta = event.meta;
-    } else {
+    rtn.alt = event.alt;
-        rtn.cmd = event.alt;
-    }
     rtn.shift = event.shift;
     rtn.repeat = event.isAutoRepeat;
     rtn.location = event.location;

waveshell/pkg/utilfn/syncmap.go

@@ -0,0 +1,84 @@
+// Copyright 2024, Command Line Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package utilfn
+
+import (
+	"sync"
+)
+
+type SyncMap[K comparable, V any] struct {
+	lock *sync.Mutex
+	m    map[K]V
+}
+
+func MakeSyncMap[K comparable, V any]() *SyncMap[K, V] {
+	return &SyncMap[K, V]{
+		lock: &sync.Mutex{},
+		m:    make(map[K]V),
+	}
+}
+
+func (sm *SyncMap[K, V]) Set(k K, v V) {
+	sm.lock.Lock()
+	defer sm.lock.Unlock()
+	sm.m[k] = v
+}
+
+func (sm *SyncMap[K, V]) Get(k K) V {
+	sm.lock.Lock()
+	defer sm.lock.Unlock()
+	return sm.m[k]
+}
+
+func (sm *SyncMap[K, V]) GetEx(k K) (V, bool) {
+	sm.lock.Lock()
+	defer sm.lock.Unlock()
+	v, ok := sm.m[k]
+	return v, ok
+}
+
+func (sm *SyncMap[K, V]) Delete(k K) {
+	sm.lock.Lock()
+	defer sm.lock.Unlock()
+	delete(sm.m, k)
+}
+
+func (sm *SyncMap[K, V]) Clear() {
+	sm.lock.Lock()
+	defer sm.lock.Unlock()
+	sm.m = make(map[K]V)
+}
+
+func (sm *SyncMap[K, V]) Len() int {
+	sm.lock.Lock()
+	defer sm.lock.Unlock()
+	return len(sm.m)
+}
+
+func (sm *SyncMap[K, V]) Keys() []K {
+	sm.lock.Lock()
+	defer sm.lock.Unlock()
+	keys := make([]K, len(sm.m))
+	i := 0
+	for k := range sm.m {
+		keys[i] = k
+		i++
+	}
+	return keys
+}
+
+func (sm *SyncMap[K, V]) Replace(newMap map[K]V) {
+	sm.lock.Lock()
+	defer sm.lock.Unlock()
+	sm.m = make(map[K]V, len(newMap))
+	for k, v := range newMap {
+		sm.m[k] = v
+	}
+}
+
+func IncSyncMap[K comparable, V int | int64](sm *SyncMap[K, V], key K, incAmt V) {
+	sm.lock.Lock()
+	defer sm.lock.Unlock()
+	sm.m[key] += incAmt
+}

wavesrv/db/migrations/000031_restart_cmd.down.sql

@@ -0,0 +1 @@
+ALTER TABLE cmd DROP COLUMN restartts;

wavesrv/db/migrations/000031_restart_cmd.up.sql

@@ -0,0 +1 @@
+ALTER TABLE cmd ADD COLUMN restartts bigint NOT NULL DEFAULT 0;

wavesrv/pkg/cmdrunner/cmdrunner.go

@@ -210,6 +210,7 @@ func init() {
 	registerCmdFn("line:setheight", LineSetHeightCommand)
 	registerCmdFn("line:view", LineViewCommand)
 	registerCmdFn("line:set", LineSetCommand)
+	registerCmdFn("line:restart", LineRestartCommand)
 
 	registerCmdFn("client", ClientCommand)
 	registerCmdFn("client:show", ClientShowCommand)
@@ -491,7 +492,12 @@ func SyncCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (sstore.
 	}
 	runPacket.Command = ":"
 	runPacket.ReturnState = true
-	cmd, callback, err := remote.RunCommand(ctx, ids.SessionId, ids.ScreenId, ids.Remote.RemotePtr, runPacket)
+	rcOpts := remote.RunCommandOpts{
+		SessionId: ids.SessionId,
+		ScreenId:  ids.ScreenId,
+		RemotePtr: ids.Remote.RemotePtr,
+	}
+	cmd, callback, err := remote.RunCommand(ctx, rcOpts, runPacket)
 	if callback != nil {
 		defer callback()
 	}
@@ -587,7 +593,12 @@ func RunCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (sstore.U
 	}
 	runPacket.Command = strings.TrimSpace(cmdStr)
 	runPacket.ReturnState = resolveBool(pk.Kwargs["rtnstate"], isRtnStateCmd)
-	cmd, callback, err := remote.RunCommand(ctx, ids.SessionId, ids.ScreenId, ids.Remote.RemotePtr, runPacket)
+	rcOpts := remote.RunCommandOpts{
+		SessionId: ids.SessionId,
+		ScreenId:  ids.ScreenId,
+		RemotePtr: ids.Remote.RemotePtr,
+	}
+	cmd, callback, err := remote.RunCommand(ctx, rcOpts, runPacket)
 	if callback != nil {
 		defer callback()
 	}
@@ -3337,6 +3348,121 @@ func LineSetHeightCommand(ctx context.Context, pk *scpacket.FeCommandPacketType)
 	return nil, nil
 }
 
+func LineRestartCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (sstore.UpdatePacket, error) {
+	ids, err := resolveUiIds(ctx, pk, R_Session|R_Screen|R_RemoteConnected)
+	if err != nil {
+		return nil, err
+	}
+	var lineId string
+	if len(pk.Args) >= 1 {
+		lineArg := pk.Args[0]
+		resolvedLineId, err := sstore.FindLineIdByArg(ctx, ids.ScreenId, lineArg)
+		if err != nil {
+			return nil, fmt.Errorf("error looking up lineid: %v", err)
+		}
+		lineId = resolvedLineId
+	} else {
+		selectedLineId, err := sstore.GetScreenSelectedLineId(ctx, ids.ScreenId)
+		if err != nil {
+			return nil, fmt.Errorf("error getting selected lineid: %v", err)
+		}
+		lineId = selectedLineId
+	}
+	if lineId == "" {
+		return nil, fmt.Errorf("%s requires a lineid to operate on", GetCmdStr(pk))
+	}
+	line, cmd, err := sstore.GetLineCmdByLineId(ctx, ids.ScreenId, lineId)
+	if err != nil {
+		return nil, fmt.Errorf("error getting line: %v", err)
+	}
+	if line == nil {
+		return nil, fmt.Errorf("line not found")
+	}
+	if cmd == nil {
+		return nil, fmt.Errorf("cannot restart line (no cmd found)")
+	}
+	if cmd.Status == sstore.CmdStatusRunning || cmd.Status == sstore.CmdStatusDetached {
+		killCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
+		defer cancel()
+		err = ids.Remote.MShell.KillRunningCommandAndWait(killCtx, base.MakeCommandKey(ids.ScreenId, lineId))
+		if err != nil {
+			return nil, err
+		}
+	}
+	ids.Remote.MShell.ResetDataPos(base.MakeCommandKey(ids.ScreenId, lineId))
+	err = sstore.ClearCmdPtyFile(ctx, ids.ScreenId, lineId)
+	if err != nil {
+		return nil, fmt.Errorf("error clearing existing pty file: %v", err)
+	}
+	runPacket := packet.MakeRunPacket()
+	runPacket.ReqId = uuid.New().String()
+	runPacket.CK = base.MakeCommandKey(ids.ScreenId, lineId)
+	runPacket.UsePty = true
+	// TODO how can we preseve the original termopts?
+	runPacket.TermOpts, err = GetUITermOpts(pk.UIContext.WinSize, DefaultPTERM)
+	if err != nil {
+		return nil, fmt.Errorf("error getting creating termopts for command: %w", err)
+	}
+	runPacket.Command = cmd.CmdStr
+	runPacket.ReturnState = false
+	rcOpts := remote.RunCommandOpts{
+		SessionId:          ids.SessionId,
+		ScreenId:           ids.ScreenId,
+		RemotePtr:          ids.Remote.RemotePtr,
+		StatePtr:           &cmd.StatePtr,
+		NoCreateCmdPtyFile: true,
+	}
+	cmd, callback, err := remote.RunCommand(ctx, rcOpts, runPacket)
+	if callback != nil {
+		defer callback()
+	}
+	if err != nil {
+		return nil, err
+	}
+	newTs := time.Now().UnixMilli()
+	err = sstore.UpdateCmdForRestart(ctx, runPacket.CK, newTs, cmd.CmdPid, cmd.RemotePid, convertTermOpts(runPacket.TermOpts))
+	if err != nil {
+		return nil, fmt.Errorf("error updating cmd for restart: %w", err)
+	}
+	line, cmd, err = sstore.GetLineCmdByLineId(ctx, ids.ScreenId, lineId)
+	if err != nil {
+		return nil, fmt.Errorf("error getting updated line/cmd: %w", err)
+	}
+	cmd.Restarted = true
+	update := &sstore.ModelUpdate{
+		Line:        line,
+		Cmd:         cmd,
+		Interactive: pk.Interactive,
+	}
+	screen, focusErr := focusScreenLine(ctx, ids.ScreenId, line.LineNum)
+	if focusErr != nil {
+		// not a fatal error, so just log
+		log.Printf("error focusing screen line: %v\n", focusErr)
+	}
+	if screen != nil {
+		update.Screens = []*sstore.ScreenType{screen}
+	}
+	return update, nil
+}
+
+func focusScreenLine(ctx context.Context, screenId string, lineNum int64) (*sstore.ScreenType, error) {
+	screen, err := sstore.GetScreenById(ctx, screenId)
+	if err != nil {
+		return nil, fmt.Errorf("error getting screen: %v", err)
+	}
+	if screen == nil {
+		return nil, fmt.Errorf("screen not found")
+	}
+	updateMap := make(map[string]interface{})
+	updateMap[sstore.ScreenField_SelectedLine] = lineNum
+	updateMap[sstore.ScreenField_Focus] = sstore.ScreenFocusCmd
+	screen, err = sstore.UpdateScreen(ctx, screenId, updateMap)
+	if err != nil {
+		return nil, fmt.Errorf("error updating screen: %v", err)
+	}
+	return screen, nil
+}
+
 func LineSetCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (sstore.UpdatePacket, error) {
 	ids, err := resolveUiIds(ctx, pk, R_Session|R_Screen)
 	if err != nil {
@@ -3768,6 +3894,10 @@ func LineShowCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (sst
 			buf.WriteString(fmt.Sprintf("  %-15s %s\n", "file", stat.Location))
 			buf.WriteString(fmt.Sprintf("  %-15s %s\n", "file-data", fileDataStr))
 		}
+		if cmd.RestartTs > 0 {
+			restartTs := time.UnixMilli(cmd.RestartTs)
+			buf.WriteString(fmt.Sprintf("  %-15s %s\n", "restartts", restartTs.Format(TsFormatStr)))
+		}
 		if cmd.DoneTs != 0 {
 			doneTs := time.UnixMilli(cmd.DoneTs)
 			buf.WriteString(fmt.Sprintf("  %-15s %s\n", "donets", doneTs.Format(TsFormatStr)))

wavesrv/pkg/cmdrunner/termopts.go

@@ -123,3 +123,12 @@ func convertTermOpts(pkto *packet.TermOpts) *sstore.TermOpts {
 		MaxPtySize: pkto.MaxPtySize,
 	}
 }
+
+func convertToPacketTermOpts(sto sstore.TermOpts) *packet.TermOpts {
+	return &packet.TermOpts{
+		Rows:       int(sto.Rows),
+		Cols:       int(sto.Cols),
+		FlexRows:   sto.FlexRows,
+		MaxPtySize: sto.MaxPtySize,
+	}
+}

wavesrv/pkg/remote/remote.go

@@ -161,6 +161,7 @@ type MShellProc struct {
 	StateMap           *server.ShellStateMap
 	NumTryConnect      int
 	InitPkShellType    string
+	DataPosMap         *utilfn.SyncMap[base.CommandKey, int64]
 
 	// install
 	InstallStatus      string
@@ -169,7 +170,6 @@ type MShellProc struct {
 	InstallErr         error
 
 	RunningCmds      map[base.CommandKey]RunCmdType
-	WaitingCmds      []RunCmdType
 	PendingStateCmds map[pendingStateKey]base.CommandKey // key=[remoteinstance name]
 	launcher         Launcher                            // for conditional launch method based on ssh library in use. remove once ssh library is stabilized
 }
@@ -209,6 +209,18 @@ func (msh *MShellProc) GetDefaultState(shellType string) *packet.ShellState {
 	return state
 }
 
+func (msh *MShellProc) EnsureShellType(ctx context.Context, shellType string) error {
+	if msh.StateMap.HasShell(shellType) {
+		return nil
+	}
+	// try to reinit the shell
+	_, err := msh.ReInit(ctx, shellType)
+	if err != nil {
+		return fmt.Errorf("error trying to initialize shell %q: %v", shellType, err)
+	}
+	return nil
+}
+
 func (msh *MShellProc) GetDefaultStatePtr(shellType string) *sstore.ShellStatePtr {
 	msh.Lock.Lock()
 	defer msh.Lock.Unlock()
@@ -692,6 +704,7 @@ func MakeMShell(r *sstore.RemoteType) *MShellProc {
 		PendingStateCmds: make(map[pendingStateKey]base.CommandKey),
 		StateMap:         server.MakeShellStateMap(),
 		launcher:         LegacyLauncher{}, // for conditional launch method based on ssh library in use. remove once ssh library is stabilized
+		DataPosMap:       utilfn.MakeSyncMap[base.CommandKey, int64](),
 	}
 	// for conditional launch method based on ssh library in use
 	// remove once ssh library is stabilized
@@ -1615,12 +1628,8 @@ func replaceHomePath(pathStr string, homeDir string) string {
 func (msh *MShellProc) IsCmdRunning(ck base.CommandKey) bool {
 	msh.Lock.Lock()
 	defer msh.Lock.Unlock()
-	for runningCk := range msh.RunningCmds {
+	_, ok := msh.RunningCmds[ck]
-		if runningCk == ck {
+	return ok
-			return true
-		}
-	}
-	return false
 }
 
 func (msh *MShellProc) SendInput(dataPk *packet.DataPacketType) error {
@@ -1633,6 +1642,30 @@ func (msh *MShellProc) SendInput(dataPk *packet.DataPacketType) error {
 	return msh.ServerProc.Input.SendPacket(dataPk)
 }
 
+func (msh *MShellProc) KillRunningCommandAndWait(ctx context.Context, ck base.CommandKey) error {
+	if !msh.IsCmdRunning(ck) {
+		return nil
+	}
+	siPk := packet.MakeSpecialInputPacket()
+	siPk.CK = ck
+	siPk.SigName = "SIGTERM"
+	err := msh.SendSpecialInput(siPk)
+	if err != nil {
+		return fmt.Errorf("error trying to kill running cmd: %w", err)
+	}
+	for {
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+		if !msh.IsCmdRunning(ck) {
+			return nil
+		}
+		// TODO fix busy wait (sync with msh.RunningCmds)
+		// not a huge deal though since this is not processor intensive and not widely used
+		time.Sleep(100 * time.Millisecond)
+	}
+}
+
 func (msh *MShellProc) SendSpecialInput(siPk *packet.SpecialInputPacketType) error {
 	if !msh.IsConnected() {
 		return fmt.Errorf("remote is not connected, cannot send input")
@@ -1682,14 +1715,25 @@ func (msh *MShellProc) removePendingStateCmd(screenId string, rptr sstore.Remote
 	}
 }
 
-// returns (cmdtype, allow-updates-callback, err)
+type RunCommandOpts struct {
-func RunCommand(ctx context.Context, sessionId string, screenId string, remotePtr sstore.RemotePtrType, runPacket *packet.RunPacketType) (rtnCmd *sstore.CmdType, rtnCallback func(), rtnErr error) {
+	SessionId string
-	rct := RunCmdType{
+	ScreenId  string
-		SessionId: sessionId,
+	RemotePtr sstore.RemotePtrType
-		ScreenId:  screenId,
+
-		RemotePtr: remotePtr,
+	// optional, if not provided shellstate will look up state from remote instance
-		RunPacket: runPacket,
+	// ReturnState cannot be used with StatePtr
-	}
+	// this will also cause this command to bypass the pending state cmd logic
+	StatePtr *sstore.ShellStatePtr
+
+	// set to true to skip creating the pty file (for restarted commands)
+	NoCreateCmdPtyFile bool
+}
+
+// returns (CmdType, allow-updates-callback, err)
+// we must persist the CmdType to the DB before calling the callback to allow updates
+// otherwise an early CmdDone packet might not get processed (since cmd will not exist in DB)
+func RunCommand(ctx context.Context, rcOpts RunCommandOpts, runPacket *packet.RunPacketType) (rtnCmd *sstore.CmdType, rtnCallback func(), rtnErr error) {
+	sessionId, screenId, remotePtr := rcOpts.SessionId, rcOpts.ScreenId, rcOpts.RemotePtr
 	if remotePtr.OwnerId != "" {
 		return nil, nil, fmt.Errorf("cannot run command against another user's remote '%s'", remotePtr.MakeFullRemoteRef())
 	}
@@ -1706,56 +1750,85 @@ func RunCommand(ctx context.Context, sessionId string, screenId string, remotePt
 	if runPacket.State != nil {
 		return nil, nil, fmt.Errorf("runPacket.State should not be set, it is set in RunCommand")
 	}
-	var newPSC *base.CommandKey
+	if rcOpts.StatePtr != nil && runPacket.ReturnState {
-	if runPacket.ReturnState {
+		return nil, nil, fmt.Errorf("RunCommand: cannot use ReturnState with StatePtr")
-		newPSC = &runPacket.CK
 	}
-	ok, existingPSC := msh.testAndSetPendingStateCmd(screenId, remotePtr, newPSC)
+
-	if !ok {
+	// pending state command logic
-		line, _, err := sstore.GetLineCmdByLineId(ctx, screenId, existingPSC.GetCmdId())
+	// if we are currently running a command that can change the state, we need to wait for it to finish
-		if err != nil {
+	if rcOpts.StatePtr == nil {
-			return nil, nil, fmt.Errorf("cannot run command while a stateful command is still running: %v", err)
+		var newPSC *base.CommandKey
+		if runPacket.ReturnState {
+			newPSC = &runPacket.CK
 		}
-		if line == nil {
+		ok, existingPSC := msh.testAndSetPendingStateCmd(screenId, remotePtr, newPSC)
-			return nil, nil, fmt.Errorf("cannot run command while a stateful command is still running %s", *existingPSC)
+		if !ok {
-		}
+			line, _, err := sstore.GetLineCmdByLineId(ctx, screenId, existingPSC.GetCmdId())
-		return nil, nil, fmt.Errorf("cannot run command while a stateful command (linenum=%d) is still running", line.LineNum)
+			if err != nil {
-	}
+				return nil, nil, fmt.Errorf("cannot run command while a stateful command is still running: %v", err)
-	startCmdWait(runPacket.CK)
-	defer func() {
-		if rtnErr != nil {
-			removeCmdWait(runPacket.CK)
-			if newPSC != nil {
-				msh.removePendingStateCmd(screenId, remotePtr, *newPSC)
 			}
+			if line == nil {
+				return nil, nil, fmt.Errorf("cannot run command while a stateful command is still running %s", *existingPSC)
+			}
+			return nil, nil, fmt.Errorf("cannot run command while a stateful command (linenum=%d) is still running", line.LineNum)
 		}
-	}()
+		if newPSC != nil {
+			defer func() {
+				// if we get an error, remove the pending state cmd
+				// if no error, PSC will get removed when we see a CmdDone or CmdFinal packet
+				if rtnErr != nil {
+					msh.removePendingStateCmd(screenId, remotePtr, *newPSC)
+				}
+			}()
+		}
+	}
+
 	// get current remote-instance state
-	statePtr, err := sstore.GetRemoteStatePtr(ctx, sessionId, screenId, remotePtr)
+	var statePtr *sstore.ShellStatePtr
-	if err != nil {
+	if rcOpts.StatePtr != nil {
-		return nil, nil, fmt.Errorf("cannot get current connection stateptr: %w", err)
+		statePtr = rcOpts.StatePtr
+	} else {
+		var err error
+		statePtr, err = sstore.GetRemoteStatePtr(ctx, sessionId, screenId, remotePtr)
+		if err != nil {
+			return nil, nil, fmt.Errorf("cannot get current connection stateptr: %w", err)
+		}
 	}
-	if statePtr == nil {
+	if statePtr == nil { // can be null if there is no remote-instance (screen has unchanged state from default)
+		err := msh.EnsureShellType(ctx, msh.GetShellPref()) // make sure shellType is initialized
+		if err != nil {
+			return nil, nil, err
+		}
 		statePtr = msh.GetDefaultStatePtr(msh.GetShellPref())
-	}
+		if statePtr == nil {
-	if statePtr == nil {
+			return nil, nil, fmt.Errorf("cannot run command, no valid connection stateptr")
-		return nil, nil, fmt.Errorf("cannot run command, no valid connection stateptr")
+		}
 	}
 	currentState, err := sstore.GetFullState(ctx, *statePtr)
 	if err != nil || currentState == nil {
-		return nil, nil, fmt.Errorf("cannot get current remote state: %w", err)
+		return nil, nil, fmt.Errorf("cannot load current remote state: %w", err)
 	}
 	runPacket.State = addScVarsToState(currentState)
 	runPacket.StateComplete = true
 	runPacket.ShellType = currentState.GetShellType()
-	// check to see if shellType is initialized
+	err = msh.EnsureShellType(ctx, runPacket.ShellType) // make sure shellType is initialized
-	if !msh.StateMap.HasShell(runPacket.ShellType) {
+	if err != nil {
-		// try to reinit the shell
+		return nil, nil, err
-		_, err := msh.ReInit(ctx, runPacket.ShellType)
-		if err != nil {
-			return nil, nil, fmt.Errorf("error trying to initialize shell %q: %v", runPacket.ShellType, err)
-		}
 	}
+
+	// start cmdwait.  must be started before sending the run packet
+	// this ensures that we don't process output, or cmddone packets until we set up the line, cmd, and ptyout file
+	startCmdWait(runPacket.CK)
+	defer func() {
+		// if we get an error, remove the cmdwait
+		// if no error, cmdwait will get removed by the caller w/ the callback fn that's returned on success
+		if rtnErr != nil {
+			removeCmdWait(runPacket.CK)
+		}
+	}()
+
+	// RegisterRpc + WaitForResponse is used to get any waveshell side errors
+	// waveshell will either return an error (in a ResponsePacketType) or a CmdStartPacketType
 	msh.ServerProc.Output.RegisterRpc(runPacket.ReqId)
 	err = shexec.SendRunPacketAndRunData(ctx, msh.ServerProc.Input, runPacket)
 	if err != nil {
@@ -1776,6 +1849,8 @@ func RunCommand(ctx context.Context, sessionId string, screenId string, remotePt
 		}
 		return nil, nil, fmt.Errorf("invalid response received from server for run packet: %s", packet.AsString(rtnPk))
 	}
+
+	// command is now successfully runnning
 	status := sstore.CmdStatusRunning
 	if runPacket.Detached {
 		status = sstore.CmdStatusDetached
@@ -1797,44 +1872,20 @@ func RunCommand(ctx context.Context, sessionId string, screenId string, remotePt
 		RunOut:     nil,
 		RtnState:   runPacket.ReturnState,
 	}
-	err = sstore.CreateCmdPtyFile(ctx, cmd.ScreenId, cmd.LineId, cmd.TermOpts.MaxPtySize)
+	if !rcOpts.NoCreateCmdPtyFile {
-	if err != nil {
+		err = sstore.CreateCmdPtyFile(ctx, cmd.ScreenId, cmd.LineId, cmd.TermOpts.MaxPtySize)
-		// TODO the cmd is running, so this is a tricky error to handle
+		if err != nil {
-		return nil, nil, fmt.Errorf("cannot create local ptyout file for running command: %v", err)
+			// TODO the cmd is running, so this is a tricky error to handle
-	}
+			return nil, nil, fmt.Errorf("cannot create local ptyout file for running command: %v", err)
-	msh.AddRunningCmd(rct)
-	return cmd, func() { removeCmdWait(runPacket.CK) }, nil
-}
-
-func (msh *MShellProc) AddWaitingCmd(rct RunCmdType) {
-	msh.Lock.Lock()
-	defer msh.Lock.Unlock()
-	msh.WaitingCmds = append(msh.WaitingCmds, rct)
-}
-
-func (msh *MShellProc) reExecSingle(rct RunCmdType) {
-	// TODO fixme
-	ctx, cancelFn := context.WithTimeout(context.Background(), 15*time.Second)
-	defer cancelFn()
-	_, callback, _ := RunCommand(ctx, rct.SessionId, rct.ScreenId, rct.RemotePtr, rct.RunPacket)
-	if callback != nil {
-		defer callback()
-	}
-}
-
-func (msh *MShellProc) ReExecWaitingCmds() {
-	msh.Lock.Lock()
-	defer msh.Lock.Unlock()
-	for len(msh.WaitingCmds) > 0 {
-		rct := msh.WaitingCmds[0]
-		go msh.reExecSingle(rct)
-		if rct.RunPacket.ReturnState {
-			break
 		}
 	}
-	if len(msh.WaitingCmds) == 0 {
+	msh.AddRunningCmd(RunCmdType{
-		msh.WaitingCmds = nil
+		SessionId: sessionId,
-	}
+		ScreenId:  screenId,
+		RemotePtr: remotePtr,
+		RunPacket: runPacket,
+	})
+	return cmd, func() { removeCmdWait(runPacket.CK) }, nil
 }
 
 func (msh *MShellProc) AddRunningCmd(rct RunCmdType) {
@@ -1940,7 +1991,6 @@ func (msh *MShellProc) notifyHangups_nolock() {
 	}
 	msh.RunningCmds = make(map[base.CommandKey]RunCmdType)
 	msh.PendingStateCmds = make(map[pendingStateKey]base.CommandKey)
-	msh.WaitingCmds = nil
 }
 
 func (msh *MShellProc) handleCmdDonePacket(donePk *packet.CmdDonePacketType) {
@@ -2057,7 +2107,11 @@ func (msh *MShellProc) handleCmdErrorPacket(errPk *packet.CmdErrorPacketType) {
 	return
 }
 
-func (msh *MShellProc) handleDataPacket(dataPk *packet.DataPacketType, dataPosMap map[base.CommandKey]int64) {
+func (msh *MShellProc) ResetDataPos(ck base.CommandKey) {
+	msh.DataPosMap.Delete(ck)
+}
+
+func (msh *MShellProc) handleDataPacket(dataPk *packet.DataPacketType, dataPosMap *utilfn.SyncMap[base.CommandKey, int64]) {
 	realData, err := base64.StdEncoding.DecodeString(dataPk.Data64)
 	if err != nil {
 		ack := makeDataAckPacket(dataPk.CK, dataPk.FdNum, 0, err)
@@ -2066,7 +2120,7 @@ func (msh *MShellProc) handleDataPacket(dataPk *packet.DataPacketType, dataPosMa
 	}
 	var ack *packet.DataAckPacketType
 	if len(realData) > 0 {
-		dataPos := dataPosMap[dataPk.CK]
+		dataPos := dataPosMap.Get(dataPk.CK)
 		rcmd := msh.GetRunningCmd(dataPk.CK)
 		update, err := sstore.AppendToCmdPtyBlob(context.Background(), rcmd.ScreenId, dataPk.CK.GetCmdId(), realData, dataPos)
 		if err != nil {
@@ -2074,7 +2128,7 @@ func (msh *MShellProc) handleDataPacket(dataPk *packet.DataPacketType, dataPosMa
 		} else {
 			ack = makeDataAckPacket(dataPk.CK, dataPk.FdNum, len(realData), nil)
 		}
-		dataPosMap[dataPk.CK] += int64(len(realData))
+		utilfn.IncSyncMap(dataPosMap, dataPk.CK, int64(len(realData)))
 		if update != nil {
 			sstore.MainBus.SendScreenUpdate(dataPk.CK.GetGroupId(), update)
 		}
@@ -2085,7 +2139,7 @@ func (msh *MShellProc) handleDataPacket(dataPk *packet.DataPacketType, dataPosMa
 	// log.Printf("data %s fd=%d len=%d eof=%v err=%v\n", dataPk.CK, dataPk.FdNum, len(realData), dataPk.Eof, dataPk.Error)
 }
 
-func (msh *MShellProc) makeHandleDataPacketClosure(dataPk *packet.DataPacketType, dataPosMap map[base.CommandKey]int64) func() {
+func (msh *MShellProc) makeHandleDataPacketClosure(dataPk *packet.DataPacketType, dataPosMap *utilfn.SyncMap[base.CommandKey, int64]) func() {
 	return func() {
 		msh.handleDataPacket(dataPk, dataPosMap)
 	}
@@ -2124,12 +2178,10 @@ func (msh *MShellProc) ProcessPackets() {
 			go sendScreenUpdates(screens)
 		}
 	})
-	// TODO need to clean dataPosMap
-	dataPosMap := make(map[base.CommandKey]int64)
 	for pk := range msh.ServerProc.Output.MainCh {
 		if pk.GetType() == packet.DataPacketStr {
 			dataPk := pk.(*packet.DataPacketType)
-			runCmdUpdateFn(dataPk.CK, msh.makeHandleDataPacketClosure(dataPk, dataPosMap))
+			runCmdUpdateFn(dataPk.CK, msh.makeHandleDataPacketClosure(dataPk, msh.DataPosMap))
 			go pushStatusIndicatorUpdate(&dataPk.CK, sstore.StatusIndicatorLevel_Output)
 			continue
 		}

wavesrv/pkg/sstore/dbops.go

@@ -763,29 +763,37 @@ func GetScreenById(ctx context.Context, screenId string) (*ScreenType, error) {
 	})
 }
 
+// special "E" returns last unarchived line, "EA" returns last line (even if archived)
 func FindLineIdByArg(ctx context.Context, screenId string, lineArg string) (string, error) {
-	var lineId string
+	return WithTxRtn(ctx, func(tx *TxWrap) (string, error) {
-	txErr := WithTx(ctx, func(tx *TxWrap) error {
+		if lineArg == "E" {
+			query := `SELECT lineid FROM line WHERE screenid = ? AND NOT archived ORDER BY linenum DESC LIMIT 1`
+			lineId := tx.GetString(query, screenId)
+			return lineId, nil
+		}
+		if lineArg == "EA" {
+			query := `SELECT lineid FROM line WHERE screenid = ? ORDER BY linenum DESC LIMIT 1`
+			lineId := tx.GetString(query, screenId)
+			return lineId, nil
+		}
 		lineNum, err := strconv.Atoi(lineArg)
 		if err == nil {
 			// valid linenum
 			query := `SELECT lineid FROM line WHERE screenid = ? AND linenum = ?`
-			lineId = tx.GetString(query, screenId, lineNum)
+			lineId := tx.GetString(query, screenId, lineNum)
+			return lineId, nil
 		} else if len(lineArg) == 8 {
 			// prefix id string match
 			query := `SELECT lineid FROM line WHERE screenid = ? AND substr(lineid, 1, 8) = ?`
-			lineId = tx.GetString(query, screenId, lineArg)
+			lineId := tx.GetString(query, screenId, lineArg)
+			return lineId, nil
 		} else {
 			// id match
 			query := `SELECT lineid FROM line WHERE screenid = ? AND lineid = ?`
-			lineId = tx.GetString(query, screenId, lineArg)
+			lineId := tx.GetString(query, screenId, lineArg)
+			return lineId, nil
 		}
-		return nil
 	})
-	if txErr != nil {
-		return "", txErr
-	}
-	return lineId, nil
 }
 
 func GetLineCmdByLineId(ctx context.Context, screenId string, lineId string) (*LineType, *CmdType, error) {
@@ -836,8 +844,8 @@ func InsertLine(ctx context.Context, line *LineType, cmd *CmdType) error {
 			cmd.OrigTermOpts = cmd.TermOpts
 			cmdMap := cmd.ToMap()
 			query = `
-INSERT INTO cmd  ( screenid, lineid, remoteownerid, remoteid, remotename, cmdstr, rawcmdstr, festate, statebasehash, statediffhasharr, termopts, origtermopts, status, cmdpid, remotepid, donets, exitcode, durationms, rtnstate, runout, rtnbasehash, rtndiffhasharr)
+INSERT INTO cmd  ( screenid, lineid, remoteownerid, remoteid, remotename, cmdstr, rawcmdstr, festate, statebasehash, statediffhasharr, termopts, origtermopts, status, cmdpid, remotepid, donets, restartts, exitcode, durationms, rtnstate, runout, rtnbasehash, rtndiffhasharr)
-          VALUES (:screenid,:lineid,:remoteownerid,:remoteid,:remotename,:cmdstr,:rawcmdstr,:festate,:statebasehash,:statediffhasharr,:termopts,:origtermopts,:status,:cmdpid,:remotepid,:donets,:exitcode,:durationms,:rtnstate,:runout,:rtnbasehash,:rtndiffhasharr)
+          VALUES (:screenid,:lineid,:remoteownerid,:remoteid,:remotename,:cmdstr,:rawcmdstr,:festate,:statebasehash,:statediffhasharr,:termopts,:origtermopts,:status,:cmdpid,:remotepid,:donets,:restartts,:exitcode,:durationms,:rtnstate,:runout,:rtnbasehash,:rtndiffhasharr)
 `
 			tx.NamedExec(query, cmdMap)
 		}
@@ -879,6 +887,20 @@ func UpdateWithUpdateOpenAICmdInfoPacket(ctx context.Context, screenId string, m
 	return UpdateWithCurrentOpenAICmdInfoChat(screenId)
 }
 
+func UpdateCmdForRestart(ctx context.Context, ck base.CommandKey, ts int64, cmdPid int, remotePid int, termOpts *TermOpts) error {
+	return WithTx(ctx, func(tx *TxWrap) error {
+		query := `UPDATE cmd
+		          SET restartts = ?, status = ?, exitcode = ?, cmdpid = ?, remotepid = ?, durationms = ?, termopts = ?, origtermopts = ?
+				  WHERE screenid = ? AND lineid = ?`
+		tx.Exec(query, ts, CmdStatusRunning, 0, cmdPid, remotePid, 0, quickJson(termOpts), quickJson(termOpts), ck.GetGroupId(), lineIdFromCK(ck))
+		query = `UPDATE history
+		         SET ts = ?, status = ?, exitcode = ?, durationms = ?
+			     WHERE screenid = ? AND lineid = ?`
+		tx.Exec(query, ts, CmdStatusRunning, 0, 0, ck.GetGroupId(), lineIdFromCK(ck))
+		return nil
+	})
+}
+
 func UpdateCmdDoneInfo(ctx context.Context, ck base.CommandKey, donePk *packet.CmdDonePacketType, status string) (*ModelUpdate, error) {
 	if donePk == nil {
 		return nil, fmt.Errorf("invalid cmddone packet")
@@ -1490,12 +1512,16 @@ func ArchiveScreenLines(ctx context.Context, screenId string) (*ModelUpdate, err
 func DeleteScreenLines(ctx context.Context, screenId string) (*ModelUpdate, error) {
 	var lineIds []string
 	txErr := WithTx(ctx, func(tx *TxWrap) error {
-		query := `SELECT lineid FROM line WHERE screenid = ?`
+		query := `SELECT lineid FROM line 
-		lineIds = tx.SelectStrings(query, screenId)
+		          WHERE screenid = ?
-		query = `DELETE FROM line WHERE screenid = ?`
+		            AND NOT EXISTS (SELECT lineid FROM cmd c WHERE c.screenid = ? AND c.lineid = line.lineid AND c.status IN ('running', 'detached'))`
-		tx.Exec(query, screenId)
+		lineIds = tx.SelectStrings(query, screenId, screenId)
-		query = `UPDATE history SET lineid = '', linenum = 0 WHERE screenid = ?`
+		query = `DELETE FROM line 
-		tx.Exec(query, screenId)
+				 WHERE screenid = ? AND lineid IN (SELECT value FROM json_each(?))`
+		tx.Exec(query, screenId, quickJsonArr(lineIds))
+		query = `UPDATE history SET lineid = '', linenum = 0 
+		         WHERE screenid = ? AND lineid IN (SELECT value FROM json_each(?))`
+		tx.Exec(query, screenId, quickJsonArr(lineIds))
 		return nil
 	})
 	if txErr != nil {
@@ -2091,6 +2117,19 @@ func SetLineArchivedById(ctx context.Context, screenId string, lineId string, ar
 	return txErr
 }
 
+func GetScreenSelectedLineId(ctx context.Context, screenId string) (string, error) {
+	return WithTxRtn(ctx, func(tx *TxWrap) (string, error) {
+		query := `SELECT selectedline FROM screen WHERE screenid = ?`
+		sline := tx.GetInt(query, screenId)
+		if sline <= 0 {
+			return "", nil
+		}
+		query = `SELECT lineid FROM line WHERE screenid = ? AND linenum = ?`
+		lineId := tx.GetString(query, screenId, sline)
+		return lineId, nil
+	})
+}
+
 // returns updated screen (only if updated)
 func FixupScreenSelectedLine(ctx context.Context, screenId string) (*ScreenType, error) {
 	return WithTxRtn(ctx, func(tx *TxWrap) (*ScreenType, error) {

wavesrv/pkg/sstore/fileops.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/wavetermdev/waveterm/waveshell/pkg/cirfile"
+	"github.com/wavetermdev/waveterm/waveshell/pkg/shexec"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/scbase"
 )
 
@@ -39,6 +40,27 @@ func StatCmdPtyFile(ctx context.Context, screenId string, lineId string) (*cirfi
 	return cirfile.StatCirFile(ctx, ptyOutFileName)
 }
 
+func ClearCmdPtyFile(ctx context.Context, screenId string, lineId string) error {
+	ptyOutFileName, err := scbase.PtyOutFile(screenId, lineId)
+	if err != nil {
+		return err
+	}
+	stat, err := cirfile.StatCirFile(ctx, ptyOutFileName)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return err
+	}
+	os.Remove(ptyOutFileName) // ignore error
+	var maxSize int64 = shexec.DefaultMaxPtySize
+	if stat != nil {
+		maxSize = stat.MaxSize
+	}
+	err = CreateCmdPtyFile(ctx, screenId, lineId, maxSize)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func AppendToCmdPtyBlob(ctx context.Context, screenId string, lineId string, data []byte, pos int64) (*PtyDataUpdate, error) {
 	if screenId == "" {
 		return nil, fmt.Errorf("cannot append to PtyBlob, screenid is not set")

wavesrv/pkg/sstore/migrate.go

@@ -22,7 +22,7 @@ import (
 	"github.com/golang-migrate/migrate/v4"
 )
 
-const MaxMigration = 30
+const MaxMigration = 31
 const MigratePrimaryScreenVersion = 9
 const CmdScreenSpecialMigration = 13
 const CmdLineSpecialMigration = 20

wavesrv/pkg/sstore/sstore.go

@@ -1118,13 +1118,15 @@ type CmdType struct {
 	Status       string              `json:"status"`
 	CmdPid       int                 `json:"cmdpid"`
 	RemotePid    int                 `json:"remotepid"`
+	RestartTs    int64               `json:"restartts,omitempty"`
 	DoneTs       int64               `json:"donets"`
 	ExitCode     int                 `json:"exitcode"`
 	DurationMs   int                 `json:"durationms"`
 	RunOut       []packet.PacketType `json:"runout,omitempty"`
 	RtnState     bool                `json:"rtnstate,omitempty"`
 	RtnStatePtr  ShellStatePtr       `json:"rtnstateptr,omitempty"`
-	Remove       bool                `json:"remove,omitempty"`
+	Remove       bool                `json:"remove,omitempty"`    // not persisted to DB
+	Restarted    bool                `json:"restarted,omitempty"` // not persisted to DB
 }
 
 func (r *RemoteType) ToMap() map[string]interface{} {
@@ -1189,6 +1191,7 @@ func (cmd *CmdType) ToMap() map[string]interface{} {
 	rtn["status"] = cmd.Status
 	rtn["cmdpid"] = cmd.CmdPid
 	rtn["remotepid"] = cmd.RemotePid
+	rtn["restartts"] = cmd.RestartTs
 	rtn["donets"] = cmd.DoneTs
 	rtn["exitcode"] = cmd.ExitCode
 	rtn["durationms"] = cmd.DurationMs
@@ -1216,6 +1219,7 @@ func (cmd *CmdType) FromMap(m map[string]interface{}) bool {
 	quickSetInt(&cmd.CmdPid, m, "cmdpid")
 	quickSetInt(&cmd.RemotePid, m, "remotepid")
 	quickSetInt64(&cmd.DoneTs, m, "donets")
+	quickSetInt64(&cmd.RestartTs, m, "restartts")
 	quickSetInt(&cmd.ExitCode, m, "exitcode")
 	quickSetInt(&cmd.DurationMs, m, "durationms")
 	quickSetJson(&cmd.RunOut, m, "runout")