big update, got statediff and state_base working. updates to remote_instance and cmd tables/structures

cmd/main-server.go

@@ -185,6 +185,17 @@ func HandleGetWindow(w http.ResponseWriter, r *http.Request) {
 }
 
 func HandleRtnState(w http.ResponseWriter, r *http.Request) {
+	defer func() {
+		r := recover()
+		if r == nil {
+			return
+		}
+		log.Printf("[error] in handlertnstate: %v\n", r)
+		debug.PrintStack()
+		w.WriteHeader(500)
+		w.Write([]byte(fmt.Sprintf("panic: %v", r)))
+		return
+	}()
 	qvals := r.URL.Query()
 	sessionId := qvals.Get("sessionid")
 	cmdId := qvals.Get("cmdid")

db/migrations/000001_init.up.sql

@@ -126,14 +126,18 @@ CREATE TABLE cmd (
     remoteid varchar(36) NOT NULL,
     remotename varchar(50) NOT NULL,
     cmdstr text NOT NULL,
-    remotestate json NOT NULL,
+    festate json NOT NULL,
+    statebasehash varchar(36) NOT NULL,
+    statediffhasharr json NOT NULL,
     termopts json NOT NULL,
     origtermopts json NOT NULL,
     status varchar(10) NOT NULL,
     startpk json NOT NULL,
-    donepk json NOT NULL,
+    doneinfo json NOT NULL,
     runout json NOT NULL,
     rtnstate bool NOT NULL,
+    rtnbasehash varchar(36) NOT NULL,
+    rtndiffhasharr json NOT NULL,
     PRIMARY KEY (sessionid, cmdid)
 );
 

pkg/cmdrunner/cmdrunner.go

@@ -942,11 +942,14 @@ func makeStaticCmd(ctx context.Context, metaCmd string, ids resolvedIds, cmdStr
 		TermOpts:  sstore.TermOpts{Rows: shexec.DefaultTermRows, Cols: shexec.DefaultTermCols, FlexRows: true, MaxPtySize: remote.DefaultMaxPtySize},
 		Status:    sstore.CmdStatusDone,
 		StartPk:   nil,
-		DonePk:    nil,
+		DoneInfo:  nil,
 		RunOut:    nil,
 	}
-	if ids.Remote.RemoteState != nil {
-		cmd.RemoteState = *ids.Remote.RemoteState
+	if ids.Remote.StatePtr != nil {
+		cmd.StatePtr = *ids.Remote.StatePtr
+	}
+	if ids.Remote.FeState != nil {
+		cmd.FeState = *ids.Remote.FeState
 	}
 	err := sstore.CreateCmdPtyFile(ctx, cmd.SessionId, cmd.CmdId, cmd.TermOpts.MaxPtySize)
 	if err != nil {
@@ -1097,7 +1100,7 @@ func doCompGen(ctx context.Context, pk *scpacket.FeCommandPacketType, prefix str
 	cgPacket.ReqId = uuid.New().String()
 	cgPacket.CompType = compType
 	cgPacket.Prefix = prefix
-	cgPacket.Cwd = ids.Remote.RemoteState.Cwd
+	cgPacket.Cwd = ids.Remote.FeState.Cwd
 	resp, err := ids.Remote.MShell.PacketRpc(ctx, cgPacket)
 	if err != nil {
 		return nil, false, err
@@ -1110,8 +1113,6 @@ func doCompGen(ctx context.Context, pk *scpacket.FeCommandPacketType, prefix str
 	return comps, hasMore, nil
 }
 
-// func DoCompGen(ctx context.Context, sp StrWithPos, compCtx CompContext) (*CompReturn, *StrWithPos, error)
-
 func CompGenCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (sstore.UpdatePacket, error) {
 	ids, err := resolveUiIds(ctx, pk, 0) // best-effort
 	if err != nil {
@@ -1138,7 +1139,9 @@ func CompGenCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (ssto
 	if ids.Remote != nil {
 		rptr := ids.Remote.RemotePtr
 		compCtx.RemotePtr = &rptr
-		compCtx.State = ids.Remote.RemoteState
+		if ids.Remote.FeState != nil {
+			compCtx.Cwd = ids.Remote.FeState.Cwd
+		}
 	}
 	compCtx.ForDisplay = showComps
 	crtn, newSP, err := comp.DoCompGen(ctx, cmdSP, compCtx)
@@ -1378,7 +1381,7 @@ func ResetCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (sstore
 		// TODO tricky error since the command was a success, but we can't show the output
 		return nil, err
 	}
-	update, err := addLineForCmd(ctx, "/cd", false, ids, cmd)
+	update, err := addLineForCmd(ctx, "/reset", false, ids, cmd)
 	if err != nil {
 		// TODO tricky error since the command was a success, but we can't show the output
 		return nil, err
@@ -1557,8 +1560,8 @@ func LineShowCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (sst
 		buf.WriteString(fmt.Sprintf("  %-15s %s\n", "cmdid", cmd.CmdId))
 		buf.WriteString(fmt.Sprintf("  %-15s %s\n", "remote", cmd.Remote.MakeFullRemoteRef()))
 		buf.WriteString(fmt.Sprintf("  %-15s %s\n", "status", cmd.Status))
-		if cmd.RemoteState.Cwd != "" {
-			buf.WriteString(fmt.Sprintf("  %-15s %s\n", "cwd", cmd.RemoteState.Cwd))
+		if cmd.FeState.Cwd != "" {
+			buf.WriteString(fmt.Sprintf("  %-15s %s\n", "cwd", cmd.FeState.Cwd))
 		}
 		buf.WriteString(fmt.Sprintf("  %-15s %s\n", "termopts", formatTermOpts(cmd.TermOpts)))
 		if cmd.TermOpts != cmd.OrigTermOpts {
@@ -1670,6 +1673,9 @@ func formatTextTable(totalCols int, data [][]string, colMeta []ColMeta) []string
 	return rtn
 }
 
+const MaxDiffKeyLen = 40
+const MaxDiffValLen = 50
+
 func displayStateUpdateDiff(buf *bytes.Buffer, oldState packet.ShellState, newState packet.ShellState) {
 	if newState.Cwd != oldState.Cwd {
 		buf.WriteString(fmt.Sprintf("cwd %s\n", newState.Cwd))
@@ -1684,13 +1690,13 @@ func displayStateUpdateDiff(buf *bytes.Buffer, oldState packet.ShellState, newSt
 				if newVal.IsExport() {
 					exportStr = "export "
 				}
-				buf.WriteString(fmt.Sprintf("%s%s=%s\n", exportStr, key, utilfn.ShellQuote(newVal.Value, false, 50)))
+				buf.WriteString(fmt.Sprintf("%s%s=%s\n", exportStr, utilfn.EllipsisStr(key, MaxDiffKeyLen), utilfn.EllipsisStr(newVal.Value, MaxDiffValLen)))
 			}
 		}
 		for key, _ := range oldEnvMap {
 			_, found := newEnvMap[key]
 			if !found {
-				buf.WriteString(fmt.Sprintf("unset %s\n", key))
+				buf.WriteString(fmt.Sprintf("unset %s\n", utilfn.EllipsisStr(key, MaxDiffKeyLen)))
 			}
 		}
 	}
@@ -1700,13 +1706,13 @@ func displayStateUpdateDiff(buf *bytes.Buffer, oldState packet.ShellState, newSt
 		for aliasName, newAliasVal := range newAliasMap {
 			oldAliasVal, found := oldAliasMap[aliasName]
 			if !found || newAliasVal != oldAliasVal {
-				buf.WriteString(fmt.Sprintf("alias %s\n", shellescape.Quote(aliasName)))
+				buf.WriteString(fmt.Sprintf("alias %s\n", utilfn.EllipsisStr(shellescape.Quote(aliasName), MaxDiffKeyLen)))
 			}
 		}
 		for aliasName, _ := range oldAliasMap {
 			_, found := newAliasMap[aliasName]
 			if !found {
-				buf.WriteString(fmt.Sprintf("unalias %s\n", shellescape.Quote(aliasName)))
+				buf.WriteString(fmt.Sprintf("unalias %s\n", utilfn.EllipsisStr(shellescape.Quote(aliasName), MaxDiffKeyLen)))
 			}
 		}
 	}
@@ -1716,13 +1722,13 @@ func displayStateUpdateDiff(buf *bytes.Buffer, oldState packet.ShellState, newSt
 		for funcName, newFuncVal := range newFuncMap {
 			oldFuncVal, found := oldFuncMap[funcName]
 			if !found || newFuncVal != oldFuncVal {
-				buf.WriteString(fmt.Sprintf("function %s\n", shellescape.Quote(funcName)))
+				buf.WriteString(fmt.Sprintf("function %s\n", utilfn.EllipsisStr(shellescape.Quote(funcName), MaxDiffKeyLen)))
 			}
 		}
 		for funcName, _ := range oldFuncMap {
 			_, found := newFuncMap[funcName]
 			if !found {
-				buf.WriteString(fmt.Sprintf("unset -f %s\n", shellescape.Quote(funcName)))
+				buf.WriteString(fmt.Sprintf("unset -f %s\n", utilfn.EllipsisStr(shellescape.Quote(funcName), MaxDiffKeyLen)))
 			}
 		}
 	}
@@ -1739,11 +1745,19 @@ func GetRtnStateDiff(ctx context.Context, sessionId string, cmdId string) ([]byt
 	if !cmd.RtnState {
 		return nil, nil
 	}
-	if cmd.DonePk == nil || cmd.DonePk.FinalState == nil {
+	if cmd.RtnStatePtr.IsEmpty() {
 		return nil, nil
 	}
 	var outputBytes bytes.Buffer
-	displayStateUpdateDiff(&outputBytes, cmd.RemoteState, *cmd.DonePk.FinalState)
+	initialState, err := sstore.GetFullState(ctx, cmd.StatePtr)
+	if err != nil {
+		return nil, fmt.Errorf("getting initial full state: %v", err)
+	}
+	rtnState, err := sstore.GetFullState(ctx, cmd.RtnStatePtr)
+	if err != nil {
+		return nil, fmt.Errorf("getting rtn full state: %v", err)
+	}
+	displayStateUpdateDiff(&outputBytes, *initialState, *rtnState)
 	return outputBytes.Bytes(), nil
 }
 

pkg/cmdrunner/resolver.go

@@ -3,12 +3,12 @@ package cmdrunner
 import (
 	"context"
 	"fmt"
+	"log"
 	"regexp"
 	"strconv"
 	"strings"
 
 	"github.com/google/uuid"
-	"github.com/scripthaus-dev/mshell/pkg/packet"
 	"github.com/scripthaus-dev/sh2-server/pkg/remote"
 	"github.com/scripthaus-dev/sh2-server/pkg/scpacket"
 	"github.com/scripthaus-dev/sh2-server/pkg/sstore"
@@ -34,8 +34,9 @@ type ResolvedRemote struct {
 	RemotePtr   sstore.RemotePtrType
 	MShell      *remote.MShellProc
 	RState      remote.RemoteRuntimeState
-	RemoteState *packet.ShellState
 	RemoteCopy  *sstore.RemoteType
+	StatePtr    *sstore.ShellStatePtr
+	FeState     *sstore.FeStateType
 }
 
 type ResolveItem = sstore.ResolveItem
@@ -264,7 +265,7 @@ func resolveUiIds(ctx context.Context, pk *scpacket.FeCommandPacketType, rtype i
 		if !rtn.Remote.RState.IsConnected() {
 			return rtn, fmt.Errorf("remote '%s' is not connected", rtn.Remote.DisplayName)
 		}
-		if rtn.Remote.RemoteState == nil {
+		if rtn.Remote.StatePtr == nil || rtn.Remote.FeState == nil {
 			return rtn, fmt.Errorf("remote '%s' state is not available", rtn.Remote.DisplayName)
 		}
 	}
@@ -453,20 +454,26 @@ func resolveRemoteFromPtr(ctx context.Context, rptr *sstore.RemotePtrType, sessi
 	rtn := &ResolvedRemote{
 		DisplayName: displayName,
 		RemotePtr:   *rptr,
-		RemoteState: nil,
 		RState:      rstate,
 		MShell:      msh,
 		RemoteCopy:  &rcopy,
+		StatePtr:    nil,
+		FeState:     nil,
 	}
 	if sessionId != "" && windowId != "" {
-		state, err := sstore.GetRemoteState(ctx, sessionId, windowId, *rptr)
+		ri, err := sstore.GetRemoteInstance(ctx, sessionId, windowId, *rptr)
 		if err != nil {
-			return nil, fmt.Errorf("cannot resolve remote state '%s': %w", displayName, err)
+			log.Printf("ERROR resolving remote state '%s': %v\n", displayName, err)
+			// continue with state set to nil
+		} else {
+			if ri == nil {
+				rtn.StatePtr = msh.GetDefaultStatePtr()
+				rtn.FeState = msh.GetDefaultFeState()
+			} else {
+				rtn.StatePtr = &sstore.ShellStatePtr{BaseHash: ri.StateBaseHash, DiffHashArr: ri.StateDiffHashArr}
+				rtn.FeState = &ri.FeState
+			}
 		}
-		if state == nil {
-			state = msh.GetDefaultState()
-		}
-		rtn.RemoteState = state
 	}
 	return rtn, nil
 }

pkg/cmdrunner/shparse.go

@@ -171,7 +171,7 @@ func IsReturnStateCommand(cmdStr string) bool {
 			if len(callExpr.Args) > 0 && len(callExpr.Args[0].Parts) > 0 {
 				lit, ok := callExpr.Args[0].Parts[0].(*syntax.Lit)
 				if ok {
-					if lit.Value == "." || lit.Value == "source" || lit.Value == "unset" || lit.Value == "cd" {
+					if lit.Value == "." || lit.Value == "source" || lit.Value == "unset" || lit.Value == "cd" || lit.Value == "alias" || lit.Value == "unalias" {
 						return true
 					}
 				}

pkg/cmdrunner/termopts.go

@@ -0,0 +1,110 @@
+package cmdrunner
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/scripthaus-dev/mshell/pkg/base"
+	"github.com/scripthaus-dev/mshell/pkg/packet"
+	"github.com/scripthaus-dev/mshell/pkg/shexec"
+	"github.com/scripthaus-dev/sh2-server/pkg/remote"
+)
+
+// PTERM=WxH,Wx25
+// PTERM="Wx25!"
+// PTERM=80x25,80x35
+
+type PTermOptsType struct {
+	Rows     string
+	RowsFlex bool
+	Cols     string
+	ColsFlex bool
+}
+
+const PTermMax = "M"
+
+func isDigits(s string) bool {
+	for _, ch := range s {
+		if ch < '0' || ch > '9' {
+			return false
+		}
+	}
+	return true
+}
+
+func atoiDefault(s string, def int) int {
+	ival, err := strconv.Atoi(s)
+	if err != nil {
+		return def
+	}
+	return ival
+}
+
+func parseTermPart(part string, partType string) (string, bool, error) {
+	flex := true
+	if strings.HasSuffix(part, "!") {
+		part = part[:len(part)-1]
+		flex = false
+	}
+	if part == "" {
+		return PTermMax, flex, nil
+	}
+	if part == PTermMax {
+		return PTermMax, flex, nil
+	}
+	if !isDigits(part) {
+		return "", false, fmt.Errorf("invalid PTERM %s: must be '%s' or [number]", partType, PTermMax)
+	}
+	return part, flex, nil
+}
+
+func parseSingleTermStr(s string) (*PTermOptsType, error) {
+	s = strings.TrimSpace(s)
+	xIdx := strings.Index(s, "x")
+	if xIdx == -1 {
+		return nil, fmt.Errorf("invalid PTERM, must include 'x' to separate width and height (e.g. WxH)")
+	}
+	rowsPart := s[0:xIdx]
+	colsPart := s[xIdx+1:]
+	rows, rowsFlex, err := parseTermPart(rowsPart, "rows")
+	if err != nil {
+		return nil, err
+	}
+	cols, colsFlex, err := parseTermPart(colsPart, "cols")
+	if err != nil {
+		return nil, err
+	}
+	return &PTermOptsType{Rows: rows, RowsFlex: rowsFlex, Cols: cols, ColsFlex: colsFlex}, nil
+}
+
+func GetUITermOpts(winSize *packet.WinSize, ptermStr string) (*packet.TermOpts, error) {
+	opts, err := parseSingleTermStr(ptermStr)
+	if err != nil {
+		return nil, err
+	}
+	termOpts := &packet.TermOpts{Rows: shexec.DefaultTermRows, Cols: shexec.DefaultTermCols, Term: remote.DefaultTerm, MaxPtySize: shexec.DefaultMaxPtySize}
+	if winSize == nil {
+		winSize = &packet.WinSize{Rows: shexec.DefaultTermRows, Cols: shexec.DefaultTermCols}
+	}
+	if winSize.Rows == 0 {
+		winSize.Rows = shexec.DefaultTermRows
+	}
+	if winSize.Cols == 0 {
+		winSize.Cols = shexec.DefaultTermCols
+	}
+	if opts.Rows == PTermMax {
+		termOpts.Rows = winSize.Rows
+	} else {
+		termOpts.Rows = atoiDefault(opts.Rows, termOpts.Rows)
+	}
+	if opts.Cols == PTermMax {
+		termOpts.Cols = winSize.Cols
+	} else {
+		termOpts.Cols = atoiDefault(opts.Cols, termOpts.Cols)
+	}
+	termOpts.MaxPtySize = base.BoundInt64(termOpts.MaxPtySize, shexec.MinMaxPtySize, shexec.MaxMaxPtySize)
+	termOpts.Cols = base.BoundInt(termOpts.Cols, shexec.MinTermCols, shexec.MaxTermCols)
+	termOpts.Rows = base.BoundInt(termOpts.Rows, shexec.MinTermRows, shexec.MaxTermRows)
+	return termOpts, nil
+}

pkg/comp/comp.go

@@ -11,7 +11,6 @@ import (
 	"unicode"
 	"unicode/utf8"
 
-	"github.com/scripthaus-dev/mshell/pkg/packet"
 	"github.com/scripthaus-dev/mshell/pkg/simpleexpand"
 	"github.com/scripthaus-dev/sh2-server/pkg/shparse"
 	"github.com/scripthaus-dev/sh2-server/pkg/sstore"
@@ -46,7 +45,7 @@ const (
 
 type CompContext struct {
 	RemotePtr  *sstore.RemotePtrType
-	State      *packet.ShellState
+	Cwd        string
 	ForDisplay bool
 }
 

pkg/comp/simplecomp.go

@@ -70,7 +70,7 @@ func doCompGen(ctx context.Context, prefix string, compType string, compCtx Comp
 	cgPacket.ReqId = uuid.New().String()
 	cgPacket.CompType = compType
 	cgPacket.Prefix = prefix
-	cgPacket.Cwd = compCtx.State.Cwd
+	cgPacket.Cwd = compCtx.Cwd
 	resp, err := msh.PacketRpc(ctx, cgPacket)
 	if err != nil {
 		return nil, err

pkg/remote/remote.go

@@ -24,6 +24,7 @@ import (
 	"github.com/scripthaus-dev/mshell/pkg/base"
 	"github.com/scripthaus-dev/mshell/pkg/packet"
 	"github.com/scripthaus-dev/mshell/pkg/shexec"
+	"github.com/scripthaus-dev/mshell/pkg/statediff"
 	"github.com/scripthaus-dev/sh2-server/pkg/scbase"
 	"github.com/scripthaus-dev/sh2-server/pkg/scpacket"
 	"github.com/scripthaus-dev/sh2-server/pkg/sstore"
@@ -150,6 +151,20 @@ func (msh *MShellProc) GetDefaultState() *packet.ShellState {
 	return msh.StateMap[msh.CurrentState]
 }
 
+func (msh *MShellProc) GetDefaultStatePtr() *sstore.ShellStatePtr {
+	msh.Lock.Lock()
+	defer msh.Lock.Unlock()
+	if msh.CurrentState == "" {
+		return nil
+	}
+	return &sstore.ShellStatePtr{BaseHash: msh.CurrentState}
+}
+
+func (msh *MShellProc) GetDefaultFeState() *sstore.FeStateType {
+	state := msh.GetDefaultState()
+	return sstore.FeStateFromShellState(state)
+}
+
 func (msh *MShellProc) GetStateByHash(hval string) *packet.ShellState {
 	msh.Lock.Lock()
 	defer msh.Lock.Unlock()
@@ -926,6 +941,7 @@ func (msh *MShellProc) ReInit(ctx context.Context) (*packet.InitPacketType, erro
 		return nil, fmt.Errorf("invalid reinit response initpk does not contain remote state")
 	}
 	hval := initPk.State.GetHashVal(false)
+	sstore.StoreStateBase(ctx, initPk.State)
 	msh.WithLock(func() {
 		msh.CurrentState = hval
 		msh.StateMap[hval] = initPk.State
@@ -950,6 +966,7 @@ func stripScVarsFromState(state *packet.ShellState) *packet.ShellState {
 		return nil
 	}
 	rtn := *state
+	rtn.HashVal = ""
 	envMap := shexec.DeclMapFromState(&rtn)
 	delete(envMap, "SCRIPTHAUS")
 	delete(envMap, "SCRIPTHAUS_VERSION")
@@ -957,6 +974,23 @@ func stripScVarsFromState(state *packet.ShellState) *packet.ShellState {
 	return &rtn
 }
 
+func stripScVarsFromStateDiff(stateDiff *packet.ShellStateDiff) *packet.ShellStateDiff {
+	if stateDiff == nil || len(stateDiff.VarsDiff) == 0 {
+		return stateDiff
+	}
+	rtn := *stateDiff
+	rtn.HashVal = ""
+	var mapDiff statediff.MapDiffType
+	err := mapDiff.Decode(stateDiff.VarsDiff)
+	if err != nil {
+		return stateDiff
+	}
+	delete(mapDiff.ToAdd, "SCRIPTHAUS")
+	delete(mapDiff.ToAdd, "SCRIPTHAUS_VERSION")
+	rtn.VarsDiff = mapDiff.Encode()
+	return &rtn
+}
+
 func (msh *MShellProc) Launch() {
 	remoteCopy := msh.GetRemoteCopy()
 	if remoteCopy.Archived {
@@ -1214,16 +1248,20 @@ func RunCommand(ctx context.Context, sessionId string, windowId string, remotePt
 		}
 	}()
 	// get current remote-instance state
-	currentState, err := sstore.GetRemoteState(ctx, sessionId, windowId, remotePtr)
+	statePtr, err := sstore.GetRemoteStatePtr(ctx, sessionId, windowId, remotePtr)
 	if err != nil {
+		return nil, nil, fmt.Errorf("cannot get current remote stateptr: %w", err)
+	}
+	if statePtr == nil {
+		statePtr = msh.GetDefaultStatePtr()
+	}
+	if statePtr == nil {
+		return nil, nil, fmt.Errorf("cannot run command, no valid remote stateptr")
+	}
+	currentState, err := sstore.GetFullState(ctx, *statePtr)
+	if err != nil || currentState == nil {
 		return nil, nil, fmt.Errorf("cannot get current remote state: %w", err)
 	}
-	if currentState == nil {
-		currentState = msh.GetDefaultState()
-	}
-	if currentState == nil {
-		return nil, nil, fmt.Errorf("cannot run command, no valid remote state")
-	}
 	runPacket.State = addScVarsToState(currentState)
 	runPacket.StateComplete = true
 	msh.ServerProc.Output.RegisterRpc(runPacket.ReqId)
@@ -1250,19 +1288,19 @@ func RunCommand(ctx context.Context, sessionId string, windowId string, remotePt
 	if runPacket.Detached {
 		status = sstore.CmdStatusDetached
 	}
-	cmdState := stripScVarsFromState(runPacket.State)
 	cmd := &sstore.CmdType{
-		SessionId:   runPacket.CK.GetSessionId(),
-		CmdId:       runPacket.CK.GetCmdId(),
-		CmdStr:      runPacket.Command,
-		Remote:      remotePtr,
-		RemoteState: *cmdState,
-		TermOpts:    makeTermOpts(runPacket),
-		Status:      status,
-		StartPk:     startPk,
-		DonePk:      nil,
-		RunOut:      nil,
-		RtnState:    runPacket.ReturnState,
+		SessionId: runPacket.CK.GetSessionId(),
+		CmdId:     runPacket.CK.GetCmdId(),
+		CmdStr:    runPacket.Command,
+		Remote:    remotePtr,
+		FeState:   *sstore.FeStateFromShellState(currentState),
+		StatePtr:  *statePtr,
+		TermOpts:  makeTermOpts(runPacket),
+		Status:    status,
+		StartPk:   startPk,
+		DoneInfo:  nil,
+		RunOut:    nil,
+		RtnState:  runPacket.ReturnState,
 	}
 	err = sstore.CreateCmdPtyFile(ctx, cmd.SessionId, cmd.CmdId, cmd.TermOpts.MaxPtySize)
 	if err != nil {
@@ -1398,8 +1436,15 @@ func (msh *MShellProc) handleCmdDonePacket(donePk *packet.CmdDonePacketType) {
 	if donePk.FinalState != nil {
 		donePk.FinalState = stripScVarsFromState(donePk.FinalState)
 	}
-
-	update, err := sstore.UpdateCmdDonePk(context.Background(), donePk)
+	if donePk.FinalStateDiff != nil {
+		donePk.FinalStateDiff = stripScVarsFromStateDiff(donePk.FinalStateDiff)
+	}
+	doneInfo := &sstore.CmdDoneInfo{
+		Ts:         donePk.Ts,
+		ExitCode:   int64(donePk.ExitCode),
+		DurationMs: donePk.DurationMs,
+	}
+	update, err := sstore.UpdateCmdDoneInfo(context.Background(), donePk.CK, doneInfo)
 	if err != nil {
 		msh.WriteToPtyBuffer("*error updating cmddone: %v\n", err)
 		return
@@ -1411,8 +1456,8 @@ func (msh *MShellProc) handleCmdDonePacket(donePk *packet.CmdDonePacketType) {
 	}
 	update.ScreenWindows = sws
 	rct := msh.GetRunningCmd(donePk.CK)
+	var statePtr *sstore.ShellStatePtr
 	if donePk.FinalState != nil && rct != nil {
-		fmt.Printf("** FINALSTATE!\n")
 		feState := sstore.FeStateFromShellState(donePk.FinalState)
 		remoteInst, err := sstore.UpdateRemoteState(context.Background(), rct.SessionId, rct.WindowId, rct.RemotePtr, *feState, donePk.FinalState, nil)
 		if err != nil {
@@ -1422,20 +1467,13 @@ func (msh *MShellProc) handleCmdDonePacket(donePk *packet.CmdDonePacketType) {
 		if remoteInst != nil {
 			update.Sessions = sstore.MakeSessionsUpdateForRemote(rct.SessionId, remoteInst)
 		}
+		statePtr = &sstore.ShellStatePtr{BaseHash: donePk.FinalState.GetHashVal(false)}
 	} else if donePk.FinalStateDiff != nil && rct != nil {
-		fmt.Printf("** STATEDIFF! %#v\n", donePk.FinalStateDiff)
-		fullState, err := msh.getFullState(donePk.FinalStateDiff)
-		if err != nil {
-			fmt.Printf("**ERR: %v\n", err)
-		}
-		donePk.FinalStateDiff.Dump()
-		shexec.DumpVarMapFromState(fullState)
 		feState, err := msh.getFeStateFromDiff(donePk.FinalStateDiff)
 		if err != nil {
 			msh.WriteToPtyBuffer("*error trying to update remotestate: %v\n", err)
 			// fall-through (nothing to do)
 		} else {
-			fmt.Printf("** festate = %#v\n", feState)
 			remoteInst, err := sstore.UpdateRemoteState(context.Background(), rct.SessionId, rct.WindowId, rct.RemotePtr, *feState, nil, donePk.FinalStateDiff)
 			if err != nil {
 				msh.WriteToPtyBuffer("*error trying to update remotestate: %v\n", err)
@@ -1444,6 +1482,16 @@ func (msh *MShellProc) handleCmdDonePacket(donePk *packet.CmdDonePacketType) {
 			if remoteInst != nil {
 				update.Sessions = sstore.MakeSessionsUpdateForRemote(rct.SessionId, remoteInst)
 			}
+			diffHashArr := append(([]string)(nil), donePk.FinalStateDiff.DiffHashArr...)
+			diffHashArr = append(diffHashArr, donePk.FinalStateDiff.GetHashVal(false))
+			statePtr = &sstore.ShellStatePtr{BaseHash: donePk.FinalStateDiff.BaseHash, DiffHashArr: diffHashArr}
+		}
+	}
+	if statePtr != nil {
+		err = sstore.UpdateCmdRtnState(context.Background(), donePk.CK, *statePtr)
+		if err != nil {
+			msh.WriteToPtyBuffer("*error trying to update cmd rtnstate: %v\n", err)
+			// fall-through (nothing to do)
 		}
 	}
 	sstore.MainBus.SendUpdate(donePk.CK.GetSessionId(), update)
@@ -1457,7 +1505,7 @@ func (msh *MShellProc) handleCmdFinalPacket(finalPk *packet.CmdFinalPacketType)
 		log.Printf("error calling GetCmdById in handleCmdFinalPacket: %v\n", err)
 		return
 	}
-	if rtnCmd == nil || rtnCmd.DonePk != nil {
+	if rtnCmd == nil || rtnCmd.DoneInfo != nil {
 		return
 	}
 	log.Printf("finalpk %s (hangup): %s\n", finalPk.CK, finalPk.Error)
@@ -1743,7 +1791,7 @@ func (msh *MShellProc) getFullState(stateDiff *packet.ShellStateDiff) (*packet.S
 		}
 		return &newState, nil
 	} else {
-		fullState, err := sstore.GetFullState(context.Background(), stateDiff.BaseHash, stateDiff.DiffHashArr)
+		fullState, err := sstore.GetFullState(context.Background(), sstore.ShellStatePtr{stateDiff.BaseHash, stateDiff.DiffHashArr})
 		if err != nil {
 			return nil, err
 		}
@@ -1762,7 +1810,7 @@ func (msh *MShellProc) getFeStateFromDiff(stateDiff *packet.ShellStateDiff) (*ss
 		}
 		return sstore.FeStateFromShellState(&newState), nil
 	} else {
-		fullState, err := sstore.GetFullState(context.Background(), stateDiff.BaseHash, stateDiff.DiffHashArr)
+		fullState, err := sstore.GetFullState(context.Background(), sstore.ShellStatePtr{stateDiff.BaseHash, stateDiff.DiffHashArr})
 		if err != nil {
 			return nil, err
 		}

pkg/scpacket/scpacket.go

@@ -54,8 +54,7 @@ type UIContextType struct {
 	ScreenId  string                `json:"screenid"`
 	WindowId  string                `json:"windowid"`
 	Remote    *sstore.RemotePtrType `json:"remote,omitempty"`
-	TermOpts  *packet.TermOpts      `json:"termopts,omitempty"`
-	WinSize   *WinSize              `json:"winsize,omitempty"`
+	WinSize   *packet.WinSize       `json:"winsize,omitempty"`
 }
 
 type FeInputPacketType struct {

pkg/shparse/extend.go

@@ -15,7 +15,7 @@ func init() {
 	noEscChars = make([]bool, 256)
 	for ch := 0; ch < 256; ch++ {
 		if (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') ||
-			ch == '-' || ch == '.' || ch == '/' || ch == ':' || ch == '=' {
+			ch == '-' || ch == '.' || ch == '/' || ch == ':' || ch == '=' || ch == '_' {
 			noEscChars[byte(ch)] = true
 		}
 	}

pkg/sstore/dbops.go

@@ -661,8 +661,8 @@ func InsertLine(ctx context.Context, line *LineType, cmd *CmdType) error {
 			cmd.OrigTermOpts = cmd.TermOpts
 			cmdMap := cmd.ToMap()
 			query = `
-INSERT INTO cmd  ( sessionid, cmdid, remoteownerid, remoteid, remotename, cmdstr, remotestate, termopts, origtermopts, status, startpk, donepk, rtnstate, runout)
-          VALUES (:sessionid,:cmdid,:remoteownerid,:remoteid,:remotename,:cmdstr,:remotestate,:termopts,:origtermopts,:status,:startpk,:donepk,:rtnstate,:runout)
+INSERT INTO cmd  ( sessionid, cmdid, remoteownerid, remoteid, remotename, cmdstr, festate, statebasehash, statediffhasharr, termopts, origtermopts, status, startpk, doneinfo, rtnstate, runout, rtnbasehash, rtndiffhasharr)
+          VALUES (:sessionid,:cmdid,:remoteownerid,:remoteid,:remotename,:cmdstr,:festate,:statebasehash,:statediffhasharr,:termopts,:origtermopts,:status,:startpk,:doneinfo,:rtnstate,:runout,:rtnbasehash,:rtndiffhasharr)
 `
 			tx.NamedExecWrap(query, cmdMap)
 		}
@@ -684,10 +684,10 @@ func GetCmdById(ctx context.Context, sessionId string, cmdId string) (*CmdType,
 	return cmd, nil
 }
 
-func HasDonePk(ctx context.Context, ck base.CommandKey) (bool, error) {
+func HasDoneInfo(ctx context.Context, ck base.CommandKey) (bool, error) {
 	var found bool
 	txErr := WithTx(ctx, func(tx *TxWrap) error {
-		found = tx.Exists(`SELECT sessionid FROM cmd WHERE sessionid = ? AND cmdid = ? AND donepk is NOT NULL`, ck.GetSessionId(), ck.GetCmdId())
+		found = tx.Exists(`SELECT sessionid FROM cmd WHERE sessionid = ? AND cmdid = ? AND doneinfo is NOT NULL`, ck.GetSessionId(), ck.GetCmdId())
 		return nil
 	})
 	if txErr != nil {
@@ -696,16 +696,19 @@ func HasDonePk(ctx context.Context, ck base.CommandKey) (bool, error) {
 	return found, nil
 }
 
-func UpdateCmdDonePk(ctx context.Context, donePk *packet.CmdDonePacketType) (*ModelUpdate, error) {
-	if donePk == nil || donePk.CK.IsEmpty() {
-		return nil, fmt.Errorf("invalid cmddone packet (no ck)")
+func UpdateCmdDoneInfo(ctx context.Context, ck base.CommandKey, doneInfo *CmdDoneInfo) (*ModelUpdate, error) {
+	if doneInfo == nil {
+		return nil, fmt.Errorf("invalid cmddone packet")
+	}
+	if ck.IsEmpty() {
+		return nil, fmt.Errorf("cannot update cmddoneinfo, empty ck")
 	}
 	var rtnCmd *CmdType
 	txErr := WithTx(ctx, func(tx *TxWrap) error {
-		query := `UPDATE cmd SET status = ?, donepk = ? WHERE sessionid = ? AND cmdid = ?`
-		tx.ExecWrap(query, CmdStatusDone, quickJson(donePk), donePk.CK.GetSessionId(), donePk.CK.GetCmdId())
+		query := `UPDATE cmd SET status = ?, doneinfo = ? WHERE sessionid = ? AND cmdid = ?`
+		tx.ExecWrap(query, CmdStatusDone, quickJson(doneInfo), ck.GetSessionId(), ck.GetCmdId())
 		var err error
-		rtnCmd, err = GetCmdById(tx.Context(), donePk.CK.GetSessionId(), donePk.CK.GetCmdId())
+		rtnCmd, err = GetCmdById(tx.Context(), ck.GetSessionId(), ck.GetCmdId())
 		if err != nil {
 			return err
 		}
@@ -715,11 +718,26 @@ func UpdateCmdDonePk(ctx context.Context, donePk *packet.CmdDonePacketType) (*Mo
 		return nil, txErr
 	}
 	if rtnCmd == nil {
-		return nil, fmt.Errorf("cmd data not found for ck[%s]", donePk.CK)
+		return nil, fmt.Errorf("cmd data not found for ck[%s]", ck)
 	}
 	return &ModelUpdate{Cmd: rtnCmd}, nil
 }
 
+func UpdateCmdRtnState(ctx context.Context, ck base.CommandKey, statePtr ShellStatePtr) error {
+	if ck.IsEmpty() {
+		return fmt.Errorf("cannot update cmdrtnstate, empty ck")
+	}
+	txErr := WithTx(ctx, func(tx *TxWrap) error {
+		query := `UPDATE cmd SET rtnbasehash = ?, rtndiffhasharr = ? WHERE sessionid = ? AND cmdid = ?`
+		tx.ExecWrap(query, statePtr.BaseHash, quickJsonArr(statePtr.DiffHashArr), ck.GetSessionId(), ck.GetCmdId())
+		return nil
+	})
+	if txErr != nil {
+		return txErr
+	}
+	return nil
+}
+
 func AppendCmdErrorPk(ctx context.Context, errPk *packet.CmdErrorPacketType) error {
 	if errPk == nil || errPk.CK.IsEmpty() {
 		return fmt.Errorf("invalid cmderror packet (no ck)")
@@ -823,8 +841,23 @@ func DeleteScreen(ctx context.Context, sessionId string, screenId string) (Updat
 	return update, nil
 }
 
-func GetRemoteState(ctx context.Context, sessionId string, windowId string, remotePtr RemotePtrType) (*packet.ShellState, error) {
-	var state *packet.ShellState
+func GetRemoteState(ctx context.Context, sessionId string, windowId string, remotePtr RemotePtrType) (*packet.ShellState, *ShellStatePtr, error) {
+	ssptr, err := GetRemoteStatePtr(ctx, sessionId, windowId, remotePtr)
+	if err != nil {
+		return nil, nil, err
+	}
+	if ssptr == nil {
+		return nil, nil, nil
+	}
+	state, err := GetFullState(ctx, *ssptr)
+	if err != nil {
+		return nil, nil, err
+	}
+	return state, ssptr, err
+}
+
+func GetRemoteStatePtr(ctx context.Context, sessionId string, windowId string, remotePtr RemotePtrType) (*ShellStatePtr, error) {
+	var ssptr *ShellStatePtr
 	txErr := WithTx(ctx, func(tx *TxWrap) error {
 		ri, err := GetRemoteInstance(tx.Context(), sessionId, windowId, remotePtr)
 		if err != nil {
@@ -833,16 +866,13 @@ func GetRemoteState(ctx context.Context, sessionId string, windowId string, remo
 		if ri == nil {
 			return nil
 		}
-		state, err = GetFullState(tx.Context(), ri.StateBaseHash, ri.StateDiffHashArr)
-		if err != nil {
-			return err
-		}
+		ssptr = &ShellStatePtr{ri.StateBaseHash, ri.StateDiffHashArr}
 		return nil
 	})
 	if txErr != nil {
 		return nil, txErr
 	}
-	return state, nil
+	return ssptr, nil
 }
 
 func validateSessionWindow(tx *TxWrap, sessionId string, windowId string) error {
@@ -882,6 +912,7 @@ func GetRemoteInstance(ctx context.Context, sessionId string, windowId string, r
 func updateRIWithState(ctx context.Context, ri *RemoteInstance, stateBase *packet.ShellState, stateDiff *packet.ShellStateDiff) error {
 	if stateBase != nil {
 		ri.StateBaseHash = stateBase.GetHashVal(false)
+		ri.StateDiffHashArr = nil
 		err := StoreStateBase(ctx, stateBase)
 		if err != nil {
 			return err
@@ -897,7 +928,6 @@ func updateRIWithState(ctx context.Context, ri *RemoteInstance, stateBase *packe
 	return nil
 }
 
-// TODO - statediff
 func UpdateRemoteState(ctx context.Context, sessionId string, windowId string, remotePtr RemotePtrType, feState FeStateType, stateBase *packet.ShellState, stateDiff *packet.ShellStateDiff) (*RemoteInstance, error) {
 	if stateBase == nil && stateDiff == nil {
 		return nil, fmt.Errorf("UpdateRemoteState, must set state or diff")
@@ -936,13 +966,13 @@ func UpdateRemoteState(ctx context.Context, sessionId string, windowId string, r
 			tx.NamedExecWrap(query, ri.ToMap())
 			return nil
 		} else {
-			query = `UPDATE remote_instance SET festate = ? WHERE riid = ?`
+			query = `UPDATE remote_instance SET festate = ?, statebasehash = ?, statediffhasharr = ? WHERE riid = ?`
 			ri.FeState = feState
 			err = updateRIWithState(tx.Context(), ri, stateBase, stateDiff)
 			if err != nil {
 				return err
 			}
-			tx.ExecWrap(query, quickJson(ri.FeState), ri.RIId)
+			tx.ExecWrap(query, quickJson(ri.FeState), ri.StateBaseHash, quickJsonArr(ri.StateDiffHashArr), ri.RIId)
 			return nil
 		}
 	})
@@ -1370,24 +1400,24 @@ func StoreStateDiff(ctx context.Context, diff *packet.ShellStateDiff) error {
 }
 
 // returns error when not found
-func GetFullState(ctx context.Context, baseHash string, diffHashArr []string) (*packet.ShellState, error) {
+func GetFullState(ctx context.Context, ssPtr ShellStatePtr) (*packet.ShellState, error) {
 	var state *packet.ShellState
-	if baseHash == "" {
+	if ssPtr.BaseHash == "" {
 		return nil, fmt.Errorf("invalid empty basehash")
 	}
 	txErr := WithTx(ctx, func(tx *TxWrap) error {
 		var stateBase StateBase
 		query := `SELECT * FROM state_base WHERE basehash = ?`
-		found := tx.GetWrap(&stateBase, query, baseHash)
+		found := tx.GetWrap(&stateBase, query, ssPtr.BaseHash)
 		if !found {
-			return fmt.Errorf("ShellState %s not found", baseHash)
+			return fmt.Errorf("ShellState %s not found", ssPtr.BaseHash)
 		}
 		state = &packet.ShellState{}
 		err := state.DecodeShellState(stateBase.Data)
 		if err != nil {
 			return err
 		}
-		for idx, diffHash := range diffHashArr {
+		for idx, diffHash := range ssPtr.DiffHashArr {
 			query = `SELECT * FROM state_diff WHERE diffhash = ?`
 			m := tx.GetMap(query, diffHash)
 			stateDiff := StateDiffFromMap(m)

pkg/sstore/sstore.go

@@ -458,6 +458,18 @@ func (opts TermOpts) Value() (driver.Value, error) {
 	return quickValueJson(opts)
 }
 
+type ShellStatePtr struct {
+	BaseHash    string
+	DiffHashArr []string
+}
+
+func (ssptr *ShellStatePtr) IsEmpty() bool {
+	if ssptr == nil || ssptr.BaseHash == "" {
+		return true
+	}
+	return false
+}
+
 type RemoteInstance struct {
 	RIId             string      `json:"riid"`
 	Name             string      `json:"name"`
@@ -625,19 +637,27 @@ func (r *RemoteType) GetName() string {
 	return r.RemoteCanonicalName
 }
 
+type CmdDoneInfo struct {
+	Ts         int64 `json:"ts"`
+	ExitCode   int64 `json:"exitcode"`
+	DurationMs int64 `json:"durationms"`
+}
+
 type CmdType struct {
 	SessionId    string                     `json:"sessionid"`
 	CmdId        string                     `json:"cmdid"`
 	Remote       RemotePtrType              `json:"remote"`
 	CmdStr       string                     `json:"cmdstr"`
-	RemoteState  packet.ShellState          `json:"remotestate"`
+	FeState      FeStateType                `json:"festate"`
+	StatePtr     ShellStatePtr              `json:"state"`
 	TermOpts     TermOpts                   `json:"termopts"`
 	OrigTermOpts TermOpts                   `json:"origtermopts"`
 	Status       string                     `json:"status"`
 	StartPk      *packet.CmdStartPacketType `json:"startpk,omitempty"`
-	DonePk       *packet.CmdDonePacketType  `json:"donepk,omitempty"`
+	DoneInfo     *CmdDoneInfo               `json:"doneinfo,omitempty"`
 	RunOut       []packet.PacketType        `json:"runout,omitempty"`
 	RtnState     bool                       `json:"rtnstate,omitempty"`
+	RtnStatePtr  ShellStatePtr              `json:"rtnstateptr,omitempty"`
 	Remove       bool                       `json:"remove,omitempty"`
 }
 
@@ -694,14 +714,18 @@ func (cmd *CmdType) ToMap() map[string]interface{} {
 	rtn["remoteid"] = cmd.Remote.RemoteId
 	rtn["remotename"] = cmd.Remote.Name
 	rtn["cmdstr"] = cmd.CmdStr
-	rtn["remotestate"] = quickJson(cmd.RemoteState)
+	rtn["festate"] = quickJson(cmd.FeState)
+	rtn["statebasehash"] = cmd.StatePtr.BaseHash
+	rtn["statediffhasharr"] = quickJsonArr(cmd.StatePtr.DiffHashArr)
 	rtn["termopts"] = quickJson(cmd.TermOpts)
 	rtn["origtermopts"] = quickJson(cmd.OrigTermOpts)
 	rtn["status"] = cmd.Status
 	rtn["startpk"] = quickJson(cmd.StartPk)
-	rtn["donepk"] = quickJson(cmd.DonePk)
+	rtn["doneinfo"] = quickJson(cmd.DoneInfo)
 	rtn["runout"] = quickJson(cmd.RunOut)
 	rtn["rtnstate"] = cmd.RtnState
+	rtn["rtnbasehash"] = cmd.RtnStatePtr.BaseHash
+	rtn["rtndiffhasharr"] = quickJsonArr(cmd.RtnStatePtr.DiffHashArr)
 	return rtn
 }
 
@@ -716,14 +740,18 @@ func CmdFromMap(m map[string]interface{}) *CmdType {
 	quickSetStr(&cmd.Remote.RemoteId, m, "remoteid")
 	quickSetStr(&cmd.Remote.Name, m, "remotename")
 	quickSetStr(&cmd.CmdStr, m, "cmdstr")
-	quickSetJson(&cmd.RemoteState, m, "remotestate")
+	quickSetJson(&cmd.FeState, m, "festate")
+	quickSetStr(&cmd.StatePtr.BaseHash, m, "statebasehash")
+	quickSetJsonArr(&cmd.StatePtr.DiffHashArr, m, "statediffhasharr")
 	quickSetJson(&cmd.TermOpts, m, "termopts")
 	quickSetJson(&cmd.OrigTermOpts, m, "origtermopts")
 	quickSetStr(&cmd.Status, m, "status")
 	quickSetJson(&cmd.StartPk, m, "startpk")
-	quickSetJson(&cmd.DonePk, m, "donepk")
+	quickSetJson(&cmd.DoneInfo, m, "doneinfo")
 	quickSetJson(&cmd.RunOut, m, "runout")
 	quickSetBool(&cmd.RtnState, m, "rtnstate")
+	quickSetStr(&cmd.RtnStatePtr.BaseHash, m, "rtnbasehash")
+	quickSetJsonArr(&cmd.RtnStatePtr.DiffHashArr, m, "rtndiffhasharr")
 	return &cmd
 }
 

pkg/utilfn/utilfn.go

@@ -84,6 +84,16 @@ func ShellQuote(val string, forceQuote bool, maxLen int) string {
 	}
 }
 
+func EllipsisStr(s string, maxLen int) string {
+	if maxLen < 4 {
+		maxLen = 4
+	}
+	if len(s) > maxLen {
+		return s[0:maxLen-3] + "..."
+	}
+	return s
+}
+
 func LongestPrefix(root string, strs []string) string {
 	if len(strs) == 0 {
 		return root