save work

* break out telemetry and playbook

* break out bookmarks

* add license disclaimers

wavesrv/cmd/main-server.go

@@ -45,6 +45,7 @@ import (
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/scpacket"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/scws"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/sstore"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/telemetry"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/wsshell"
 )
 
@@ -211,7 +212,7 @@ func HandleLogActiveState(w http.ResponseWriter, r *http.Request) {
 		WriteJsonError(w, fmt.Errorf(ErrorDecodingJson, err))
 		return
 	}
-	activity := sstore.ActivityUpdate{}
+	activity := telemetry.ActivityUpdate{}
 	if activeState.Fg {
 		activity.FgMinutes = 1
 	}
@@ -222,7 +223,7 @@ func HandleLogActiveState(w http.ResponseWriter, r *http.Request) {
 		activity.OpenMinutes = 1
 	}
 	activity.NumConns = remote.NumRemotes()
-	err = sstore.UpdateCurrentActivity(r.Context(), activity)
+	err = telemetry.UpdateCurrentActivity(r.Context(), activity)
 	if err != nil {
 		WriteJsonError(w, fmt.Errorf("error updating activity: %w", err))
 		return
@@ -998,7 +999,7 @@ func main() {
 	}
 
 	log.Printf("PCLOUD_ENDPOINT=%s\n", pcloud.GetEndpoint())
-	sstore.UpdateActivityWrap(context.Background(), sstore.ActivityUpdate{NumConns: remote.NumRemotes()}, "numconns") // set at least one record into activity
+	telemetry.UpdateActivityWrap(context.Background(), telemetry.ActivityUpdate{NumConns: remote.NumRemotes()}, "numconns") // set at least one record into activity
 	installSignalHandlers()
 	go telemetryLoop()
 	go stdinReadWatch()

wavesrv/pkg/bookmarks/bookmarks.go

@@ -0,0 +1,204 @@
+// Copyright 2024, Command Line Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package bookmarks
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/dbutil"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/sstore"
+)
+
+type BookmarkType struct {
+	BookmarkId  string   `json:"bookmarkid"`
+	CreatedTs   int64    `json:"createdts"`
+	CmdStr      string   `json:"cmdstr"`
+	Alias       string   `json:"alias,omitempty"`
+	Tags        []string `json:"tags"`
+	Description string   `json:"description"`
+	OrderIdx    int64    `json:"orderidx"`
+	Remove      bool     `json:"remove,omitempty"`
+}
+
+func (bm *BookmarkType) GetSimpleKey() string {
+	return bm.BookmarkId
+}
+
+func (bm *BookmarkType) ToMap() map[string]interface{} {
+	rtn := make(map[string]interface{})
+	rtn["bookmarkid"] = bm.BookmarkId
+	rtn["createdts"] = bm.CreatedTs
+	rtn["cmdstr"] = bm.CmdStr
+	rtn["alias"] = bm.Alias
+	rtn["description"] = bm.Description
+	rtn["tags"] = dbutil.QuickJsonArr(bm.Tags)
+	return rtn
+}
+
+func (bm *BookmarkType) FromMap(m map[string]interface{}) bool {
+	dbutil.QuickSetStr(&bm.BookmarkId, m, "bookmarkid")
+	dbutil.QuickSetInt64(&bm.CreatedTs, m, "createdts")
+	dbutil.QuickSetStr(&bm.Alias, m, "alias")
+	dbutil.QuickSetStr(&bm.CmdStr, m, "cmdstr")
+	dbutil.QuickSetStr(&bm.Description, m, "description")
+	dbutil.QuickSetJsonArr(&bm.Tags, m, "tags")
+	return true
+}
+
+type bookmarkOrderType struct {
+	BookmarkId string
+	OrderIdx   int64
+}
+
+func GetBookmarks(ctx context.Context, tag string) ([]*BookmarkType, error) {
+	var bms []*BookmarkType
+	txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		var query string
+		if tag == "" {
+			query = `SELECT * FROM bookmark`
+			bms = dbutil.SelectMapsGen[*BookmarkType](tx, query)
+		} else {
+			query = `SELECT * FROM bookmark WHERE EXISTS (SELECT 1 FROM json_each(tags) WHERE value = ?)`
+			bms = dbutil.SelectMapsGen[*BookmarkType](tx, query, tag)
+		}
+		bmMap := dbutil.MakeGenMap(bms)
+		var orders []bookmarkOrderType
+		query = `SELECT bookmarkid, orderidx FROM bookmark_order WHERE tag = ?`
+		tx.Select(&orders, query, tag)
+		for _, bmOrder := range orders {
+			bm := bmMap[bmOrder.BookmarkId]
+			if bm != nil {
+				bm.OrderIdx = bmOrder.OrderIdx
+			}
+		}
+		return nil
+	})
+	if txErr != nil {
+		return nil, txErr
+	}
+	return bms, nil
+}
+
+func GetBookmarkById(ctx context.Context, bookmarkId string, tag string) (*BookmarkType, error) {
+	var rtn *BookmarkType
+	txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		query := `SELECT * FROM bookmark WHERE bookmarkid = ?`
+		rtn = dbutil.GetMapGen[*BookmarkType](tx, query, bookmarkId)
+		if rtn == nil {
+			return nil
+		}
+		query = `SELECT orderidx FROM bookmark_order WHERE bookmarkid = ? AND tag = ?`
+		orderIdx := tx.GetInt(query, bookmarkId, tag)
+		rtn.OrderIdx = int64(orderIdx)
+		return nil
+	})
+	if txErr != nil {
+		return nil, txErr
+	}
+	return rtn, nil
+}
+
+func GetBookmarkIdByArg(ctx context.Context, bookmarkArg string) (string, error) {
+	var rtnId string
+	txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		if len(bookmarkArg) == 8 {
+			query := `SELECT bookmarkid FROM bookmark WHERE bookmarkid LIKE (? || '%')`
+			rtnId = tx.GetString(query, bookmarkArg)
+			return nil
+		}
+		query := `SELECT bookmarkid FROM bookmark WHERE bookmarkid = ?`
+		rtnId = tx.GetString(query, bookmarkArg)
+		return nil
+	})
+	if txErr != nil {
+		return "", txErr
+	}
+	return rtnId, nil
+}
+
+func GetBookmarkIdsByCmdStr(ctx context.Context, cmdStr string) ([]string, error) {
+	return sstore.WithTxRtn(ctx, func(tx *sstore.TxWrap) ([]string, error) {
+		query := `SELECT bookmarkid FROM bookmark WHERE cmdstr = ?`
+		bmIds := tx.SelectStrings(query, cmdStr)
+		return bmIds, nil
+	})
+}
+
+// ignores OrderIdx field
+func InsertBookmark(ctx context.Context, bm *BookmarkType) error {
+	if bm == nil || bm.BookmarkId == "" {
+		return fmt.Errorf("invalid empty bookmark id")
+	}
+	txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		query := `SELECT bookmarkid FROM bookmark WHERE bookmarkid = ?`
+		if tx.Exists(query, bm.BookmarkId) {
+			return fmt.Errorf("bookmarkid already exists")
+		}
+		query = `INSERT INTO bookmark ( bookmarkid, createdts, cmdstr, alias, tags, description)
+                               VALUES (:bookmarkid,:createdts,:cmdstr,:alias,:tags,:description)`
+		tx.NamedExec(query, bm.ToMap())
+		for _, tag := range append(bm.Tags, "") {
+			query = `SELECT COALESCE(max(orderidx), 0) FROM bookmark_order WHERE tag = ?`
+			maxOrder := tx.GetInt(query, tag)
+			query = `INSERT INTO bookmark_order (tag, bookmarkid, orderidx) VALUES (?, ?, ?)`
+			tx.Exec(query, tag, bm.BookmarkId, maxOrder+1)
+		}
+		return nil
+	})
+	return txErr
+}
+
+const (
+	BookmarkField_Desc   = "desc"
+	BookmarkField_CmdStr = "cmdstr"
+)
+
+func EditBookmark(ctx context.Context, bookmarkId string, editMap map[string]interface{}) error {
+	txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		query := `SELECT bookmarkid FROM bookmark WHERE bookmarkid = ?`
+		if !tx.Exists(query, bookmarkId) {
+			return fmt.Errorf("bookmark not found")
+		}
+		if desc, found := editMap[BookmarkField_Desc]; found {
+			query = `UPDATE bookmark SET description = ? WHERE bookmarkid = ?`
+			tx.Exec(query, desc, bookmarkId)
+		}
+		if cmdStr, found := editMap[BookmarkField_CmdStr]; found {
+			query = `UPDATE bookmark SET cmdstr = ? WHERE bookmarkid = ?`
+			tx.Exec(query, cmdStr, bookmarkId)
+		}
+		return nil
+	})
+	return txErr
+}
+
+func fixupBookmarkOrder(tx *sstore.TxWrap) {
+	query := `
+WITH new_order AS (
+  SELECT tag, bookmarkid, row_number() OVER (PARTITION BY tag ORDER BY orderidx) AS newidx FROM bookmark_order
+)
+UPDATE bookmark_order
+SET orderidx = new_order.newidx
+FROM new_order
+WHERE bookmark_order.tag = new_order.tag AND bookmark_order.bookmarkid = new_order.bookmarkid
+`
+	tx.Exec(query)
+}
+
+func DeleteBookmark(ctx context.Context, bookmarkId string) error {
+	txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		query := `SELECT bookmarkid FROM bookmark WHERE bookmarkid = ?`
+		if !tx.Exists(query, bookmarkId) {
+			return fmt.Errorf("bookmark not found")
+		}
+		query = `DELETE FROM bookmark WHERE bookmarkid = ?`
+		tx.Exec(query, bookmarkId)
+		query = `DELETE FROM bookmark_order WHERE bookmarkid = ?`
+		tx.Exec(query, bookmarkId)
+		fixupBookmarkOrder(tx)
+		return nil
+	})
+	return txErr
+}

wavesrv/pkg/bookmarks/updatetypes.go

@@ -0,0 +1,23 @@
+// Copyright 2024, Command Line Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package bookmarks
+
+import "github.com/wavetermdev/waveterm/wavesrv/pkg/scbus"
+
+type BookmarksUpdate struct {
+	Bookmarks        []*BookmarkType `json:"bookmarks"`
+	SelectedBookmark string          `json:"selectedbookmark,omitempty"`
+}
+
+func (BookmarksUpdate) GetType() string {
+	return "bookmarks"
+}
+
+func AddBookmarksUpdate(update *scbus.ModelUpdatePacketType, bookmarks []*BookmarkType, selectedBookmark *string) {
+	if selectedBookmark == nil {
+		update.AddUpdate(BookmarksUpdate{Bookmarks: bookmarks})
+	} else {
+		update.AddUpdate(BookmarksUpdate{Bookmarks: bookmarks, SelectedBookmark: *selectedBookmark})
+	}
+}

wavesrv/pkg/cmdrunner/cmdrunner.go

@@ -35,8 +35,10 @@ import (
 	"github.com/wavetermdev/waveterm/waveshell/pkg/shellutil"
 	"github.com/wavetermdev/waveterm/waveshell/pkg/shexec"
 	"github.com/wavetermdev/waveterm/waveshell/pkg/utilfn"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/bookmarks"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/comp"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/dbutil"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/history"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/pcloud"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/promptenc"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/releasechecker"
@@ -47,6 +49,7 @@ import (
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/scbus"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/scpacket"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/sstore"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/telemetry"
 	"golang.org/x/mod/semver"
 )
 
@@ -473,7 +476,7 @@ func doHistoryExpansion(ctx context.Context, ids resolvedIds, hnum int) (string,
 	foundHistoryNum := hnum
 	if hnum == -1 {
 		var err error
-		foundHistoryNum, err = sstore.GetLastHistoryLineNum(ctx, ids.ScreenId)
+		foundHistoryNum, err = history.GetLastHistoryLineNum(ctx, ids.ScreenId)
 		if err != nil {
 			return "", fmt.Errorf("cannot expand history, error finding last history item: %v", err)
 		}
@@ -481,7 +484,7 @@ func doHistoryExpansion(ctx context.Context, ids resolvedIds, hnum int) (string,
 			return "", fmt.Errorf("cannot expand history, no last history item")
 		}
 	}
-	hitem, err := sstore.GetHistoryItemByLineNum(ctx, ids.ScreenId, foundHistoryNum)
+	hitem, err := history.GetHistoryItemByLineNum(ctx, ids.ScreenId, foundHistoryNum)
 	if err != nil {
 		return "", fmt.Errorf("cannot get history item '%d': %v", foundHistoryNum, err)
 	}
@@ -666,7 +669,7 @@ func addToHistory(ctx context.Context, pk *scpacket.FeCommandPacketType, history
 	if err != nil {
 		return err
 	}
-	hitem := &sstore.HistoryItemType{
+	hitem := &history.HistoryItemType{
 		HistoryId: scbase.GenWaveUUID(),
 		Ts:        time.Now().UnixMilli(),
 		UserId:    DefaultUserId,
@@ -690,7 +693,7 @@ func addToHistory(ctx context.Context, pk *scpacket.FeCommandPacketType, history
 	if !isMetaCmd && historyContext.RemotePtr != nil {
 		hitem.Remote = *historyContext.RemotePtr
 	}
-	err = sstore.InsertHistoryItem(ctx, hitem)
+	err = history.InsertHistoryItem(ctx, hitem)
 	if err != nil {
 		return err
 	}
@@ -706,7 +709,7 @@ func EvalCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (scbus.U
 	}
 	evalDepth := getEvalDepth(ctx)
 	if pk.Interactive && evalDepth == 0 {
-		sstore.UpdateActivityWrap(ctx, sstore.ActivityUpdate{NumCommands: 1}, "numcommands")
+		telemetry.UpdateActivityWrap(ctx, telemetry.ActivityUpdate{NumCommands: 1}, "numcommands")
 	}
 	if evalDepth > MaxEvalDepth {
 		return nil, fmt.Errorf("alias/history expansion max-depth exceeded")
@@ -3352,8 +3355,8 @@ func validateRemoteColor(color string, typeStr string) error {
 }
 
 func SessionOpenSharedCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (scbus.UpdatePacket, error) {
-	activity := sstore.ActivityUpdate{ClickShared: 1}
-	sstore.UpdateActivityWrap(ctx, activity, "click-shared")
+	activity := telemetry.ActivityUpdate{ClickShared: 1}
+	telemetry.UpdateActivityWrap(ctx, activity, "click-shared")
 	return nil, fmt.Errorf("shared sessions are not available in this version of prompt (stay tuned)")
 }
 
@@ -3615,11 +3618,11 @@ func MainViewCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (scb
 	update := scbus.MakeUpdatePacket()
 	mainViewArg := pk.Args[0]
 	if mainViewArg == sstore.MainViewSession {
-		update.AddUpdate(&sstore.MainViewUpdate{MainView: sstore.MainViewSession})
+		update.AddUpdate(&MainViewUpdate{MainView: sstore.MainViewSession})
 	} else if mainViewArg == sstore.MainViewConnections {
-		update.AddUpdate(&sstore.MainViewUpdate{MainView: sstore.MainViewConnections})
+		update.AddUpdate(&MainViewUpdate{MainView: sstore.MainViewConnections})
 	} else if mainViewArg == sstore.MainViewSettings {
-		update.AddUpdate(&sstore.MainViewUpdate{MainView: sstore.MainViewSettings})
+		update.AddUpdate(&MainViewUpdate{MainView: sstore.MainViewSettings})
 	} else if mainViewArg == sstore.MainViewHistory {
 		return nil, fmt.Errorf("use /history instead")
 	} else if mainViewArg == sstore.MainViewBookmarks {
@@ -3825,7 +3828,7 @@ func HistoryPurgeCommand(ctx context.Context, pk *scpacket.FeCommandPacketType)
 		}
 		historyIds = append(historyIds, historyArg)
 	}
-	err := sstore.PurgeHistoryByIds(ctx, historyIds)
+	err := history.PurgeHistoryByIds(ctx, historyIds)
 	if err != nil {
 		return nil, fmt.Errorf("/history:purge error purging items: %v", err)
 	}
@@ -3837,7 +3840,7 @@ const HistoryViewPageSize = 50
 var cmdFilterLs = regexp.MustCompile(`^ls(\s|$)`)
 var cmdFilterCd = regexp.MustCompile(`^cd(\s|$)`)
 
-func historyCmdFilter(hitem *sstore.HistoryItemType) bool {
+func historyCmdFilter(hitem *history.HistoryItemType) bool {
 	cmdStr := hitem.CmdStr
 	if cmdStr == "" || strings.Index(cmdStr, ";") != -1 || strings.Index(cmdStr, "\n") != -1 {
 		return true
@@ -3864,7 +3867,7 @@ func HistoryViewAllCommand(ctx context.Context, pk *scpacket.FeCommandPacketType
 	if err != nil {
 		return nil, err
 	}
-	opts := sstore.HistoryQueryOpts{MaxItems: HistoryViewPageSize, Offset: offset, RawOffset: rawOffset}
+	opts := history.HistoryQueryOpts{MaxItems: HistoryViewPageSize, Offset: offset, RawOffset: rawOffset}
 	if pk.Kwargs["text"] != "" {
 		opts.SearchText = pk.Kwargs["text"]
 	}
@@ -3903,25 +3906,25 @@ func HistoryViewAllCommand(ctx context.Context, pk *scpacket.FeCommandPacketType
 	if err != nil {
 		return nil, fmt.Errorf("invalid meta arg (must be boolean): %v", err)
 	}
-	hresult, err := sstore.GetHistoryItems(ctx, opts)
+	hresult, err := history.GetHistoryItems(ctx, opts)
 	if err != nil {
 		return nil, err
 	}
-	hvdata := &sstore.HistoryViewData{
+	hvdata := &history.HistoryViewData{
 		Items:         hresult.Items,
 		Offset:        hresult.Offset,
 		RawOffset:     hresult.RawOffset,
 		NextRawOffset: hresult.NextRawOffset,
 		HasMore:       hresult.HasMore,
 	}
-	lines, cmds, err := sstore.GetLineCmdsFromHistoryItems(ctx, hvdata.Items)
+	lines, cmds, err := history.GetLineCmdsFromHistoryItems(ctx, hvdata.Items)
 	if err != nil {
 		return nil, err
 	}
 	hvdata.Lines = lines
 	hvdata.Cmds = cmds
 	update := scbus.MakeUpdatePacket()
-	update.AddUpdate(&sstore.MainViewUpdate{MainView: sstore.MainViewHistory, HistoryView: hvdata})
+	update.AddUpdate(&MainViewUpdate{MainView: sstore.MainViewHistory, HistoryView: hvdata})
 	return update, nil
 }
 
@@ -3957,17 +3960,17 @@ func HistoryCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (scbu
 	} else if htype == HistoryTypeSession {
 		hScreenId = ""
 	}
-	hopts := sstore.HistoryQueryOpts{MaxItems: maxItems, SessionId: hSessionId, ScreenId: hScreenId}
-	hresult, err := sstore.GetHistoryItems(ctx, hopts)
+	hopts := history.HistoryQueryOpts{MaxItems: maxItems, SessionId: hSessionId, ScreenId: hScreenId}
+	hresult, err := history.GetHistoryItems(ctx, hopts)
 	if err != nil {
 		return nil, err
 	}
 	show := !resolveBool(pk.Kwargs["noshow"], false)
 	if show {
-		sstore.UpdateActivityWrap(ctx, sstore.ActivityUpdate{HistoryView: 1}, "history")
+		telemetry.UpdateActivityWrap(ctx, telemetry.ActivityUpdate{HistoryView: 1}, "history")
 	}
 	update := scbus.MakeUpdatePacket()
-	update.AddUpdate(sstore.HistoryInfoType{
+	update.AddUpdate(history.HistoryInfoType{
 		HistoryType: htype,
 		SessionId:   ids.SessionId,
 		ScreenId:    ids.ScreenId,
@@ -4310,16 +4313,16 @@ func BookmarksShowCommand(ctx context.Context, pk *scpacket.FeCommandPacketType)
 	if len(pk.Args) > 0 {
 		tagName = pk.Args[0]
 	}
-	bms, err := sstore.GetBookmarks(ctx, tagName)
+	bms, err := bookmarks.GetBookmarks(ctx, tagName)
 	if err != nil {
 		return nil, fmt.Errorf("cannot retrieve bookmarks: %v", err)
 	}
-	sstore.UpdateActivityWrap(ctx, sstore.ActivityUpdate{BookmarksView: 1}, "bookmarks")
+	telemetry.UpdateActivityWrap(ctx, telemetry.ActivityUpdate{BookmarksView: 1}, "bookmarks")
 	update := scbus.MakeUpdatePacket()
 
-	update.AddUpdate(&sstore.MainViewUpdate{
+	update.AddUpdate(&MainViewUpdate{
 		MainView:      sstore.MainViewBookmarks,
-		BookmarksView: &sstore.BookmarksUpdate{Bookmarks: bms},
+		BookmarksView: &bookmarks.BookmarksUpdate{Bookmarks: bms},
 	})
 	return update, nil
 }
@@ -4329,7 +4332,7 @@ func BookmarkSetCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (
 		return nil, fmt.Errorf("/bookmark:set requires one argument (bookmark id)")
 	}
 	bookmarkArg := pk.Args[0]
-	bookmarkId, err := sstore.GetBookmarkIdByArg(ctx, bookmarkArg)
+	bookmarkId, err := bookmarks.GetBookmarkIdByArg(ctx, bookmarkArg)
 	if err != nil {
 		return nil, fmt.Errorf("error trying to resolve bookmark: %v", err)
 	}
@@ -4338,25 +4341,25 @@ func BookmarkSetCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (
 	}
 	editMap := make(map[string]interface{})
 	if descStr, found := pk.Kwargs["desc"]; found {
-		editMap[sstore.BookmarkField_Desc] = descStr
+		editMap[bookmarks.BookmarkField_Desc] = descStr
 	}
 	if cmdStr, found := pk.Kwargs["cmdstr"]; found {
-		editMap[sstore.BookmarkField_CmdStr] = cmdStr
+		editMap[bookmarks.BookmarkField_CmdStr] = cmdStr
 	}
 	if len(editMap) == 0 {
 		return nil, fmt.Errorf("no fields set, can set %s", formatStrs([]string{"desc", "cmdstr"}, "or", false))
 	}
-	err = sstore.EditBookmark(ctx, bookmarkId, editMap)
+	err = bookmarks.EditBookmark(ctx, bookmarkId, editMap)
 	if err != nil {
 		return nil, fmt.Errorf("error trying to edit bookmark: %v", err)
 	}
-	bm, err := sstore.GetBookmarkById(ctx, bookmarkId, "")
+	bm, err := bookmarks.GetBookmarkById(ctx, bookmarkId, "")
 	if err != nil {
 		return nil, fmt.Errorf("error retrieving edited bookmark: %v", err)
 	}
-	bms := []*sstore.BookmarkType{bm}
+	bms := []*bookmarks.BookmarkType{bm}
 	update := scbus.MakeUpdatePacket()
-	sstore.AddBookmarksUpdate(update, bms, nil)
+	bookmarks.AddBookmarksUpdate(update, bms, nil)
 	update.AddUpdate(sstore.InfoMsgUpdate("bookmark edited"))
 	return update, nil
 }
@@ -4366,20 +4369,20 @@ func BookmarkDeleteCommand(ctx context.Context, pk *scpacket.FeCommandPacketType
 		return nil, fmt.Errorf("/bookmark:delete requires one argument (bookmark id)")
 	}
 	bookmarkArg := pk.Args[0]
-	bookmarkId, err := sstore.GetBookmarkIdByArg(ctx, bookmarkArg)
+	bookmarkId, err := bookmarks.GetBookmarkIdByArg(ctx, bookmarkArg)
 	if err != nil {
 		return nil, fmt.Errorf("error trying to resolve bookmark: %v", err)
 	}
 	if bookmarkId == "" {
 		return nil, fmt.Errorf("bookmark not found")
 	}
-	err = sstore.DeleteBookmark(ctx, bookmarkId)
+	err = bookmarks.DeleteBookmark(ctx, bookmarkId)
 	if err != nil {
 		return nil, fmt.Errorf("error deleting bookmark: %v", err)
 	}
 	update := scbus.MakeUpdatePacket()
-	bms := []*sstore.BookmarkType{{BookmarkId: bookmarkId, Remove: true}}
-	sstore.AddBookmarksUpdate(update, bms, nil)
+	bms := []*bookmarks.BookmarkType{{BookmarkId: bookmarkId, Remove: true}}
+	bookmarks.AddBookmarksUpdate(update, bms, nil)
 	update.AddUpdate(sstore.InfoMsgUpdate("bookmark deleted"))
 	return update, nil
 }
@@ -4407,7 +4410,7 @@ func LineBookmarkCommand(ctx context.Context, pk *scpacket.FeCommandPacketType)
 	if cmdObj == nil {
 		return nil, fmt.Errorf("cannot bookmark non-cmd line")
 	}
-	existingBmIds, err := sstore.GetBookmarkIdsByCmdStr(ctx, cmdObj.CmdStr)
+	existingBmIds, err := bookmarks.GetBookmarkIdsByCmdStr(ctx, cmdObj.CmdStr)
 	if err != nil {
 		return nil, fmt.Errorf("error trying to retrieve current boookmarks: %v", err)
 	}
@@ -4415,7 +4418,7 @@ func LineBookmarkCommand(ctx context.Context, pk *scpacket.FeCommandPacketType)
 	if len(existingBmIds) > 0 {
 		newBmId = existingBmIds[0]
 	} else {
-		newBm := &sstore.BookmarkType{
+		newBm := &bookmarks.BookmarkType{
 			BookmarkId:  uuid.New().String(),
 			CreatedTs:   time.Now().UnixMilli(),
 			CmdStr:      cmdObj.CmdStr,
@@ -4423,17 +4426,17 @@ func LineBookmarkCommand(ctx context.Context, pk *scpacket.FeCommandPacketType)
 			Tags:        nil,
 			Description: "",
 		}
-		err = sstore.InsertBookmark(ctx, newBm)
+		err = bookmarks.InsertBookmark(ctx, newBm)
 		if err != nil {
 			return nil, fmt.Errorf("cannot insert bookmark: %v", err)
 		}
 		newBmId = newBm.BookmarkId
 	}
-	bms, err := sstore.GetBookmarks(ctx, "")
+	bms, err := bookmarks.GetBookmarks(ctx, "")
 	update := scbus.MakeUpdatePacket()
-	update.AddUpdate(&sstore.MainViewUpdate{
+	update.AddUpdate(&MainViewUpdate{
 		MainView:      sstore.MainViewBookmarks,
-		BookmarksView: &sstore.BookmarksUpdate{Bookmarks: bms, SelectedBookmark: newBmId},
+		BookmarksView: &bookmarks.BookmarksUpdate{Bookmarks: bms, SelectedBookmark: newBmId},
 	})
 	return update, nil
 }

wavesrv/pkg/cmdrunner/updatetypes.go

@@ -0,0 +1,19 @@
+// Copyright 2024, Command Line Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package cmdrunner
+
+import (
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/bookmarks"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/history"
+)
+
+type MainViewUpdate struct {
+	MainView      string                     `json:"mainview"`
+	HistoryView   *history.HistoryViewData   `json:"historyview,omitempty"`
+	BookmarksView *bookmarks.BookmarksUpdate `json:"bookmarksview,omitempty"`
+}
+
+func (MainViewUpdate) GetType() string {
+	return "mainview"
+}

wavesrv/pkg/history/history.go

@@ -0,0 +1,327 @@
+// Copyright 2024, Command Line Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package history
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/google/uuid"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/dbutil"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/sstore"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/workspaces/line"
+)
+
+type HistoryItemType struct {
+	HistoryId  string               `json:"historyid"`
+	Ts         int64                `json:"ts"`
+	UserId     string               `json:"userid"`
+	SessionId  string               `json:"sessionid"`
+	ScreenId   string               `json:"screenid"`
+	LineId     string               `json:"lineid"`
+	HadError   bool                 `json:"haderror"`
+	CmdStr     string               `json:"cmdstr"`
+	Remote     sstore.RemotePtrType `json:"remote"`
+	IsMetaCmd  bool                 `json:"ismetacmd"`
+	ExitCode   *int64               `json:"exitcode,omitempty"`
+	DurationMs *int64               `json:"durationms,omitempty"`
+	FeState    sstore.FeStateType   `json:"festate,omitempty"`
+	Tags       map[string]bool      `json:"tags,omitempty"`
+	LineNum    int64                `json:"linenum" dbmap:"-"`
+	Status     string               `json:"status"`
+
+	// only for updates
+	Remove bool `json:"remove" dbmap:"-"`
+
+	// transient (string because of different history orderings)
+	HistoryNum string `json:"historynum" dbmap:"-"`
+}
+
+func (h *HistoryItemType) ToMap() map[string]interface{} {
+	rtn := make(map[string]interface{})
+	rtn["historyid"] = h.HistoryId
+	rtn["ts"] = h.Ts
+	rtn["userid"] = h.UserId
+	rtn["sessionid"] = h.SessionId
+	rtn["screenid"] = h.ScreenId
+	rtn["lineid"] = h.LineId
+	rtn["linenum"] = h.LineNum
+	rtn["haderror"] = h.HadError
+	rtn["cmdstr"] = h.CmdStr
+	rtn["remoteownerid"] = h.Remote.OwnerId
+	rtn["remoteid"] = h.Remote.RemoteId
+	rtn["remotename"] = h.Remote.Name
+	rtn["ismetacmd"] = h.IsMetaCmd
+	rtn["exitcode"] = h.ExitCode
+	rtn["durationms"] = h.DurationMs
+	rtn["festate"] = dbutil.QuickJson(h.FeState)
+	rtn["tags"] = dbutil.QuickJson(h.Tags)
+	rtn["status"] = h.Status
+	return rtn
+}
+
+func (h *HistoryItemType) FromMap(m map[string]interface{}) bool {
+	dbutil.QuickSetStr(&h.HistoryId, m, "historyid")
+	dbutil.QuickSetInt64(&h.Ts, m, "ts")
+	dbutil.QuickSetStr(&h.UserId, m, "userid")
+	dbutil.QuickSetStr(&h.SessionId, m, "sessionid")
+	dbutil.QuickSetStr(&h.ScreenId, m, "screenid")
+	dbutil.QuickSetStr(&h.LineId, m, "lineid")
+	dbutil.QuickSetBool(&h.HadError, m, "haderror")
+	dbutil.QuickSetStr(&h.CmdStr, m, "cmdstr")
+	dbutil.QuickSetStr(&h.Remote.OwnerId, m, "remoteownerid")
+	dbutil.QuickSetStr(&h.Remote.RemoteId, m, "remoteid")
+	dbutil.QuickSetStr(&h.Remote.Name, m, "remotename")
+	dbutil.QuickSetBool(&h.IsMetaCmd, m, "ismetacmd")
+	dbutil.QuickSetStr(&h.HistoryNum, m, "historynum")
+	dbutil.QuickSetInt64(&h.LineNum, m, "linenum")
+	dbutil.QuickSetNullableInt64(&h.ExitCode, m, "exitcode")
+	dbutil.QuickSetNullableInt64(&h.DurationMs, m, "durationms")
+	dbutil.QuickSetJson(&h.FeState, m, "festate")
+	dbutil.QuickSetJson(&h.Tags, m, "tags")
+	dbutil.QuickSetStr(&h.Status, m, "status")
+	return true
+}
+
+type HistoryQueryOpts struct {
+	Offset     int
+	MaxItems   int
+	FromTs     int64
+	SearchText string
+	SessionId  string
+	RemoteId   string
+	ScreenId   string
+	NoMeta     bool
+	RawOffset  int
+	FilterFn   func(*HistoryItemType) bool
+}
+
+type HistoryQueryResult struct {
+	MaxItems      int
+	Items         []*HistoryItemType
+	Offset        int // the offset shown to user
+	RawOffset     int // internal offset
+	HasMore       bool
+	NextRawOffset int // internal offset used by pager for next query
+
+	prevItems int // holds number of items skipped by RawOffset
+}
+
+type HistoryViewData struct {
+	Items         []*HistoryItemType `json:"items"`
+	Offset        int                `json:"offset"`
+	RawOffset     int                `json:"rawoffset"`
+	NextRawOffset int                `json:"nextrawoffset"`
+	HasMore       bool               `json:"hasmore"`
+	Lines         []*line.LineType   `json:"lines"`
+	Cmds          []*sstore.CmdType  `json:"cmds"`
+}
+
+const HistoryCols = "h.historyid, h.ts, h.userid, h.sessionid, h.screenid, h.lineid, h.haderror, h.cmdstr, h.remoteownerid, h.remoteid, h.remotename, h.ismetacmd, h.linenum, h.exitcode, h.durationms, h.festate, h.tags, h.status"
+const DefaultMaxHistoryItems = 1000
+
+func InsertHistoryItem(ctx context.Context, hitem *HistoryItemType) error {
+	if hitem == nil {
+		return fmt.Errorf("cannot insert nil history item")
+	}
+	txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		query := `INSERT INTO history 
+                  ( historyid, ts, userid, sessionid, screenid, lineid, haderror, cmdstr, remoteownerid, remoteid, remotename, ismetacmd, linenum, exitcode, durationms, festate, tags, status) VALUES
+                  (:historyid,:ts,:userid,:sessionid,:screenid,:lineid,:haderror,:cmdstr,:remoteownerid,:remoteid,:remotename,:ismetacmd,:linenum,:exitcode,:durationms,:festate,:tags,:status)`
+		tx.NamedExec(query, hitem.ToMap())
+		return nil
+	})
+	return txErr
+}
+
+const HistoryQueryChunkSize = 1000
+
+func _getNextHistoryItem(items []*HistoryItemType, index int, filterFn func(*HistoryItemType) bool) (*HistoryItemType, int) {
+	for ; index < len(items); index++ {
+		item := items[index]
+		if filterFn(item) {
+			return item, index
+		}
+	}
+	return nil, index
+}
+
+// returns true if done, false if we still need to process more items
+func (result *HistoryQueryResult) processItem(item *HistoryItemType, rawOffset int) bool {
+	if result.prevItems < result.Offset {
+		result.prevItems++
+		return false
+	}
+	if len(result.Items) == result.MaxItems {
+		result.HasMore = true
+		result.NextRawOffset = rawOffset
+		return true
+	}
+	if len(result.Items) == 0 {
+		result.RawOffset = rawOffset
+	}
+	result.Items = append(result.Items, item)
+	return false
+}
+
+func runHistoryQueryWithFilter(tx *sstore.TxWrap, opts HistoryQueryOpts) (*HistoryQueryResult, error) {
+	if opts.MaxItems == 0 {
+		return nil, fmt.Errorf("invalid query, maxitems is 0")
+	}
+	rtn := &HistoryQueryResult{Offset: opts.Offset, MaxItems: opts.MaxItems}
+	var rawOffset int
+	if opts.RawOffset >= opts.Offset {
+		rtn.prevItems = opts.Offset
+		rawOffset = opts.RawOffset
+	} else {
+		rawOffset = 0
+	}
+	for {
+		resultItems, err := runHistoryQuery(tx, opts, rawOffset, HistoryQueryChunkSize)
+		if err != nil {
+			return nil, err
+		}
+		isDone := false
+		for resultIdx := 0; resultIdx < len(resultItems); resultIdx++ {
+			if opts.FilterFn != nil && !opts.FilterFn(resultItems[resultIdx]) {
+				continue
+			}
+			isDone = rtn.processItem(resultItems[resultIdx], rawOffset+resultIdx)
+			if isDone {
+				break
+			}
+		}
+		if isDone {
+			break
+		}
+		if len(resultItems) < HistoryQueryChunkSize {
+			break
+		}
+		rawOffset += HistoryQueryChunkSize
+	}
+	return rtn, nil
+}
+
+func runHistoryQuery(tx *sstore.TxWrap, opts HistoryQueryOpts, realOffset int, itemLimit int) ([]*HistoryItemType, error) {
+	// check sessionid/screenid format because we are directly inserting them into the SQL
+	if opts.SessionId != "" {
+		_, err := uuid.Parse(opts.SessionId)
+		if err != nil {
+			return nil, fmt.Errorf("malformed sessionid")
+		}
+	}
+	if opts.ScreenId != "" {
+		_, err := uuid.Parse(opts.ScreenId)
+		if err != nil {
+			return nil, fmt.Errorf("malformed screenid")
+		}
+	}
+	if opts.RemoteId != "" {
+		_, err := uuid.Parse(opts.RemoteId)
+		if err != nil {
+			return nil, fmt.Errorf("malformed remoteid")
+		}
+	}
+	whereClause := "WHERE 1"
+	var queryArgs []interface{}
+	hNumStr := ""
+	if opts.SessionId != "" && opts.ScreenId != "" {
+		whereClause += fmt.Sprintf(" AND h.sessionid = '%s' AND h.screenid = '%s'", opts.SessionId, opts.ScreenId)
+		hNumStr = ""
+	} else if opts.SessionId != "" {
+		whereClause += fmt.Sprintf(" AND h.sessionid = '%s'", opts.SessionId)
+		hNumStr = "s"
+	} else {
+		hNumStr = "g"
+	}
+	if opts.SearchText != "" {
+		whereClause += " AND h.cmdstr LIKE ? ESCAPE '\\'"
+		likeArg := opts.SearchText
+		likeArg = strings.ReplaceAll(likeArg, "%", "\\%")
+		likeArg = strings.ReplaceAll(likeArg, "_", "\\_")
+		queryArgs = append(queryArgs, "%"+likeArg+"%")
+	}
+	if opts.FromTs > 0 {
+		whereClause += fmt.Sprintf(" AND h.ts <= %d", opts.FromTs)
+	}
+	if opts.RemoteId != "" {
+		whereClause += fmt.Sprintf(" AND h.remoteid = '%s'", opts.RemoteId)
+	}
+	if opts.NoMeta {
+		whereClause += " AND NOT h.ismetacmd"
+	}
+	query := fmt.Sprintf("SELECT %s, ('%s' || CAST((row_number() OVER win) as text)) historynum FROM history h %s WINDOW win AS (ORDER BY h.ts, h.historyid) ORDER BY h.ts DESC, h.historyid DESC LIMIT %d OFFSET %d", HistoryCols, hNumStr, whereClause, itemLimit, realOffset)
+	marr := tx.SelectMaps(query, queryArgs...)
+	rtn := make([]*HistoryItemType, len(marr))
+	for idx, m := range marr {
+		hitem := dbutil.FromMap[*HistoryItemType](m)
+		rtn[idx] = hitem
+	}
+	return rtn, nil
+}
+
+func GetHistoryItems(ctx context.Context, opts HistoryQueryOpts) (*HistoryQueryResult, error) {
+	var rtn *HistoryQueryResult
+	txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		var err error
+		rtn, err = runHistoryQueryWithFilter(tx, opts)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	if txErr != nil {
+		return nil, txErr
+	}
+	return rtn, nil
+}
+
+func GetHistoryItemByLineNum(ctx context.Context, screenId string, lineNum int) (*HistoryItemType, error) {
+	return sstore.WithTxRtn(ctx, func(tx *sstore.TxWrap) (*HistoryItemType, error) {
+		query := `SELECT * FROM history WHERE screenid = ? AND linenum = ?`
+		hitem := dbutil.GetMapGen[*HistoryItemType](tx, query, screenId, lineNum)
+		return hitem, nil
+	})
+}
+
+func GetLastHistoryLineNum(ctx context.Context, screenId string) (int, error) {
+	return sstore.WithTxRtn(ctx, func(tx *sstore.TxWrap) (int, error) {
+		query := `SELECT COALESCE(max(linenum), 0) FROM history WHERE screenid = ?`
+		maxLineNum := tx.GetInt(query, screenId)
+		return maxLineNum, nil
+	})
+}
+
+func getLineIdsFromHistoryItems(historyItems []*HistoryItemType) []string {
+	var rtn []string
+	for _, hitem := range historyItems {
+		if hitem.LineId != "" {
+			rtn = append(rtn, hitem.LineId)
+		}
+	}
+	return rtn
+}
+
+func GetLineCmdsFromHistoryItems(ctx context.Context, historyItems []*HistoryItemType) ([]*line.LineType, []*sstore.CmdType, error) {
+	if len(historyItems) == 0 {
+		return nil, nil, nil
+	}
+	return sstore.WithTxRtn3(ctx, func(tx *sstore.TxWrap) ([]*line.LineType, []*sstore.CmdType, error) {
+		lineIdsJsonArr := dbutil.QuickJsonArr(getLineIdsFromHistoryItems(historyItems))
+		query := `SELECT * FROM line WHERE lineid IN (SELECT value FROM json_each(?))`
+		lineArr := dbutil.SelectMappable[*line.LineType](tx, query, lineIdsJsonArr)
+		query = `SELECT * FROM cmd WHERE lineid IN (SELECT value FROM json_each(?))`
+		cmdArr := dbutil.SelectMapsGen[*sstore.CmdType](tx, query, lineIdsJsonArr)
+		return lineArr, cmdArr, nil
+	})
+}
+
+func PurgeHistoryByIds(ctx context.Context, historyIds []string) error {
+	return sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		query := `DELETE FROM history WHERE historyid IN (SELECT value FROM json_each(?))`
+		tx.Exec(query, dbutil.QuickJsonArr(historyIds))
+		return nil
+	})
+}

wavesrv/pkg/history/updatetypes.go

@@ -0,0 +1,16 @@
+// Copyright 2024, Command Line Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package history
+
+type HistoryInfoType struct {
+	HistoryType string             `json:"historytype"`
+	SessionId   string             `json:"sessionid,omitempty"`
+	ScreenId    string             `json:"screenid,omitempty"`
+	Items       []*HistoryItemType `json:"items"`
+	Show        bool               `json:"show"`
+}
+
+func (HistoryInfoType) GetType() string {
+	return "history"
+}

wavesrv/pkg/pcloud/pcloud.go

@@ -23,6 +23,7 @@ import (
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/rtnstate"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/scbase"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/sstore"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/telemetry"
 )
 
 const PCloudEndpoint = "https://api.waveterm.dev/central"
@@ -156,7 +157,7 @@ func SendTelemetry(ctx context.Context, force bool) error {
 	if !force && clientData.ClientOpts.NoTelemetry {
 		return nil
 	}
-	activity, err := sstore.GetNonUploadedActivity(ctx)
+	activity, err := telemetry.GetNonUploadedActivity(ctx)
 	if err != nil {
 		return fmt.Errorf("cannot get activity: %v", err)
 	}
@@ -164,7 +165,7 @@ func SendTelemetry(ctx context.Context, force bool) error {
 		return nil
 	}
 	log.Printf("[pcloud] sending telemetry data\n")
-	dayStr := sstore.GetCurDayStr()
+	dayStr := telemetry.GetCurDayStr()
 	defaultShellType := shellapi.DetectLocalShellType()
 	input := TelemetryInputType{UserId: clientData.UserId, ClientId: clientData.ClientId, CurDay: dayStr, DefaultShell: defaultShellType, Activity: activity}
 	req, err := makeAnonPostReq(ctx, TelemetryUrl, input)
@@ -175,7 +176,7 @@ func SendTelemetry(ctx context.Context, force bool) error {
 	if err != nil {
 		return err
 	}
-	err = sstore.MarkActivityAsUploaded(ctx, activity)
+	err = telemetry.MarkActivityAsUploaded(ctx, activity)
 	if err != nil {
 		return fmt.Errorf("error marking activity as uploaded: %v", err)
 	}

wavesrv/pkg/pcloud/pclouddata.go

@@ -11,6 +11,7 @@ import (
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/remote"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/rtnstate"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/sstore"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/telemetry"
 )
 
 type NoTelemetryInputType struct {
@@ -19,11 +20,11 @@ type NoTelemetryInputType struct {
 }
 
 type TelemetryInputType struct {
-	UserId       string                 `json:"userid"`
-	ClientId     string                 `json:"clientid"`
-	CurDay       string                 `json:"curday"`
-	DefaultShell string                 `json:"defaultshell"`
-	Activity     []*sstore.ActivityType `json:"activity"`
+	UserId       string                    `json:"userid"`
+	ClientId     string                    `json:"clientid"`
+	CurDay       string                    `json:"curday"`
+	DefaultShell string                    `json:"defaultshell"`
+	Activity     []*telemetry.ActivityType `json:"activity"`
 }
 
 type WebShareUpdateType struct {

wavesrv/pkg/playbook/playbook.go

@@ -0,0 +1,162 @@
+// Copyright 2024, Command Line Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package playbook
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/google/uuid"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/dbutil"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/sstore"
+)
+
+type PlaybookType struct {
+	PlaybookId   string   `json:"playbookid"`
+	PlaybookName string   `json:"playbookname"`
+	Description  string   `json:"description"`
+	EntryIds     []string `json:"entryids"`
+
+	// this is not persisted to DB, just for transport to FE
+	Entries []*PlaybookEntry `json:"entries"`
+}
+
+func (p *PlaybookType) ToMap() map[string]interface{} {
+	rtn := make(map[string]interface{})
+	rtn["playbookid"] = p.PlaybookId
+	rtn["playbookname"] = p.PlaybookName
+	rtn["description"] = p.Description
+	rtn["entryids"] = dbutil.QuickJsonArr(p.EntryIds)
+	return rtn
+}
+
+func (p *PlaybookType) FromMap(m map[string]interface{}) bool {
+	dbutil.QuickSetStr(&p.PlaybookId, m, "playbookid")
+	dbutil.QuickSetStr(&p.PlaybookName, m, "playbookname")
+	dbutil.QuickSetStr(&p.Description, m, "description")
+	dbutil.QuickSetJsonArr(&p.Entries, m, "entries")
+	return true
+}
+
+// reorders p.Entries to match p.EntryIds
+func (p *PlaybookType) OrderEntries() {
+	if len(p.Entries) == 0 {
+		return
+	}
+	m := make(map[string]*PlaybookEntry)
+	for _, entry := range p.Entries {
+		m[entry.EntryId] = entry
+	}
+	newList := make([]*PlaybookEntry, 0, len(p.EntryIds))
+	for _, entryId := range p.EntryIds {
+		entry := m[entryId]
+		if entry != nil {
+			newList = append(newList, entry)
+		}
+	}
+	p.Entries = newList
+}
+
+// removes from p.EntryIds (not from p.Entries)
+func (p *PlaybookType) RemoveEntry(entryIdToRemove string) {
+	if len(p.EntryIds) == 0 {
+		return
+	}
+	newList := make([]string, 0, len(p.EntryIds)-1)
+	for _, entryId := range p.EntryIds {
+		if entryId == entryIdToRemove {
+			continue
+		}
+		newList = append(newList, entryId)
+	}
+	p.EntryIds = newList
+}
+
+type PlaybookEntry struct {
+	PlaybookId  string `json:"playbookid"`
+	EntryId     string `json:"entryid"`
+	Alias       string `json:"alias"`
+	CmdStr      string `json:"cmdstr"`
+	UpdatedTs   int64  `json:"updatedts"`
+	CreatedTs   int64  `json:"createdts"`
+	Description string `json:"description"`
+	Remove      bool   `json:"remove,omitempty"`
+}
+
+func CreatePlaybook(ctx context.Context, name string) (*PlaybookType, error) {
+	return sstore.WithTxRtn(ctx, func(tx *sstore.TxWrap) (*PlaybookType, error) {
+		query := `SELECT playbookid FROM playbook WHERE name = ?`
+		if tx.Exists(query, name) {
+			return nil, fmt.Errorf("playbook %q already exists", name)
+		}
+		rtn := &PlaybookType{}
+		rtn.PlaybookId = uuid.New().String()
+		rtn.PlaybookName = name
+		query = `INSERT INTO playbook ( playbookid, playbookname, description, entryids)
+                               VALUES (:playbookid,:playbookname,:description,:entryids)`
+		tx.Exec(query, rtn.ToMap())
+		return rtn, nil
+	})
+}
+
+func selectPlaybook(tx *sstore.TxWrap, playbookId string) *PlaybookType {
+	query := `SELECT * FROM playbook where playbookid = ?`
+	playbook := dbutil.GetMapGen[*PlaybookType](tx, query, playbookId)
+	return playbook
+}
+
+func AddPlaybookEntry(ctx context.Context, entry *PlaybookEntry) error {
+	if entry.EntryId == "" {
+		return fmt.Errorf("invalid entryid")
+	}
+	return sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		playbook := selectPlaybook(tx, entry.PlaybookId)
+		if playbook == nil {
+			return fmt.Errorf("cannot add entry, playbook does not exist")
+		}
+		query := `SELECT entryid FROM playbook_entry WHERE entryid = ?`
+		if tx.Exists(query, entry.EntryId) {
+			return fmt.Errorf("cannot add entry, entryid already exists")
+		}
+		query = `INSERT INTO playbook_entry ( entryid, playbookid, description, alias, cmdstr, createdts, updatedts)
+                                     VALUES (:entryid,:playbookid,:description,:alias,:cmdstr,:createdts,:updatedts)`
+		tx.Exec(query, entry)
+		playbook.EntryIds = append(playbook.EntryIds, entry.EntryId)
+		query = `UPDATE playbook SET entryids = ? WHERE playbookid = ?`
+		tx.Exec(query, dbutil.QuickJsonArr(playbook.EntryIds), entry.PlaybookId)
+		return nil
+	})
+}
+
+func RemovePlaybookEntry(ctx context.Context, playbookId string, entryId string) error {
+	return sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		playbook := selectPlaybook(tx, playbookId)
+		if playbook == nil {
+			return fmt.Errorf("cannot remove playbook entry, playbook does not exist")
+		}
+		query := `SELECT entryid FROM playbook_entry WHERE entryid = ?`
+		if !tx.Exists(query, entryId) {
+			return fmt.Errorf("cannot remove playbook entry, entry does not exist")
+		}
+		query = `DELETE FROM playbook_entry WHERE entryid = ?`
+		tx.Exec(query, entryId)
+		playbook.RemoveEntry(entryId)
+		query = `UPDATE playbook SET entryids = ? WHERE playbookid = ?`
+		tx.Exec(query, dbutil.QuickJsonArr(playbook.EntryIds), playbookId)
+		return nil
+	})
+}
+
+func GetPlaybookById(ctx context.Context, playbookId string) (*PlaybookType, error) {
+	return sstore.WithTxRtn(ctx, func(tx *sstore.TxWrap) (*PlaybookType, error) {
+		rtn := selectPlaybook(tx, playbookId)
+		if rtn == nil {
+			return nil, nil
+		}
+		query := `SELECT * FROM playbook_entry WHERE playbookid = ?`
+		tx.Select(&rtn.Entries, query, playbookId)
+		rtn.OrderEntries()
+		return rtn, nil
+	})
+}

wavesrv/pkg/remote/remote.go

@@ -39,6 +39,7 @@ import (
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/scbus"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/scpacket"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/sstore"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/telemetry"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/userinput"
 
 	"golang.org/x/crypto/ssh"
@@ -1419,8 +1420,8 @@ func getStateVarsFromInitPk(initPk *packet.InitPacketType) map[string]string {
 	return rtn
 }
 
-func makeReinitErrorUpdate(shellType string) sstore.ActivityUpdate {
-	rtn := sstore.ActivityUpdate{}
+func makeReinitErrorUpdate(shellType string) telemetry.ActivityUpdate {
+	rtn := telemetry.ActivityUpdate{}
 	if shellType == packet.ShellType_bash {
 		rtn.ReinitBashErrors = 1
 	} else if shellType == packet.ShellType_zsh {
@@ -1441,7 +1442,7 @@ func (msh *MShellProc) ReInit(ctx context.Context, ck base.CommandKey, shellType
 	}
 	defer func() {
 		if rtnErr != nil {
-			sstore.UpdateActivityWrap(ctx, makeReinitErrorUpdate(shellType), "reiniterror")
+			telemetry.UpdateActivityWrap(ctx, makeReinitErrorUpdate(shellType), "reiniterror")
 		}
 	}()
 	startTs := time.Now()

wavesrv/pkg/sstore/fileops.go

@@ -11,10 +11,8 @@ import (
 	"io/fs"
 	"log"
 	"os"
-	"path"
 	"time"
 
-	"github.com/google/uuid"
 	"github.com/wavetermdev/waveterm/waveshell/pkg/cirfile"
 	"github.com/wavetermdev/waveterm/waveshell/pkg/shexec"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/scbase"
@@ -126,69 +124,6 @@ func ReadPtyOutFile(ctx context.Context, screenId string, lineId string, offset
 	return f.ReadAtWithMax(ctx, offset, maxSize)
 }
 
-type SessionDiskSizeType struct {
-	NumFiles   int
-	TotalSize  int64
-	ErrorCount int
-	Location   string
-}
-
-func directorySize(dirName string) (SessionDiskSizeType, error) {
-	var rtn SessionDiskSizeType
-	rtn.Location = dirName
-	entries, err := os.ReadDir(dirName)
-	if err != nil {
-		return rtn, err
-	}
-	for _, entry := range entries {
-		if entry.IsDir() {
-			rtn.ErrorCount++
-			continue
-		}
-		finfo, err := entry.Info()
-		if err != nil {
-			rtn.ErrorCount++
-			continue
-		}
-		rtn.NumFiles++
-		rtn.TotalSize += finfo.Size()
-	}
-	return rtn, nil
-}
-
-func SessionDiskSize(sessionId string) (SessionDiskSizeType, error) {
-	sessionDir, err := scbase.EnsureSessionDir(sessionId)
-	if err != nil {
-		return SessionDiskSizeType{}, err
-	}
-	return directorySize(sessionDir)
-}
-
-func FullSessionDiskSize() (map[string]SessionDiskSizeType, error) {
-	sdir := scbase.GetSessionsDir()
-	entries, err := os.ReadDir(sdir)
-	if err != nil {
-		return nil, err
-	}
-	rtn := make(map[string]SessionDiskSizeType)
-	for _, entry := range entries {
-		if !entry.IsDir() {
-			continue
-		}
-		name := entry.Name()
-		_, err = uuid.Parse(name)
-		if err != nil {
-			continue
-		}
-		diskSize, err := directorySize(path.Join(sdir, name))
-		if err != nil {
-			continue
-		}
-		rtn[name] = diskSize
-	}
-	return rtn, nil
-}
-
 func DeletePtyOutFile(ctx context.Context, screenId string, lineId string) error {
 	ptyOutFileName, err := scbase.PtyOutFile(screenId, lineId)
 	if err != nil {

wavesrv/pkg/sstore/updatetypes.go

@@ -11,12 +11,6 @@ import (
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/scbus"
 )
 
-type ActiveSessionIdUpdate string
-
-func (ActiveSessionIdUpdate) GetType() string {
-	return "activesessionid"
-}
-
 type LineUpdate struct {
 	Line LineType `json:"line"`
 	Cmd  CmdType  `json:"cmd,omitempty"`
@@ -91,74 +85,12 @@ func (ClearInfoUpdate) GetType() string {
 	return "clearinfo"
 }
 
-type HistoryInfoType struct {
-	HistoryType string             `json:"historytype"`
-	SessionId   string             `json:"sessionid,omitempty"`
-	ScreenId    string             `json:"screenid,omitempty"`
-	Items       []*HistoryItemType `json:"items"`
-	Show        bool               `json:"show"`
-}
-
-func (HistoryInfoType) GetType() string {
-	return "history"
-}
-
 type InteractiveUpdate bool
 
 func (InteractiveUpdate) GetType() string {
 	return "interactive"
 }
 
-type ConnectUpdate struct {
-	Sessions                 []*SessionType                  `json:"sessions,omitempty"`
-	Screens                  []*ScreenType                   `json:"screens,omitempty"`
-	Remotes                  []*RemoteRuntimeState           `json:"remotes,omitempty"`
-	ScreenStatusIndicators   []*ScreenStatusIndicatorType    `json:"screenstatusindicators,omitempty"`
-	ScreenNumRunningCommands []*ScreenNumRunningCommandsType `json:"screennumrunningcommands,omitempty"`
-	ActiveSessionId          string                          `json:"activesessionid,omitempty"`
-}
-
-func (ConnectUpdate) GetType() string {
-	return "connect"
-}
-
-type MainViewUpdate struct {
-	MainView      string           `json:"mainview"`
-	HistoryView   *HistoryViewData `json:"historyview,omitempty"`
-	BookmarksView *BookmarksUpdate `json:"bookmarksview,omitempty"`
-}
-
-func (MainViewUpdate) GetType() string {
-	return "mainview"
-}
-
-type BookmarksUpdate struct {
-	Bookmarks        []*BookmarkType `json:"bookmarks"`
-	SelectedBookmark string          `json:"selectedbookmark,omitempty"`
-}
-
-func (BookmarksUpdate) GetType() string {
-	return "bookmarks"
-}
-
-func AddBookmarksUpdate(update *scbus.ModelUpdatePacketType, bookmarks []*BookmarkType, selectedBookmark *string) {
-	if selectedBookmark == nil {
-		update.AddUpdate(BookmarksUpdate{Bookmarks: bookmarks})
-	} else {
-		update.AddUpdate(BookmarksUpdate{Bookmarks: bookmarks, SelectedBookmark: *selectedBookmark})
-	}
-}
-
-type HistoryViewData struct {
-	Items         []*HistoryItemType `json:"items"`
-	Offset        int                `json:"offset"`
-	RawOffset     int                `json:"rawoffset"`
-	NextRawOffset int                `json:"nextrawoffset"`
-	HasMore       bool               `json:"hasmore"`
-	Lines         []*LineType        `json:"lines"`
-	Cmds          []*CmdType         `json:"cmds"`
-}
-
 type RemoteEditType struct {
 	RemoteEdit  bool   `json:"remoteedit"`
 	RemoteId    string `json:"remoteid,omitempty"`

wavesrv/pkg/telemetry/telemetry.go

@@ -0,0 +1,153 @@
+// Copyright 2024, Command Line Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package telemetry
+
+import (
+	"context"
+	"database/sql/driver"
+	"log"
+	"time"
+
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/dbutil"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/scbase"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/sstore"
+)
+
+const MaxTzNameLen = 50
+
+type ActivityUpdate struct {
+	FgMinutes        int
+	ActiveMinutes    int
+	OpenMinutes      int
+	NumCommands      int
+	ClickShared      int
+	HistoryView      int
+	BookmarksView    int
+	NumConns         int
+	WebShareLimit    int
+	ReinitBashErrors int
+	ReinitZshErrors  int
+	BuildTime        string
+}
+
+type ActivityType struct {
+	Day           string        `json:"day"`
+	Uploaded      bool          `json:"-"`
+	TData         TelemetryData `json:"tdata"`
+	TzName        string        `json:"tzname"`
+	TzOffset      int           `json:"tzoffset"`
+	ClientVersion string        `json:"clientversion"`
+	ClientArch    string        `json:"clientarch"`
+	BuildTime     string        `json:"buildtime"`
+	DefaultShell  string        `json:"defaultshell"`
+	OSRelease     string        `json:"osrelease"`
+}
+
+type TelemetryData struct {
+	NumCommands      int `json:"numcommands"`
+	ActiveMinutes    int `json:"activeminutes"`
+	FgMinutes        int `json:"fgminutes"`
+	OpenMinutes      int `json:"openminutes"`
+	ClickShared      int `json:"clickshared,omitempty"`
+	HistoryView      int `json:"historyview,omitempty"`
+	BookmarksView    int `json:"bookmarksview,omitempty"`
+	NumConns         int `json:"numconns"`
+	WebShareLimit    int `json:"websharelimit,omitempty"`
+	ReinitBashErrors int `json:"reinitbasherrors,omitempty"`
+	ReinitZshErrors  int `json:"reinitzsherrors,omitempty"`
+}
+
+func (tdata TelemetryData) Value() (driver.Value, error) {
+	return dbutil.QuickValueJson(tdata)
+}
+
+func (tdata *TelemetryData) Scan(val interface{}) error {
+	return dbutil.QuickScanJson(tdata, val)
+}
+
+// Wraps UpdateCurrentActivity, but ignores errors
+func UpdateActivityWrap(ctx context.Context, update ActivityUpdate, debugStr string) {
+	err := UpdateCurrentActivity(ctx, update)
+	if err != nil {
+		// ignore error, just log, since this is not critical
+		log.Printf("error updating current activity (%s): %v\n", debugStr, err)
+	}
+}
+
+func GetCurDayStr() string {
+	now := time.Now()
+	dayStr := now.Format("2006-01-02")
+	return dayStr
+}
+
+func UpdateCurrentActivity(ctx context.Context, update ActivityUpdate) error {
+	now := time.Now()
+	dayStr := GetCurDayStr()
+	txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		var tdata TelemetryData
+		query := `SELECT tdata FROM activity WHERE day = ?`
+		found := tx.Get(&tdata, query, dayStr)
+		if !found {
+			query = `INSERT INTO activity (day, uploaded, tdata, tzname, tzoffset, clientversion, clientarch, buildtime, osrelease)
+                                   VALUES (?,   0,        ?,     ?,      ?,        ?,             ?         , ?        , ?)`
+			tzName, tzOffset := now.Zone()
+			if len(tzName) > MaxTzNameLen {
+				tzName = tzName[0:MaxTzNameLen]
+			}
+			tx.Exec(query, dayStr, tdata, tzName, tzOffset, scbase.WaveVersion, scbase.ClientArch(), scbase.BuildTime, scbase.UnameKernelRelease())
+		}
+		tdata.NumCommands += update.NumCommands
+		tdata.FgMinutes += update.FgMinutes
+		tdata.ActiveMinutes += update.ActiveMinutes
+		tdata.OpenMinutes += update.OpenMinutes
+		tdata.ClickShared += update.ClickShared
+		tdata.HistoryView += update.HistoryView
+		tdata.BookmarksView += update.BookmarksView
+		tdata.ReinitBashErrors += update.ReinitBashErrors
+		tdata.ReinitZshErrors += update.ReinitZshErrors
+		if update.NumConns > 0 {
+			tdata.NumConns = update.NumConns
+		}
+		query = `UPDATE activity
+                 SET tdata = ?,
+                     clientversion = ?,
+                     buildtime = ?
+                 WHERE day = ?`
+		tx.Exec(query, tdata, scbase.WaveVersion, scbase.BuildTime, dayStr)
+		return nil
+	})
+	if txErr != nil {
+		return txErr
+	}
+	return nil
+}
+
+func GetNonUploadedActivity(ctx context.Context) ([]*ActivityType, error) {
+	var rtn []*ActivityType
+	txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		query := `SELECT * FROM activity WHERE uploaded = 0 ORDER BY day DESC LIMIT 30`
+		tx.Select(&rtn, query)
+		return nil
+	})
+	if txErr != nil {
+		return nil, txErr
+	}
+	return rtn, nil
+}
+
+// note, will not mark the current day as uploaded
+func MarkActivityAsUploaded(ctx context.Context, activityArr []*ActivityType) error {
+	dayStr := GetCurDayStr()
+	txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		query := `UPDATE activity SET uploaded = 1 WHERE day = ?`
+		for _, activity := range activityArr {
+			if activity.Day == dayStr {
+				continue
+			}
+			tx.Exec(query, activity.Day)
+		}
+		return nil
+	})
+	return txErr
+}

wavesrv/pkg/workspaces/line/dbops.go

@@ -0,0 +1,118 @@
+package line
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/dbutil"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/sstore"
+)
+
+// special "E" returns last unarchived line, "EA" returns last line (even if archived)
+func FindLineIdByArg(ctx context.Context, screenId string, lineArg string) (string, error) {
+	return sstore.WithTxRtn(ctx, func(tx *sstore.TxWrap) (string, error) {
+		if lineArg == "E" {
+			query := `SELECT lineid FROM line WHERE screenid = ? AND NOT archived ORDER BY linenum DESC LIMIT 1`
+			lineId := tx.GetString(query, screenId)
+			return lineId, nil
+		}
+		if lineArg == "EA" {
+			query := `SELECT lineid FROM line WHERE screenid = ? ORDER BY linenum DESC LIMIT 1`
+			lineId := tx.GetString(query, screenId)
+			return lineId, nil
+		}
+		lineNum, err := strconv.Atoi(lineArg)
+		if err == nil {
+			// valid linenum
+			query := `SELECT lineid FROM line WHERE screenid = ? AND linenum = ?`
+			lineId := tx.GetString(query, screenId, lineNum)
+			return lineId, nil
+		} else if len(lineArg) == 8 {
+			// prefix id string match
+			query := `SELECT lineid FROM line WHERE screenid = ? AND substr(lineid, 1, 8) = ?`
+			lineId := tx.GetString(query, screenId, lineArg)
+			return lineId, nil
+		} else {
+			// id match
+			query := `SELECT lineid FROM line WHERE screenid = ? AND lineid = ?`
+			lineId := tx.GetString(query, screenId, lineArg)
+			return lineId, nil
+		}
+	})
+}
+
+func GetLineCmdByLineId(ctx context.Context, screenId string, lineId string) (*LineType, *sstore.CmdType, error) {
+	return sstore.WithTxRtn3(ctx, func(tx *sstore.TxWrap) (*LineType, *sstore.CmdType, error) {
+		query := `SELECT * FROM line WHERE screenid = ? AND lineid = ?`
+		lineVal := dbutil.GetMappable[*LineType](tx, query, screenId, lineId)
+		if lineVal == nil {
+			return nil, nil, nil
+		}
+		var cmdRtn *sstore.CmdType
+		query = `SELECT * FROM cmd WHERE screenid = ? AND lineid = ?`
+		cmdRtn = dbutil.GetMapGen[*sstore.CmdType](tx, query, screenId, lineId)
+		return lineVal, cmdRtn, nil
+	})
+}
+
+func InsertLine(ctx context.Context, line *LineType, cmd *sstore.CmdType) error {
+	if line == nil {
+		return fmt.Errorf("line cannot be nil")
+	}
+	if line.LineId == "" {
+		return fmt.Errorf("line must have lineid set")
+	}
+	if line.LineNum != 0 {
+		return fmt.Errorf("line should not hage linenum set")
+	}
+	if cmd != nil && cmd.ScreenId == "" {
+		return fmt.Errorf("cmd should have screenid set")
+	}
+	qjs := dbutil.QuickJson(line.LineState)
+	if len(qjs) > MaxLineStateSize {
+		return fmt.Errorf("linestate exceeds maxsize, size[%d] max[%d]", len(qjs), MaxLineStateSize)
+	}
+	return sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		query := `SELECT screenid FROM screen WHERE screenid = ?`
+		if !tx.Exists(query, line.ScreenId) {
+			return fmt.Errorf("screen not found, cannot insert line[%s]", line.ScreenId)
+		}
+		query = `SELECT nextlinenum FROM screen WHERE screenid = ?`
+		nextLineNum := tx.GetInt(query, line.ScreenId)
+		line.LineNum = int64(nextLineNum)
+		query = `INSERT INTO line  ( screenid, userid, lineid, ts, linenum, linenumtemp, linelocal, linetype, linestate, text, renderer, ephemeral, contentheight, star, archived)
+                            VALUES (:screenid,:userid,:lineid,:ts,:linenum,:linenumtemp,:linelocal,:linetype,:linestate,:text,:renderer,:ephemeral,:contentheight,:star,:archived)`
+		tx.NamedExec(query, dbutil.ToDBMap(line, false))
+		query = `UPDATE screen SET nextlinenum = ? WHERE screenid = ?`
+		tx.Exec(query, nextLineNum+1, line.ScreenId)
+		if cmd != nil {
+			cmd.OrigTermOpts = cmd.TermOpts
+			cmdMap := cmd.ToMap()
+			query = `
+INSERT INTO cmd  ( screenid, lineid, remoteownerid, remoteid, remotename, cmdstr, rawcmdstr, festate, statebasehash, statediffhasharr, termopts, origtermopts, status, cmdpid, remotepid, donets, restartts, exitcode, durationms, rtnstate, runout, rtnbasehash, rtndiffhasharr)
+          VALUES (:screenid,:lineid,:remoteownerid,:remoteid,:remotename,:cmdstr,:rawcmdstr,:festate,:statebasehash,:statediffhasharr,:termopts,:origtermopts,:status,:cmdpid,:remotepid,:donets,:restartts,:exitcode,:durationms,:rtnstate,:runout,:rtnbasehash,:rtndiffhasharr)
+`
+			tx.NamedExec(query, cmdMap)
+		}
+		if sstore.IsWebShare(tx, line.ScreenId) {
+			sstore.InsertScreenLineUpdate(tx, line.ScreenId, line.LineId, sstore.UpdateType_LineNew)
+		}
+		return nil
+	})
+}
+
+func GetScreenLinesById(ctx context.Context, screenId string) (*ScreenLinesType, error) {
+	return sstore.WithTxRtn(ctx, func(tx *sstore.TxWrap) (*ScreenLinesType, error) {
+		query := `SELECT screenid FROM screen WHERE screenid = ?`
+		screen := dbutil.GetMappable[*ScreenLinesType](tx, query, screenId)
+		if screen == nil {
+			return nil, nil
+		}
+		query = `SELECT * FROM line WHERE screenid = ? ORDER BY linenum`
+		screen.Lines = dbutil.SelectMappable[*LineType](tx, query, screen.ScreenId)
+		query = `SELECT * FROM cmd WHERE screenid = ?`
+		screen.Cmds = dbutil.SelectMapsGen[*sstore.CmdType](tx, query, screen.ScreenId)
+		return screen, nil
+	})
+}

wavesrv/pkg/workspaces/line/line.go

@@ -0,0 +1,135 @@
+package line
+
+import (
+	"context"
+	"time"
+
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/scbase"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/sstore"
+)
+
+const MaxLineStateSize = 4 * 1024 // 4k for now, can raise if needed
+const LineNoHeight = -1
+
+const (
+	LineTypeCmd    = "cmd"
+	LineTypeText   = "text"
+	LineTypeOpenAI = "openai"
+)
+
+const (
+	LineState_Source   = "prompt:source"
+	LineState_File     = "prompt:file"
+	LineState_FileUrl  = "wave:fileurl"
+	LineState_Min      = "wave:min"
+	LineState_Template = "template"
+	LineState_Mode     = "mode"
+	LineState_Lang     = "lang"
+	LineState_Minimap  = "minimap"
+)
+
+type LineType struct {
+	ScreenId      string         `json:"screenid"`
+	UserId        string         `json:"userid"`
+	LineId        string         `json:"lineid"`
+	Ts            int64          `json:"ts"`
+	LineNum       int64          `json:"linenum"`
+	LineNumTemp   bool           `json:"linenumtemp,omitempty"`
+	LineLocal     bool           `json:"linelocal"`
+	LineType      string         `json:"linetype"`
+	LineState     map[string]any `json:"linestate"`
+	Renderer      string         `json:"renderer,omitempty"`
+	Text          string         `json:"text,omitempty"`
+	Ephemeral     bool           `json:"ephemeral,omitempty"`
+	ContentHeight int64          `json:"contentheight,omitempty"`
+	Star          bool           `json:"star,omitempty"`
+	Archived      bool           `json:"archived,omitempty"`
+	Remove        bool           `json:"remove,omitempty"`
+}
+
+func (LineType) UseDBMap() {}
+
+type ScreenLinesType struct {
+	ScreenId string            `json:"screenid"`
+	Lines    []*LineType       `json:"lines" dbmap:"-"`
+	Cmds     []*sstore.CmdType `json:"cmds" dbmap:"-"`
+}
+
+func (ScreenLinesType) UseDBMap() {}
+
+func (ScreenLinesType) GetType() string {
+	return "screenlines"
+}
+
+func makeNewLineCmd(screenId string, userId string, lineId string, renderer string, lineState map[string]any) *LineType {
+	rtn := &LineType{}
+	rtn.ScreenId = screenId
+	rtn.UserId = userId
+	rtn.LineId = lineId
+	rtn.Ts = time.Now().UnixMilli()
+	rtn.LineLocal = true
+	rtn.LineType = LineTypeCmd
+	rtn.LineId = lineId
+	rtn.ContentHeight = LineNoHeight
+	rtn.Renderer = renderer
+	if lineState == nil {
+		lineState = make(map[string]any)
+	}
+	rtn.LineState = lineState
+	return rtn
+}
+
+func makeNewLineText(screenId string, userId string, text string) *LineType {
+	rtn := &LineType{}
+	rtn.ScreenId = screenId
+	rtn.UserId = userId
+	rtn.LineId = scbase.GenWaveUUID()
+	rtn.Ts = time.Now().UnixMilli()
+	rtn.LineLocal = true
+	rtn.LineType = LineTypeText
+	rtn.Text = text
+	rtn.ContentHeight = LineNoHeight
+	rtn.LineState = make(map[string]any)
+	return rtn
+}
+
+func makeNewLineOpenAI(screenId string, userId string, lineId string) *LineType {
+	rtn := &LineType{}
+	rtn.ScreenId = screenId
+	rtn.UserId = userId
+	rtn.LineId = lineId
+	rtn.Ts = time.Now().UnixMilli()
+	rtn.LineLocal = true
+	rtn.LineType = LineTypeOpenAI
+	rtn.ContentHeight = LineNoHeight
+	rtn.Renderer = sstore.CmdRendererOpenAI
+	rtn.LineState = make(map[string]any)
+	return rtn
+}
+
+func AddCommentLine(ctx context.Context, screenId string, userId string, commentText string) (*LineType, error) {
+	rtnLine := makeNewLineText(screenId, userId, commentText)
+	err := InsertLine(ctx, rtnLine, nil)
+	if err != nil {
+		return nil, err
+	}
+	return rtnLine, nil
+}
+
+func AddOpenAILine(ctx context.Context, screenId string, userId string, cmd *sstore.CmdType) (*LineType, error) {
+	rtnLine := makeNewLineOpenAI(screenId, userId, cmd.LineId)
+	err := InsertLine(ctx, rtnLine, cmd)
+	if err != nil {
+		return nil, err
+	}
+	return rtnLine, nil
+}
+
+func AddCmdLine(ctx context.Context, screenId string, userId string, cmd *sstore.CmdType, renderer string, lineState map[string]any) (*LineType, error) {
+	rtnLine := makeNewLineCmd(screenId, userId, cmd.LineId, renderer, lineState)
+	err := InsertLine(ctx, rtnLine, cmd)
+	if err != nil {
+		return nil, err
+	}
+	return rtnLine, nil
+}

wavesrv/pkg/workspaces/screen/dbops.go

@@ -0,0 +1,132 @@
+package screen
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/dbutil"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/sstore"
+)
+
+func GetScreenById(ctx context.Context, screenId string) (*ScreenType, error) {
+	return sstore.WithTxRtn(ctx, func(tx *sstore.TxWrap) (*ScreenType, error) {
+		query := `SELECT * FROM screen WHERE screenid = ?`
+		screen := dbutil.GetMapGen[*ScreenType](tx, query, screenId)
+		return screen, nil
+	})
+}
+
+// includes archived screens
+func GetSessionScreens(ctx context.Context, sessionId string) ([]*ScreenType, error) {
+	return sstore.WithTxRtn(ctx, func(tx *sstore.TxWrap) ([]*ScreenType, error) {
+		query := `SELECT * FROM screen WHERE sessionid = ? ORDER BY archived, screenidx, archivedts`
+		rtn := dbutil.SelectMapsGen[*ScreenType](tx, query, sessionId)
+		return rtn, nil
+	})
+}
+
+// screen may not exist at this point (so don't query screen table)
+func cleanScreenCmds(ctx context.Context, screenId string) error {
+	var removedCmds []string
+	txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		query := `SELECT lineid FROM cmd WHERE screenid = ? AND lineid NOT IN (SELECT lineid FROM line WHERE screenid = ?)`
+		removedCmds = tx.SelectStrings(query, screenId, screenId)
+		query = `DELETE FROM cmd WHERE screenid = ? AND lineid NOT IN (SELECT lineid FROM line WHERE screenid = ?)`
+		tx.Exec(query, screenId, screenId)
+		return nil
+	})
+	if txErr != nil {
+		return txErr
+	}
+	for _, lineId := range removedCmds {
+		sstore.DeletePtyOutFile(ctx, screenId, lineId)
+	}
+	return nil
+}
+
+func UnArchiveScreen(ctx context.Context, sessionId string, screenId string) error {
+	txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		query := `SELECT screenid FROM screen WHERE sessionid = ? AND screenid = ? AND archived`
+		if !tx.Exists(query, sessionId, screenId) {
+			return fmt.Errorf("cannot re-open screen (not found or not archived)")
+		}
+		maxScreenIdx := tx.GetInt(`SELECT COALESCE(max(screenidx), 0) FROM screen WHERE sessionid = ? AND NOT archived`, sessionId)
+		query = `UPDATE screen SET archived = 0, screenidx = ? WHERE sessionid = ? AND screenid = ?`
+		tx.Exec(query, maxScreenIdx+1, sessionId, screenId)
+		return nil
+	})
+	return txErr
+}
+
+func insertScreenUpdate(tx *sstore.TxWrap, screenId string, updateType string) {
+	if screenId == "" {
+		tx.SetErr(errors.New("invalid screen-update, screenid is empty"))
+		return
+	}
+	nowTs := time.Now().UnixMilli()
+	query := `INSERT INTO screenupdate (screenid, lineid, updatetype, updatets) VALUES (?, ?, ?, ?)`
+	tx.Exec(query, screenId, "", updateType, nowTs)
+	sstore.NotifyUpdateWriter()
+}
+
+func insertScreenNewUpdate(tx *sstore.TxWrap, screenId string) {
+	nowTs := time.Now().UnixMilli()
+	query := `INSERT INTO screenupdate (screenid, lineid, updatetype, updatets)
+              SELECT screenid, lineid, ?, ? FROM line WHERE screenid = ? AND NOT archived ORDER BY linenum DESC`
+	tx.Exec(query, sstore.UpdateType_LineNew, nowTs, screenId)
+	query = `INSERT INTO screenupdate (screenid, lineid, updatetype, updatets)
+             SELECT c.screenid, c.lineid, ?, ? FROM cmd c, line l WHERE c.screenid = ? AND l.lineid = c.lineid AND NOT l.archived ORDER BY l.linenum DESC`
+	tx.Exec(query, sstore.UpdateType_PtyPos, nowTs, screenId)
+	sstore.NotifyUpdateWriter()
+}
+
+func handleScreenDelUpdate(tx *sstore.TxWrap, screenId string) {
+	query := `DELETE FROM screenupdate WHERE screenid = ?`
+	tx.Exec(query, screenId)
+	query = `DELETE FROM webptypos WHERE screenid = ?`
+	tx.Exec(query, screenId)
+	// don't insert UpdateType_ScreenDel (we already processed it in cmdrunner)
+}
+
+func InsertScreenDelUpdate(tx *sstore.TxWrap, screenId string) {
+	handleScreenDelUpdate(tx, screenId)
+	insertScreenUpdate(tx, screenId, sstore.UpdateType_ScreenDel)
+	// don't insert UpdateType_ScreenDel (we already processed it in cmdrunner)
+}
+
+func GetScreenUpdates(ctx context.Context, maxNum int) ([]*ScreenUpdateType, error) {
+	return sstore.WithTxRtn(ctx, func(tx *sstore.TxWrap) ([]*ScreenUpdateType, error) {
+		var updates []*ScreenUpdateType
+		query := `SELECT * FROM screenupdate ORDER BY updateid LIMIT ?`
+		tx.Select(&updates, query, maxNum)
+		return updates, nil
+	})
+}
+
+func RemoveScreenUpdate(ctx context.Context, updateId int64) error {
+	if updateId < 0 {
+		return nil // in-memory updates (not from DB)
+	}
+	return sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		query := `DELETE FROM screenupdate WHERE updateid = ?`
+		tx.Exec(query, updateId)
+		return nil
+	})
+}
+
+func CountScreenUpdates(ctx context.Context) (int, error) {
+	return sstore.WithTxRtn(ctx, func(tx *sstore.TxWrap) (int, error) {
+		query := `SELECT count(*) FROM screenupdate`
+		return tx.GetInt(query), nil
+	})
+}
+
+func RemoveScreenUpdates(ctx context.Context, updateIds []int64) error {
+	return sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		query := `DELETE FROM screenupdate WHERE updateid IN (SELECT value FROM json_each(?))`
+		tx.Exec(query, dbutil.QuickJsonArr(updateIds))
+		return nil
+	})
+}

wavesrv/pkg/workspaces/screen/screen.go

@@ -0,0 +1,150 @@
+package screen
+
+import (
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/dbutil"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/scbus"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/sstore"
+)
+
+const (
+	ScreenFocusInput = "input"
+	ScreenFocusCmd   = "cmd"
+)
+
+type ScreenOptsType struct {
+	TabColor string `json:"tabcolor,omitempty"`
+	TabIcon  string `json:"tabicon,omitempty"`
+	PTerm    string `json:"pterm,omitempty"`
+}
+
+type ScreenWebShareOpts struct {
+	ShareName string `json:"sharename"`
+	ViewKey   string `json:"viewkey"`
+}
+
+type ScreenCreateOpts struct {
+	BaseScreenId string
+	CopyRemote   bool
+	CopyCwd      bool
+	CopyEnv      bool
+}
+
+func (sco ScreenCreateOpts) HasCopy() bool {
+	return sco.CopyRemote || sco.CopyCwd || sco.CopyEnv
+}
+
+type ScreenSidebarOptsType struct {
+	Open  bool   `json:"open,omitempty"`
+	Width string `json:"width,omitempty"`
+
+	// this used to be more complicated (sections with types).  simplified for this release
+	SidebarLineId string `json:"sidebarlineid,omitempty"`
+}
+
+type ScreenViewOptsType struct {
+	Sidebar *ScreenSidebarOptsType `json:"sidebar,omitempty"`
+}
+
+type ScreenAnchorType struct {
+	AnchorLine   int `json:"anchorline,omitempty"`
+	AnchorOffset int `json:"anchoroffset,omitempty"`
+}
+
+type ScreenType struct {
+	SessionId      string               `json:"sessionid"`
+	ScreenId       string               `json:"screenid"`
+	Name           string               `json:"name"`
+	ScreenIdx      int64                `json:"screenidx"`
+	ScreenOpts     ScreenOptsType       `json:"screenopts"`
+	ScreenViewOpts ScreenViewOptsType   `json:"screenviewopts"`
+	OwnerId        string               `json:"ownerid"`
+	ShareMode      string               `json:"sharemode"`
+	WebShareOpts   *ScreenWebShareOpts  `json:"webshareopts,omitempty"`
+	CurRemote      sstore.RemotePtrType `json:"curremote"`
+	NextLineNum    int64                `json:"nextlinenum"`
+	SelectedLine   int64                `json:"selectedline"`
+	Anchor         ScreenAnchorType     `json:"anchor"`
+	FocusType      string               `json:"focustype"`
+	Archived       bool                 `json:"archived,omitempty"`
+	ArchivedTs     int64                `json:"archivedts,omitempty"`
+
+	// only for updates
+	Remove bool `json:"remove,omitempty"`
+}
+
+func (s *ScreenType) ToMap() map[string]interface{} {
+	rtn := make(map[string]interface{})
+	rtn["sessionid"] = s.SessionId
+	rtn["screenid"] = s.ScreenId
+	rtn["name"] = s.Name
+	rtn["screenidx"] = s.ScreenIdx
+	rtn["screenopts"] = dbutil.QuickJson(s.ScreenOpts)
+	rtn["screenviewopts"] = dbutil.QuickJson(s.ScreenViewOpts)
+	rtn["ownerid"] = s.OwnerId
+	rtn["sharemode"] = s.ShareMode
+	rtn["webshareopts"] = dbutil.QuickNullableJson(s.WebShareOpts)
+	rtn["curremoteownerid"] = s.CurRemote.OwnerId
+	rtn["curremoteid"] = s.CurRemote.RemoteId
+	rtn["curremotename"] = s.CurRemote.Name
+	rtn["nextlinenum"] = s.NextLineNum
+	rtn["selectedline"] = s.SelectedLine
+	rtn["anchor"] = dbutil.QuickJson(s.Anchor)
+	rtn["focustype"] = s.FocusType
+	rtn["archived"] = s.Archived
+	rtn["archivedts"] = s.ArchivedTs
+	return rtn
+}
+
+func (s *ScreenType) FromMap(m map[string]interface{}) bool {
+	dbutil.QuickSetStr(&s.SessionId, m, "sessionid")
+	dbutil.QuickSetStr(&s.ScreenId, m, "screenid")
+	dbutil.QuickSetStr(&s.Name, m, "name")
+	dbutil.QuickSetInt64(&s.ScreenIdx, m, "screenidx")
+	dbutil.QuickSetJson(&s.ScreenOpts, m, "screenopts")
+	dbutil.QuickSetJson(&s.ScreenViewOpts, m, "screenviewopts")
+	dbutil.QuickSetStr(&s.OwnerId, m, "ownerid")
+	dbutil.QuickSetStr(&s.ShareMode, m, "sharemode")
+	dbutil.QuickSetNullableJson(&s.WebShareOpts, m, "webshareopts")
+	dbutil.QuickSetStr(&s.CurRemote.OwnerId, m, "curremoteownerid")
+	dbutil.QuickSetStr(&s.CurRemote.RemoteId, m, "curremoteid")
+	dbutil.QuickSetStr(&s.CurRemote.Name, m, "curremotename")
+	dbutil.QuickSetInt64(&s.NextLineNum, m, "nextlinenum")
+	dbutil.QuickSetInt64(&s.SelectedLine, m, "selectedline")
+	dbutil.QuickSetJson(&s.Anchor, m, "anchor")
+	dbutil.QuickSetStr(&s.FocusType, m, "focustype")
+	dbutil.QuickSetBool(&s.Archived, m, "archived")
+	dbutil.QuickSetInt64(&s.ArchivedTs, m, "archivedts")
+	return true
+}
+
+func (ScreenType) GetType() string {
+	return "screen"
+}
+
+func AddScreenUpdate(update *scbus.ModelUpdatePacketType, newScreen *ScreenType) {
+	if newScreen == nil {
+		return
+	}
+	screenUpdates := scbus.GetUpdateItems[ScreenType](update)
+	for _, screenUpdate := range screenUpdates {
+		if screenUpdate.ScreenId == newScreen.ScreenId {
+			screenUpdate = newScreen
+			return
+		}
+	}
+	update.AddUpdate(newScreen)
+}
+
+type ScreenTombstoneType struct {
+	ScreenId   string         `json:"screenid"`
+	SessionId  string         `json:"sessionid"`
+	Name       string         `json:"name"`
+	DeletedTs  int64          `json:"deletedts"`
+	ScreenOpts ScreenOptsType `json:"screenopts"`
+}
+
+func (ScreenTombstoneType) UseDBMap() {}
+
+func (ScreenTombstoneType) GetType() string {
+	return "screentombstone"
+}