fix: integrate skeema knownhosts fix

This fix makes it possible to differentiate between keys when multiple are provided by the remote server. It does not solve the case of multiple keys of the same type being shared, but it handles multiple keys of different types being shared, which is much more common.
User input keybindings fix (#494 )
2026-04-22 15:26:58 -07:00 · 2024-08-28 07:14:35 -07:00 · 2024-03-26 15:05:18 -07:00 · 2024-03-25 20:20:52 -07:00 · 2024-03-25 09:55:15 -07:00
21 changed files with 1057 additions and 964 deletions
@@ -134,14 +134,28 @@ github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaD
 gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b/go.mod h1:T3BPAOm2cqquPa0MKWeNkmOM5RQsRhkrwMWonFMN7fE=
 go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
 golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
+golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A=
 golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
+golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
+golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
@@ -2,7 +2,6 @@ import * as React from "react";
 import { GlobalModel } from "@/models";
 import { Choose, When, If } from "tsx-control-statements/components";
 import { Modal, PasswordField, TextField, Markdown, Checkbox } from "@/elements";
-import { checkKeyPressed, adaptFromReactOrNativeKeyEvent, KeybindManager } from "@/util/keyutil";

 import "./userinput.less";

@@ -44,22 +43,6 @@ export const UserInputModal = (userInputRequest: UserInputRequest) => {
        [userInputRequest]
    );

-    function handleTextFocus() {
-        let keybindManager = GlobalModel.keybindManager;
-        keybindManager.registerKeybinding("modal", "userinput", "generic:confirm", (waveEvent) => {
-            handleSendText();
-            return true;
-        });
-        keybindManager.registerKeybinding("modal", "userinput", "generic:cancel", (waveEvent) => {
-            handleSendCancel();
-            return true;
-        });
-    }
-
-    function handleTextBlur() {
-        GlobalModel.keybindManager.unregisterDomain("userinput");
-    }
-
    React.useEffect(() => {
        let timeout: ReturnType<typeof setTimeout>;
        if (countdown == 0) {
@@ -92,8 +75,6 @@ export const UserInputModal = (userInputRequest: UserInputRequest) => {
                                value={responseText}
                                maxLength={400}
                                autoFocus={true}
-                                onFocus={() => handleTextFocus()}
-                                onBlur={() => handleTextBlur()}
                            />
                        </If>
                        <If condition={!userInputRequest.publictext}>
@@ -102,8 +83,6 @@ export const UserInputModal = (userInputRequest: UserInputRequest) => {
                                value={responseText}
                                maxLength={400}
                                autoFocus={true}
-                                onFocus={() => handleTextFocus()}
-                                onBlur={() => handleTextBlur()}
                            />
                        </If>
                    </If>
@@ -45,6 +45,7 @@ import (
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/scpacket"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/scws"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/sstore"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/telemetry"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/wsshell"
 )

@@ -211,7 +212,7 @@ func HandleLogActiveState(w http.ResponseWriter, r *http.Request) {
 		WriteJsonError(w, fmt.Errorf(ErrorDecodingJson, err))
 		return
 	}
-	activity := sstore.ActivityUpdate{}
+	activity := telemetry.ActivityUpdate{}
 	if activeState.Fg {
 		activity.FgMinutes = 1
 	}
@@ -222,7 +223,7 @@ func HandleLogActiveState(w http.ResponseWriter, r *http.Request) {
 		activity.OpenMinutes = 1
 	}
 	activity.NumConns = remote.NumRemotes()
-	err = sstore.UpdateCurrentActivity(r.Context(), activity)
+	err = telemetry.UpdateCurrentActivity(r.Context(), activity)
 	if err != nil {
 		WriteJsonError(w, fmt.Errorf("error updating activity: %w", err))
 		return
@@ -998,7 +999,7 @@ func main() {
 	}

 	log.Printf("PCLOUD_ENDPOINT=%s\n", pcloud.GetEndpoint())
-	sstore.UpdateActivityWrap(context.Background(), sstore.ActivityUpdate{NumConns: remote.NumRemotes()}, "numconns") // set at least one record into activity
+	telemetry.UpdateActivityWrap(context.Background(), telemetry.ActivityUpdate{NumConns: remote.NumRemotes()}, "numconns") // set at least one record into activity
 	installSignalHandlers()
 	go telemetryLoop()
 	go stdinReadWatch()
@@ -19,9 +19,9 @@ require (
 	github.com/sashabaranov/go-openai v1.9.0
 	github.com/sawka/txwrap v0.1.2
 	github.com/wavetermdev/waveterm/waveshell v0.0.0
-	golang.org/x/crypto v0.17.0
+	golang.org/x/crypto v0.24.0
 	golang.org/x/mod v0.10.0
-	golang.org/x/sys v0.15.0
+	golang.org/x/sys v0.21.0
 	mvdan.cc/sh/v3 v3.7.0
 )

@@ -29,6 +29,7 @@ require (
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/skeema/knownhosts v1.3.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 )

@@ -51,6 +51,8 @@ github.com/sashabaranov/go-openai v1.9.0 h1:NoiO++IISxxJ1pRc0n7uZvMGMake0G+FJ1XP
 github.com/sashabaranov/go-openai v1.9.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
 github.com/sawka/txwrap v0.1.2 h1:v8xS0Z1LE7/6vMZA81PYihI+0TSR6Zm1MalzzBIuXKc=
 github.com/sawka/txwrap v0.1.2/go.mod h1:T3nlw2gVpuolo6/XEetvBbk1oMXnY978YmBFy1UyHvw=
+github.com/skeema/knownhosts v1.3.0 h1:AM+y0rI04VksttfwjkSTNQorvGqmwATnvnAHpSgc0LY=
+github.com/skeema/knownhosts v1.3.0/go.mod h1:sPINvnADmT/qYH1kfv+ePMmOBTH6Tbl7b5LvTDjFK7M=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
@@ -61,12 +63,17 @@ go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
 golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
+golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
 golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
 golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
 golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
+golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
 golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
+golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
@@ -0,0 +1,204 @@
+// Copyright 2024, Command Line Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package bookmarks
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/dbutil"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/sstore"
+)
+
+type BookmarkType struct {
+	BookmarkId  string   `json:"bookmarkid"`
+	CreatedTs   int64    `json:"createdts"`
+	CmdStr      string   `json:"cmdstr"`
+	Alias       string   `json:"alias,omitempty"`
+	Tags        []string `json:"tags"`
+	Description string   `json:"description"`
+	OrderIdx    int64    `json:"orderidx"`
+	Remove      bool     `json:"remove,omitempty"`
+}
+
+func (bm *BookmarkType) GetSimpleKey() string {
+	return bm.BookmarkId
+}
+
+func (bm *BookmarkType) ToMap() map[string]interface{} {
+	rtn := make(map[string]interface{})
+	rtn["bookmarkid"] = bm.BookmarkId
+	rtn["createdts"] = bm.CreatedTs
+	rtn["cmdstr"] = bm.CmdStr
+	rtn["alias"] = bm.Alias
+	rtn["description"] = bm.Description
+	rtn["tags"] = dbutil.QuickJsonArr(bm.Tags)
+	return rtn
+}
+
+func (bm *BookmarkType) FromMap(m map[string]interface{}) bool {
+	dbutil.QuickSetStr(&bm.BookmarkId, m, "bookmarkid")
+	dbutil.QuickSetInt64(&bm.CreatedTs, m, "createdts")
+	dbutil.QuickSetStr(&bm.Alias, m, "alias")
+	dbutil.QuickSetStr(&bm.CmdStr, m, "cmdstr")
+	dbutil.QuickSetStr(&bm.Description, m, "description")
+	dbutil.QuickSetJsonArr(&bm.Tags, m, "tags")
+	return true
+}
+
+type bookmarkOrderType struct {
+	BookmarkId string
+	OrderIdx   int64
+}
+
+func GetBookmarks(ctx context.Context, tag string) ([]*BookmarkType, error) {
+	var bms []*BookmarkType
+	txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		var query string
+		if tag == "" {
+			query = `SELECT * FROM bookmark`
+			bms = dbutil.SelectMapsGen[*BookmarkType](tx, query)
+		} else {
+			query = `SELECT * FROM bookmark WHERE EXISTS (SELECT 1 FROM json_each(tags) WHERE value = ?)`
+			bms = dbutil.SelectMapsGen[*BookmarkType](tx, query, tag)
+		}
+		bmMap := dbutil.MakeGenMap(bms)
+		var orders []bookmarkOrderType
+		query = `SELECT bookmarkid, orderidx FROM bookmark_order WHERE tag = ?`
+		tx.Select(&orders, query, tag)
+		for _, bmOrder := range orders {
+			bm := bmMap[bmOrder.BookmarkId]
+			if bm != nil {
+				bm.OrderIdx = bmOrder.OrderIdx
+			}
+		}
+		return nil
+	})
+	if txErr != nil {
+		return nil, txErr
+	}
+	return bms, nil
+}
+
+func GetBookmarkById(ctx context.Context, bookmarkId string, tag string) (*BookmarkType, error) {
+	var rtn *BookmarkType
+	txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		query := `SELECT * FROM bookmark WHERE bookmarkid = ?`
+		rtn = dbutil.GetMapGen[*BookmarkType](tx, query, bookmarkId)
+		if rtn == nil {
+			return nil
+		}
+		query = `SELECT orderidx FROM bookmark_order WHERE bookmarkid = ? AND tag = ?`
+		orderIdx := tx.GetInt(query, bookmarkId, tag)
+		rtn.OrderIdx = int64(orderIdx)
+		return nil
+	})
+	if txErr != nil {
+		return nil, txErr
+	}
+	return rtn, nil
+}
+
+func GetBookmarkIdByArg(ctx context.Context, bookmarkArg string) (string, error) {
+	var rtnId string
+	txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		if len(bookmarkArg) == 8 {
+			query := `SELECT bookmarkid FROM bookmark WHERE bookmarkid LIKE (? || '%')`
+			rtnId = tx.GetString(query, bookmarkArg)
+			return nil
+		}
+		query := `SELECT bookmarkid FROM bookmark WHERE bookmarkid = ?`
+		rtnId = tx.GetString(query, bookmarkArg)
+		return nil
+	})
+	if txErr != nil {
+		return "", txErr
+	}
+	return rtnId, nil
+}
+
+func GetBookmarkIdsByCmdStr(ctx context.Context, cmdStr string) ([]string, error) {
+	return sstore.WithTxRtn(ctx, func(tx *sstore.TxWrap) ([]string, error) {
+		query := `SELECT bookmarkid FROM bookmark WHERE cmdstr = ?`
+		bmIds := tx.SelectStrings(query, cmdStr)
+		return bmIds, nil
+	})
+}
+
+// ignores OrderIdx field
+func InsertBookmark(ctx context.Context, bm *BookmarkType) error {
+	if bm == nil || bm.BookmarkId == "" {
+		return fmt.Errorf("invalid empty bookmark id")
+	}
+	txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		query := `SELECT bookmarkid FROM bookmark WHERE bookmarkid = ?`
+		if tx.Exists(query, bm.BookmarkId) {
+			return fmt.Errorf("bookmarkid already exists")
+		}
+		query = `INSERT INTO bookmark ( bookmarkid, createdts, cmdstr, alias, tags, description)
+                               VALUES (:bookmarkid,:createdts,:cmdstr,:alias,:tags,:description)`
+		tx.NamedExec(query, bm.ToMap())
+		for _, tag := range append(bm.Tags, "") {
+			query = `SELECT COALESCE(max(orderidx), 0) FROM bookmark_order WHERE tag = ?`
+			maxOrder := tx.GetInt(query, tag)
+			query = `INSERT INTO bookmark_order (tag, bookmarkid, orderidx) VALUES (?, ?, ?)`
+			tx.Exec(query, tag, bm.BookmarkId, maxOrder+1)
+		}
+		return nil
+	})
+	return txErr
+}
+
+const (
+	BookmarkField_Desc   = "desc"
+	BookmarkField_CmdStr = "cmdstr"
+)
+
+func EditBookmark(ctx context.Context, bookmarkId string, editMap map[string]interface{}) error {
+	txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		query := `SELECT bookmarkid FROM bookmark WHERE bookmarkid = ?`
+		if !tx.Exists(query, bookmarkId) {
+			return fmt.Errorf("bookmark not found")
+		}
+		if desc, found := editMap[BookmarkField_Desc]; found {
+			query = `UPDATE bookmark SET description = ? WHERE bookmarkid = ?`
+			tx.Exec(query, desc, bookmarkId)
+		}
+		if cmdStr, found := editMap[BookmarkField_CmdStr]; found {
+			query = `UPDATE bookmark SET cmdstr = ? WHERE bookmarkid = ?`
+			tx.Exec(query, cmdStr, bookmarkId)
+		}
+		return nil
+	})
+	return txErr
+}
+
+func fixupBookmarkOrder(tx *sstore.TxWrap) {
+	query := `
+WITH new_order AS (
+  SELECT tag, bookmarkid, row_number() OVER (PARTITION BY tag ORDER BY orderidx) AS newidx FROM bookmark_order
+)
+UPDATE bookmark_order
+SET orderidx = new_order.newidx
+FROM new_order
+WHERE bookmark_order.tag = new_order.tag AND bookmark_order.bookmarkid = new_order.bookmarkid
+`
+	tx.Exec(query)
+}
+
+func DeleteBookmark(ctx context.Context, bookmarkId string) error {
+	txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		query := `SELECT bookmarkid FROM bookmark WHERE bookmarkid = ?`
+		if !tx.Exists(query, bookmarkId) {
+			return fmt.Errorf("bookmark not found")
+		}
+		query = `DELETE FROM bookmark WHERE bookmarkid = ?`
+		tx.Exec(query, bookmarkId)
+		query = `DELETE FROM bookmark_order WHERE bookmarkid = ?`
+		tx.Exec(query, bookmarkId)
+		fixupBookmarkOrder(tx)
+		return nil
+	})
+	return txErr
+}
@@ -0,0 +1,23 @@
+// Copyright 2024, Command Line Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package bookmarks
+
+import "github.com/wavetermdev/waveterm/wavesrv/pkg/scbus"
+
+type BookmarksUpdate struct {
+	Bookmarks        []*BookmarkType `json:"bookmarks"`
+	SelectedBookmark string          `json:"selectedbookmark,omitempty"`
+}
+
+func (BookmarksUpdate) GetType() string {
+	return "bookmarks"
+}
+
+func AddBookmarksUpdate(update *scbus.ModelUpdatePacketType, bookmarks []*BookmarkType, selectedBookmark *string) {
+	if selectedBookmark == nil {
+		update.AddUpdate(BookmarksUpdate{Bookmarks: bookmarks})
+	} else {
+		update.AddUpdate(BookmarksUpdate{Bookmarks: bookmarks, SelectedBookmark: *selectedBookmark})
+	}
+}
@@ -35,8 +35,10 @@ import (
 	"github.com/wavetermdev/waveterm/waveshell/pkg/shellutil"
 	"github.com/wavetermdev/waveterm/waveshell/pkg/shexec"
 	"github.com/wavetermdev/waveterm/waveshell/pkg/utilfn"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/bookmarks"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/comp"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/dbutil"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/history"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/pcloud"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/promptenc"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/releasechecker"
@@ -47,6 +49,7 @@ import (
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/scbus"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/scpacket"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/sstore"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/telemetry"
 	"golang.org/x/mod/semver"
 )

@@ -473,7 +476,7 @@ func doHistoryExpansion(ctx context.Context, ids resolvedIds, hnum int) (string,
 	foundHistoryNum := hnum
 	if hnum == -1 {
 		var err error
-		foundHistoryNum, err = sstore.GetLastHistoryLineNum(ctx, ids.ScreenId)
+		foundHistoryNum, err = history.GetLastHistoryLineNum(ctx, ids.ScreenId)
 		if err != nil {
 			return "", fmt.Errorf("cannot expand history, error finding last history item: %v", err)
 		}
@@ -481,7 +484,7 @@ func doHistoryExpansion(ctx context.Context, ids resolvedIds, hnum int) (string,
 			return "", fmt.Errorf("cannot expand history, no last history item")
 		}
 	}
-	hitem, err := sstore.GetHistoryItemByLineNum(ctx, ids.ScreenId, foundHistoryNum)
+	hitem, err := history.GetHistoryItemByLineNum(ctx, ids.ScreenId, foundHistoryNum)
 	if err != nil {
 		return "", fmt.Errorf("cannot get history item '%d': %v", foundHistoryNum, err)
 	}
@@ -666,7 +669,7 @@ func addToHistory(ctx context.Context, pk *scpacket.FeCommandPacketType, history
 	if err != nil {
 		return err
 	}
-	hitem := &sstore.HistoryItemType{
+	hitem := &history.HistoryItemType{
 		HistoryId: scbase.GenWaveUUID(),
 		Ts:        time.Now().UnixMilli(),
 		UserId:    DefaultUserId,
@@ -690,7 +693,7 @@ func addToHistory(ctx context.Context, pk *scpacket.FeCommandPacketType, history
 	if !isMetaCmd && historyContext.RemotePtr != nil {
 		hitem.Remote = *historyContext.RemotePtr
 	}
-	err = sstore.InsertHistoryItem(ctx, hitem)
+	err = history.InsertHistoryItem(ctx, hitem)
 	if err != nil {
 		return err
 	}
@@ -706,7 +709,7 @@ func EvalCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (scbus.U
 	}
 	evalDepth := getEvalDepth(ctx)
 	if pk.Interactive && evalDepth == 0 {
-		sstore.UpdateActivityWrap(ctx, sstore.ActivityUpdate{NumCommands: 1}, "numcommands")
+		telemetry.UpdateActivityWrap(ctx, telemetry.ActivityUpdate{NumCommands: 1}, "numcommands")
 	}
 	if evalDepth > MaxEvalDepth {
 		return nil, fmt.Errorf("alias/history expansion max-depth exceeded")
@@ -3352,8 +3355,8 @@ func validateRemoteColor(color string, typeStr string) error {
 }

 func SessionOpenSharedCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (scbus.UpdatePacket, error) {
-	activity := sstore.ActivityUpdate{ClickShared: 1}
-	sstore.UpdateActivityWrap(ctx, activity, "click-shared")
+	activity := telemetry.ActivityUpdate{ClickShared: 1}
+	telemetry.UpdateActivityWrap(ctx, activity, "click-shared")
 	return nil, fmt.Errorf("shared sessions are not available in this version of prompt (stay tuned)")
 }

@@ -3615,11 +3618,11 @@ func MainViewCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (scb
 	update := scbus.MakeUpdatePacket()
 	mainViewArg := pk.Args[0]
 	if mainViewArg == sstore.MainViewSession {
-		update.AddUpdate(&sstore.MainViewUpdate{MainView: sstore.MainViewSession})
+		update.AddUpdate(&MainViewUpdate{MainView: sstore.MainViewSession})
 	} else if mainViewArg == sstore.MainViewConnections {
-		update.AddUpdate(&sstore.MainViewUpdate{MainView: sstore.MainViewConnections})
+		update.AddUpdate(&MainViewUpdate{MainView: sstore.MainViewConnections})
 	} else if mainViewArg == sstore.MainViewSettings {
-		update.AddUpdate(&sstore.MainViewUpdate{MainView: sstore.MainViewSettings})
+		update.AddUpdate(&MainViewUpdate{MainView: sstore.MainViewSettings})
 	} else if mainViewArg == sstore.MainViewHistory {
 		return nil, fmt.Errorf("use /history instead")
 	} else if mainViewArg == sstore.MainViewBookmarks {
@@ -3825,7 +3828,7 @@ func HistoryPurgeCommand(ctx context.Context, pk *scpacket.FeCommandPacketType)
 		}
 		historyIds = append(historyIds, historyArg)
 	}
-	err := sstore.PurgeHistoryByIds(ctx, historyIds)
+	err := history.PurgeHistoryByIds(ctx, historyIds)
 	if err != nil {
 		return nil, fmt.Errorf("/history:purge error purging items: %v", err)
 	}
@@ -3837,7 +3840,7 @@ const HistoryViewPageSize = 50
 var cmdFilterLs = regexp.MustCompile(`^ls(\s|$)`)
 var cmdFilterCd = regexp.MustCompile(`^cd(\s|$)`)

-func historyCmdFilter(hitem *sstore.HistoryItemType) bool {
+func historyCmdFilter(hitem *history.HistoryItemType) bool {
 	cmdStr := hitem.CmdStr
 	if cmdStr == "" || strings.Index(cmdStr, ";") != -1 || strings.Index(cmdStr, "\n") != -1 {
 		return true
@@ -3864,7 +3867,7 @@ func HistoryViewAllCommand(ctx context.Context, pk *scpacket.FeCommandPacketType
 	if err != nil {
 		return nil, err
 	}
-	opts := sstore.HistoryQueryOpts{MaxItems: HistoryViewPageSize, Offset: offset, RawOffset: rawOffset}
+	opts := history.HistoryQueryOpts{MaxItems: HistoryViewPageSize, Offset: offset, RawOffset: rawOffset}
 	if pk.Kwargs["text"] != "" {
 		opts.SearchText = pk.Kwargs["text"]
 	}
@@ -3903,25 +3906,25 @@ func HistoryViewAllCommand(ctx context.Context, pk *scpacket.FeCommandPacketType
 	if err != nil {
 		return nil, fmt.Errorf("invalid meta arg (must be boolean): %v", err)
 	}
-	hresult, err := sstore.GetHistoryItems(ctx, opts)
+	hresult, err := history.GetHistoryItems(ctx, opts)
 	if err != nil {
 		return nil, err
 	}
-	hvdata := &sstore.HistoryViewData{
+	hvdata := &history.HistoryViewData{
 		Items:         hresult.Items,
 		Offset:        hresult.Offset,
 		RawOffset:     hresult.RawOffset,
 		NextRawOffset: hresult.NextRawOffset,
 		HasMore:       hresult.HasMore,
 	}
-	lines, cmds, err := sstore.GetLineCmdsFromHistoryItems(ctx, hvdata.Items)
+	lines, cmds, err := history.GetLineCmdsFromHistoryItems(ctx, hvdata.Items)
 	if err != nil {
 		return nil, err
 	}
 	hvdata.Lines = lines
 	hvdata.Cmds = cmds
 	update := scbus.MakeUpdatePacket()
-	update.AddUpdate(&sstore.MainViewUpdate{MainView: sstore.MainViewHistory, HistoryView: hvdata})
+	update.AddUpdate(&MainViewUpdate{MainView: sstore.MainViewHistory, HistoryView: hvdata})
 	return update, nil
 }

@@ -3957,17 +3960,17 @@ func HistoryCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (scbu
 	} else if htype == HistoryTypeSession {
 		hScreenId = ""
 	}
-	hopts := sstore.HistoryQueryOpts{MaxItems: maxItems, SessionId: hSessionId, ScreenId: hScreenId}
-	hresult, err := sstore.GetHistoryItems(ctx, hopts)
+	hopts := history.HistoryQueryOpts{MaxItems: maxItems, SessionId: hSessionId, ScreenId: hScreenId}
+	hresult, err := history.GetHistoryItems(ctx, hopts)
 	if err != nil {
 		return nil, err
 	}
 	show := !resolveBool(pk.Kwargs["noshow"], false)
 	if show {
-		sstore.UpdateActivityWrap(ctx, sstore.ActivityUpdate{HistoryView: 1}, "history")
+		telemetry.UpdateActivityWrap(ctx, telemetry.ActivityUpdate{HistoryView: 1}, "history")
 	}
 	update := scbus.MakeUpdatePacket()
-	update.AddUpdate(sstore.HistoryInfoType{
+	update.AddUpdate(history.HistoryInfoType{
 		HistoryType: htype,
 		SessionId:   ids.SessionId,
 		ScreenId:    ids.ScreenId,
@@ -4310,16 +4313,16 @@ func BookmarksShowCommand(ctx context.Context, pk *scpacket.FeCommandPacketType)
 	if len(pk.Args) > 0 {
 		tagName = pk.Args[0]
 	}
-	bms, err := sstore.GetBookmarks(ctx, tagName)
+	bms, err := bookmarks.GetBookmarks(ctx, tagName)
 	if err != nil {
 		return nil, fmt.Errorf("cannot retrieve bookmarks: %v", err)
 	}
-	sstore.UpdateActivityWrap(ctx, sstore.ActivityUpdate{BookmarksView: 1}, "bookmarks")
+	telemetry.UpdateActivityWrap(ctx, telemetry.ActivityUpdate{BookmarksView: 1}, "bookmarks")
 	update := scbus.MakeUpdatePacket()

-	update.AddUpdate(&sstore.MainViewUpdate{
+	update.AddUpdate(&MainViewUpdate{
 		MainView:      sstore.MainViewBookmarks,
-		BookmarksView: &sstore.BookmarksUpdate{Bookmarks: bms},
+		BookmarksView: &bookmarks.BookmarksUpdate{Bookmarks: bms},
 	})
 	return update, nil
 }
@@ -4329,7 +4332,7 @@ func BookmarkSetCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (
 		return nil, fmt.Errorf("/bookmark:set requires one argument (bookmark id)")
 	}
 	bookmarkArg := pk.Args[0]
-	bookmarkId, err := sstore.GetBookmarkIdByArg(ctx, bookmarkArg)
+	bookmarkId, err := bookmarks.GetBookmarkIdByArg(ctx, bookmarkArg)
 	if err != nil {
 		return nil, fmt.Errorf("error trying to resolve bookmark: %v", err)
 	}
@@ -4338,25 +4341,25 @@ func BookmarkSetCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (
 	}
 	editMap := make(map[string]interface{})
 	if descStr, found := pk.Kwargs["desc"]; found {
-		editMap[sstore.BookmarkField_Desc] = descStr
+		editMap[bookmarks.BookmarkField_Desc] = descStr
 	}
 	if cmdStr, found := pk.Kwargs["cmdstr"]; found {
-		editMap[sstore.BookmarkField_CmdStr] = cmdStr
+		editMap[bookmarks.BookmarkField_CmdStr] = cmdStr
 	}
 	if len(editMap) == 0 {
 		return nil, fmt.Errorf("no fields set, can set %s", formatStrs([]string{"desc", "cmdstr"}, "or", false))
 	}
-	err = sstore.EditBookmark(ctx, bookmarkId, editMap)
+	err = bookmarks.EditBookmark(ctx, bookmarkId, editMap)
 	if err != nil {
 		return nil, fmt.Errorf("error trying to edit bookmark: %v", err)
 	}
-	bm, err := sstore.GetBookmarkById(ctx, bookmarkId, "")
+	bm, err := bookmarks.GetBookmarkById(ctx, bookmarkId, "")
 	if err != nil {
 		return nil, fmt.Errorf("error retrieving edited bookmark: %v", err)
 	}
-	bms := []*sstore.BookmarkType{bm}
+	bms := []*bookmarks.BookmarkType{bm}
 	update := scbus.MakeUpdatePacket()
-	sstore.AddBookmarksUpdate(update, bms, nil)
+	bookmarks.AddBookmarksUpdate(update, bms, nil)
 	update.AddUpdate(sstore.InfoMsgUpdate("bookmark edited"))
 	return update, nil
 }
@@ -4366,20 +4369,20 @@ func BookmarkDeleteCommand(ctx context.Context, pk *scpacket.FeCommandPacketType
 		return nil, fmt.Errorf("/bookmark:delete requires one argument (bookmark id)")
 	}
 	bookmarkArg := pk.Args[0]
-	bookmarkId, err := sstore.GetBookmarkIdByArg(ctx, bookmarkArg)
+	bookmarkId, err := bookmarks.GetBookmarkIdByArg(ctx, bookmarkArg)
 	if err != nil {
 		return nil, fmt.Errorf("error trying to resolve bookmark: %v", err)
 	}
 	if bookmarkId == "" {
 		return nil, fmt.Errorf("bookmark not found")
 	}
-	err = sstore.DeleteBookmark(ctx, bookmarkId)
+	err = bookmarks.DeleteBookmark(ctx, bookmarkId)
 	if err != nil {
 		return nil, fmt.Errorf("error deleting bookmark: %v", err)
 	}
 	update := scbus.MakeUpdatePacket()
-	bms := []*sstore.BookmarkType{{BookmarkId: bookmarkId, Remove: true}}
-	sstore.AddBookmarksUpdate(update, bms, nil)
+	bms := []*bookmarks.BookmarkType{{BookmarkId: bookmarkId, Remove: true}}
+	bookmarks.AddBookmarksUpdate(update, bms, nil)
 	update.AddUpdate(sstore.InfoMsgUpdate("bookmark deleted"))
 	return update, nil
 }
@@ -4407,7 +4410,7 @@ func LineBookmarkCommand(ctx context.Context, pk *scpacket.FeCommandPacketType)
 	if cmdObj == nil {
 		return nil, fmt.Errorf("cannot bookmark non-cmd line")
 	}
-	existingBmIds, err := sstore.GetBookmarkIdsByCmdStr(ctx, cmdObj.CmdStr)
+	existingBmIds, err := bookmarks.GetBookmarkIdsByCmdStr(ctx, cmdObj.CmdStr)
 	if err != nil {
 		return nil, fmt.Errorf("error trying to retrieve current boookmarks: %v", err)
 	}
@@ -4415,7 +4418,7 @@ func LineBookmarkCommand(ctx context.Context, pk *scpacket.FeCommandPacketType)
 	if len(existingBmIds) > 0 {
 		newBmId = existingBmIds[0]
 	} else {
-		newBm := &sstore.BookmarkType{
+		newBm := &bookmarks.BookmarkType{
 			BookmarkId:  uuid.New().String(),
 			CreatedTs:   time.Now().UnixMilli(),
 			CmdStr:      cmdObj.CmdStr,
@@ -4423,17 +4426,17 @@ func LineBookmarkCommand(ctx context.Context, pk *scpacket.FeCommandPacketType)
 			Tags:        nil,
 			Description: "",
 		}
-		err = sstore.InsertBookmark(ctx, newBm)
+		err = bookmarks.InsertBookmark(ctx, newBm)
 		if err != nil {
 			return nil, fmt.Errorf("cannot insert bookmark: %v", err)
 		}
 		newBmId = newBm.BookmarkId
 	}
-	bms, err := sstore.GetBookmarks(ctx, "")
+	bms, err := bookmarks.GetBookmarks(ctx, "")
 	update := scbus.MakeUpdatePacket()
-	update.AddUpdate(&sstore.MainViewUpdate{
+	update.AddUpdate(&MainViewUpdate{
 		MainView:      sstore.MainViewBookmarks,
-		BookmarksView: &sstore.BookmarksUpdate{Bookmarks: bms, SelectedBookmark: newBmId},
+		BookmarksView: &bookmarks.BookmarksUpdate{Bookmarks: bms, SelectedBookmark: newBmId},
 	})
 	return update, nil
 }
@@ -0,0 +1,19 @@
+// Copyright 2024, Command Line Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package cmdrunner
+
+import (
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/bookmarks"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/history"
+)
+
+type MainViewUpdate struct {
+	MainView      string                     `json:"mainview"`
+	HistoryView   *history.HistoryViewData   `json:"historyview,omitempty"`
+	BookmarksView *bookmarks.BookmarksUpdate `json:"bookmarksview,omitempty"`
+}
+
+func (MainViewUpdate) GetType() string {
+	return "mainview"
+}
@@ -0,0 +1,326 @@
+// Copyright 2024, Command Line Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package history
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/google/uuid"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/dbutil"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/sstore"
+)
+
+type HistoryItemType struct {
+	HistoryId  string               `json:"historyid"`
+	Ts         int64                `json:"ts"`
+	UserId     string               `json:"userid"`
+	SessionId  string               `json:"sessionid"`
+	ScreenId   string               `json:"screenid"`
+	LineId     string               `json:"lineid"`
+	HadError   bool                 `json:"haderror"`
+	CmdStr     string               `json:"cmdstr"`
+	Remote     sstore.RemotePtrType `json:"remote"`
+	IsMetaCmd  bool                 `json:"ismetacmd"`
+	ExitCode   *int64               `json:"exitcode,omitempty"`
+	DurationMs *int64               `json:"durationms,omitempty"`
+	FeState    sstore.FeStateType   `json:"festate,omitempty"`
+	Tags       map[string]bool      `json:"tags,omitempty"`
+	LineNum    int64                `json:"linenum" dbmap:"-"`
+	Status     string               `json:"status"`
+
+	// only for updates
+	Remove bool `json:"remove" dbmap:"-"`
+
+	// transient (string because of different history orderings)
+	HistoryNum string `json:"historynum" dbmap:"-"`
+}
+
+func (h *HistoryItemType) ToMap() map[string]interface{} {
+	rtn := make(map[string]interface{})
+	rtn["historyid"] = h.HistoryId
+	rtn["ts"] = h.Ts
+	rtn["userid"] = h.UserId
+	rtn["sessionid"] = h.SessionId
+	rtn["screenid"] = h.ScreenId
+	rtn["lineid"] = h.LineId
+	rtn["linenum"] = h.LineNum
+	rtn["haderror"] = h.HadError
+	rtn["cmdstr"] = h.CmdStr
+	rtn["remoteownerid"] = h.Remote.OwnerId
+	rtn["remoteid"] = h.Remote.RemoteId
+	rtn["remotename"] = h.Remote.Name
+	rtn["ismetacmd"] = h.IsMetaCmd
+	rtn["exitcode"] = h.ExitCode
+	rtn["durationms"] = h.DurationMs
+	rtn["festate"] = dbutil.QuickJson(h.FeState)
+	rtn["tags"] = dbutil.QuickJson(h.Tags)
+	rtn["status"] = h.Status
+	return rtn
+}
+
+func (h *HistoryItemType) FromMap(m map[string]interface{}) bool {
+	dbutil.QuickSetStr(&h.HistoryId, m, "historyid")
+	dbutil.QuickSetInt64(&h.Ts, m, "ts")
+	dbutil.QuickSetStr(&h.UserId, m, "userid")
+	dbutil.QuickSetStr(&h.SessionId, m, "sessionid")
+	dbutil.QuickSetStr(&h.ScreenId, m, "screenid")
+	dbutil.QuickSetStr(&h.LineId, m, "lineid")
+	dbutil.QuickSetBool(&h.HadError, m, "haderror")
+	dbutil.QuickSetStr(&h.CmdStr, m, "cmdstr")
+	dbutil.QuickSetStr(&h.Remote.OwnerId, m, "remoteownerid")
+	dbutil.QuickSetStr(&h.Remote.RemoteId, m, "remoteid")
+	dbutil.QuickSetStr(&h.Remote.Name, m, "remotename")
+	dbutil.QuickSetBool(&h.IsMetaCmd, m, "ismetacmd")
+	dbutil.QuickSetStr(&h.HistoryNum, m, "historynum")
+	dbutil.QuickSetInt64(&h.LineNum, m, "linenum")
+	dbutil.QuickSetNullableInt64(&h.ExitCode, m, "exitcode")
+	dbutil.QuickSetNullableInt64(&h.DurationMs, m, "durationms")
+	dbutil.QuickSetJson(&h.FeState, m, "festate")
+	dbutil.QuickSetJson(&h.Tags, m, "tags")
+	dbutil.QuickSetStr(&h.Status, m, "status")
+	return true
+}
+
+type HistoryQueryOpts struct {
+	Offset     int
+	MaxItems   int
+	FromTs     int64
+	SearchText string
+	SessionId  string
+	RemoteId   string
+	ScreenId   string
+	NoMeta     bool
+	RawOffset  int
+	FilterFn   func(*HistoryItemType) bool
+}
+
+type HistoryQueryResult struct {
+	MaxItems      int
+	Items         []*HistoryItemType
+	Offset        int // the offset shown to user
+	RawOffset     int // internal offset
+	HasMore       bool
+	NextRawOffset int // internal offset used by pager for next query
+
+	prevItems int // holds number of items skipped by RawOffset
+}
+
+type HistoryViewData struct {
+	Items         []*HistoryItemType `json:"items"`
+	Offset        int                `json:"offset"`
+	RawOffset     int                `json:"rawoffset"`
+	NextRawOffset int                `json:"nextrawoffset"`
+	HasMore       bool               `json:"hasmore"`
+	Lines         []*sstore.LineType `json:"lines"`
+	Cmds          []*sstore.CmdType  `json:"cmds"`
+}
+
+const HistoryCols = "h.historyid, h.ts, h.userid, h.sessionid, h.screenid, h.lineid, h.haderror, h.cmdstr, h.remoteownerid, h.remoteid, h.remotename, h.ismetacmd, h.linenum, h.exitcode, h.durationms, h.festate, h.tags, h.status"
+const DefaultMaxHistoryItems = 1000
+
+func InsertHistoryItem(ctx context.Context, hitem *HistoryItemType) error {
+	if hitem == nil {
+		return fmt.Errorf("cannot insert nil history item")
+	}
+	txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		query := `INSERT INTO history 
+                  ( historyid, ts, userid, sessionid, screenid, lineid, haderror, cmdstr, remoteownerid, remoteid, remotename, ismetacmd, linenum, exitcode, durationms, festate, tags, status) VALUES
+                  (:historyid,:ts,:userid,:sessionid,:screenid,:lineid,:haderror,:cmdstr,:remoteownerid,:remoteid,:remotename,:ismetacmd,:linenum,:exitcode,:durationms,:festate,:tags,:status)`
+		tx.NamedExec(query, hitem.ToMap())
+		return nil
+	})
+	return txErr
+}
+
+const HistoryQueryChunkSize = 1000
+
+func _getNextHistoryItem(items []*HistoryItemType, index int, filterFn func(*HistoryItemType) bool) (*HistoryItemType, int) {
+	for ; index < len(items); index++ {
+		item := items[index]
+		if filterFn(item) {
+			return item, index
+		}
+	}
+	return nil, index
+}
+
+// returns true if done, false if we still need to process more items
+func (result *HistoryQueryResult) processItem(item *HistoryItemType, rawOffset int) bool {
+	if result.prevItems < result.Offset {
+		result.prevItems++
+		return false
+	}
+	if len(result.Items) == result.MaxItems {
+		result.HasMore = true
+		result.NextRawOffset = rawOffset
+		return true
+	}
+	if len(result.Items) == 0 {
+		result.RawOffset = rawOffset
+	}
+	result.Items = append(result.Items, item)
+	return false
+}
+
+func runHistoryQueryWithFilter(tx *sstore.TxWrap, opts HistoryQueryOpts) (*HistoryQueryResult, error) {
+	if opts.MaxItems == 0 {
+		return nil, fmt.Errorf("invalid query, maxitems is 0")
+	}
+	rtn := &HistoryQueryResult{Offset: opts.Offset, MaxItems: opts.MaxItems}
+	var rawOffset int
+	if opts.RawOffset >= opts.Offset {
+		rtn.prevItems = opts.Offset
+		rawOffset = opts.RawOffset
+	} else {
+		rawOffset = 0
+	}
+	for {
+		resultItems, err := runHistoryQuery(tx, opts, rawOffset, HistoryQueryChunkSize)
+		if err != nil {
+			return nil, err
+		}
+		isDone := false
+		for resultIdx := 0; resultIdx < len(resultItems); resultIdx++ {
+			if opts.FilterFn != nil && !opts.FilterFn(resultItems[resultIdx]) {
+				continue
+			}
+			isDone = rtn.processItem(resultItems[resultIdx], rawOffset+resultIdx)
+			if isDone {
+				break
+			}
+		}
+		if isDone {
+			break
+		}
+		if len(resultItems) < HistoryQueryChunkSize {
+			break
+		}
+		rawOffset += HistoryQueryChunkSize
+	}
+	return rtn, nil
+}
+
+func runHistoryQuery(tx *sstore.TxWrap, opts HistoryQueryOpts, realOffset int, itemLimit int) ([]*HistoryItemType, error) {
+	// check sessionid/screenid format because we are directly inserting them into the SQL
+	if opts.SessionId != "" {
+		_, err := uuid.Parse(opts.SessionId)
+		if err != nil {
+			return nil, fmt.Errorf("malformed sessionid")
+		}
+	}
+	if opts.ScreenId != "" {
+		_, err := uuid.Parse(opts.ScreenId)
+		if err != nil {
+			return nil, fmt.Errorf("malformed screenid")
+		}
+	}
+	if opts.RemoteId != "" {
+		_, err := uuid.Parse(opts.RemoteId)
+		if err != nil {
+			return nil, fmt.Errorf("malformed remoteid")
+		}
+	}
+	whereClause := "WHERE 1"
+	var queryArgs []interface{}
+	hNumStr := ""
+	if opts.SessionId != "" && opts.ScreenId != "" {
+		whereClause += fmt.Sprintf(" AND h.sessionid = '%s' AND h.screenid = '%s'", opts.SessionId, opts.ScreenId)
+		hNumStr = ""
+	} else if opts.SessionId != "" {
+		whereClause += fmt.Sprintf(" AND h.sessionid = '%s'", opts.SessionId)
+		hNumStr = "s"
+	} else {
+		hNumStr = "g"
+	}
+	if opts.SearchText != "" {
+		whereClause += " AND h.cmdstr LIKE ? ESCAPE '\\'"
+		likeArg := opts.SearchText
+		likeArg = strings.ReplaceAll(likeArg, "%", "\\%")
+		likeArg = strings.ReplaceAll(likeArg, "_", "\\_")
+		queryArgs = append(queryArgs, "%"+likeArg+"%")
+	}
+	if opts.FromTs > 0 {
+		whereClause += fmt.Sprintf(" AND h.ts <= %d", opts.FromTs)
+	}
+	if opts.RemoteId != "" {
+		whereClause += fmt.Sprintf(" AND h.remoteid = '%s'", opts.RemoteId)
+	}
+	if opts.NoMeta {
+		whereClause += " AND NOT h.ismetacmd"
+	}
+	query := fmt.Sprintf("SELECT %s, ('%s' || CAST((row_number() OVER win) as text)) historynum FROM history h %s WINDOW win AS (ORDER BY h.ts, h.historyid) ORDER BY h.ts DESC, h.historyid DESC LIMIT %d OFFSET %d", HistoryCols, hNumStr, whereClause, itemLimit, realOffset)
+	marr := tx.SelectMaps(query, queryArgs...)
+	rtn := make([]*HistoryItemType, len(marr))
+	for idx, m := range marr {
+		hitem := dbutil.FromMap[*HistoryItemType](m)
+		rtn[idx] = hitem
+	}
+	return rtn, nil
+}
+
+func GetHistoryItems(ctx context.Context, opts HistoryQueryOpts) (*HistoryQueryResult, error) {
+	var rtn *HistoryQueryResult
+	txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		var err error
+		rtn, err = runHistoryQueryWithFilter(tx, opts)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	if txErr != nil {
+		return nil, txErr
+	}
+	return rtn, nil
+}
+
+func GetHistoryItemByLineNum(ctx context.Context, screenId string, lineNum int) (*HistoryItemType, error) {
+	return sstore.WithTxRtn(ctx, func(tx *sstore.TxWrap) (*HistoryItemType, error) {
+		query := `SELECT * FROM history WHERE screenid = ? AND linenum = ?`
+		hitem := dbutil.GetMapGen[*HistoryItemType](tx, query, screenId, lineNum)
+		return hitem, nil
+	})
+}
+
+func GetLastHistoryLineNum(ctx context.Context, screenId string) (int, error) {
+	return sstore.WithTxRtn(ctx, func(tx *sstore.TxWrap) (int, error) {
+		query := `SELECT COALESCE(max(linenum), 0) FROM history WHERE screenid = ?`
+		maxLineNum := tx.GetInt(query, screenId)
+		return maxLineNum, nil
+	})
+}
+
+func getLineIdsFromHistoryItems(historyItems []*HistoryItemType) []string {
+	var rtn []string
+	for _, hitem := range historyItems {
+		if hitem.LineId != "" {
+			rtn = append(rtn, hitem.LineId)
+		}
+	}
+	return rtn
+}
+
+func GetLineCmdsFromHistoryItems(ctx context.Context, historyItems []*HistoryItemType) ([]*sstore.LineType, []*sstore.CmdType, error) {
+	if len(historyItems) == 0 {
+		return nil, nil, nil
+	}
+	return sstore.WithTxRtn3(ctx, func(tx *sstore.TxWrap) ([]*sstore.LineType, []*sstore.CmdType, error) {
+		lineIdsJsonArr := dbutil.QuickJsonArr(getLineIdsFromHistoryItems(historyItems))
+		query := `SELECT * FROM line WHERE lineid IN (SELECT value FROM json_each(?))`
+		lineArr := dbutil.SelectMappable[*sstore.LineType](tx, query, lineIdsJsonArr)
+		query = `SELECT * FROM cmd WHERE lineid IN (SELECT value FROM json_each(?))`
+		cmdArr := dbutil.SelectMapsGen[*sstore.CmdType](tx, query, lineIdsJsonArr)
+		return lineArr, cmdArr, nil
+	})
+}
+
+func PurgeHistoryByIds(ctx context.Context, historyIds []string) error {
+	return sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		query := `DELETE FROM history WHERE historyid IN (SELECT value FROM json_each(?))`
+		tx.Exec(query, dbutil.QuickJsonArr(historyIds))
+		return nil
+	})
+}
@@ -0,0 +1,16 @@
+// Copyright 2024, Command Line Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package history
+
+type HistoryInfoType struct {
+	HistoryType string             `json:"historytype"`
+	SessionId   string             `json:"sessionid,omitempty"`
+	ScreenId    string             `json:"screenid,omitempty"`
+	Items       []*HistoryItemType `json:"items"`
+	Show        bool               `json:"show"`
+}
+
+func (HistoryInfoType) GetType() string {
+	return "history"
+}
@@ -23,6 +23,7 @@ import (
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/rtnstate"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/scbase"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/sstore"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/telemetry"
 )

 const PCloudEndpoint = "https://api.waveterm.dev/central"
@@ -156,7 +157,7 @@ func SendTelemetry(ctx context.Context, force bool) error {
 	if !force && clientData.ClientOpts.NoTelemetry {
 		return nil
 	}
-	activity, err := sstore.GetNonUploadedActivity(ctx)
+	activity, err := telemetry.GetNonUploadedActivity(ctx)
 	if err != nil {
 		return fmt.Errorf("cannot get activity: %v", err)
 	}
@@ -164,7 +165,7 @@ func SendTelemetry(ctx context.Context, force bool) error {
 		return nil
 	}
 	log.Printf("[pcloud] sending telemetry data\n")
-	dayStr := sstore.GetCurDayStr()
+	dayStr := telemetry.GetCurDayStr()
 	defaultShellType := shellapi.DetectLocalShellType()
 	input := TelemetryInputType{UserId: clientData.UserId, ClientId: clientData.ClientId, CurDay: dayStr, DefaultShell: defaultShellType, Activity: activity}
 	req, err := makeAnonPostReq(ctx, TelemetryUrl, input)
@@ -175,7 +176,7 @@ func SendTelemetry(ctx context.Context, force bool) error {
 	if err != nil {
 		return err
 	}
-	err = sstore.MarkActivityAsUploaded(ctx, activity)
+	err = telemetry.MarkActivityAsUploaded(ctx, activity)
 	if err != nil {
 		return fmt.Errorf("error marking activity as uploaded: %v", err)
 	}
@@ -11,6 +11,7 @@ import (
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/remote"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/rtnstate"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/sstore"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/telemetry"
 )

 type NoTelemetryInputType struct {
@@ -19,11 +20,11 @@ type NoTelemetryInputType struct {
 }

 type TelemetryInputType struct {
-	UserId       string                 `json:"userid"`
-	ClientId     string                 `json:"clientid"`
-	CurDay       string                 `json:"curday"`
-	DefaultShell string                 `json:"defaultshell"`
-	Activity     []*sstore.ActivityType `json:"activity"`
+	UserId       string                    `json:"userid"`
+	ClientId     string                    `json:"clientid"`
+	CurDay       string                    `json:"curday"`
+	DefaultShell string                    `json:"defaultshell"`
+	Activity     []*telemetry.ActivityType `json:"activity"`
 }

 type WebShareUpdateType struct {
@@ -0,0 +1,162 @@
+// Copyright 2024, Command Line Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package playbook
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/google/uuid"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/dbutil"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/sstore"
+)
+
+type PlaybookType struct {
+	PlaybookId   string   `json:"playbookid"`
+	PlaybookName string   `json:"playbookname"`
+	Description  string   `json:"description"`
+	EntryIds     []string `json:"entryids"`
+
+	// this is not persisted to DB, just for transport to FE
+	Entries []*PlaybookEntry `json:"entries"`
+}
+
+func (p *PlaybookType) ToMap() map[string]interface{} {
+	rtn := make(map[string]interface{})
+	rtn["playbookid"] = p.PlaybookId
+	rtn["playbookname"] = p.PlaybookName
+	rtn["description"] = p.Description
+	rtn["entryids"] = dbutil.QuickJsonArr(p.EntryIds)
+	return rtn
+}
+
+func (p *PlaybookType) FromMap(m map[string]interface{}) bool {
+	dbutil.QuickSetStr(&p.PlaybookId, m, "playbookid")
+	dbutil.QuickSetStr(&p.PlaybookName, m, "playbookname")
+	dbutil.QuickSetStr(&p.Description, m, "description")
+	dbutil.QuickSetJsonArr(&p.Entries, m, "entries")
+	return true
+}
+
+// reorders p.Entries to match p.EntryIds
+func (p *PlaybookType) OrderEntries() {
+	if len(p.Entries) == 0 {
+		return
+	}
+	m := make(map[string]*PlaybookEntry)
+	for _, entry := range p.Entries {
+		m[entry.EntryId] = entry
+	}
+	newList := make([]*PlaybookEntry, 0, len(p.EntryIds))
+	for _, entryId := range p.EntryIds {
+		entry := m[entryId]
+		if entry != nil {
+			newList = append(newList, entry)
+		}
+	}
+	p.Entries = newList
+}
+
+// removes from p.EntryIds (not from p.Entries)
+func (p *PlaybookType) RemoveEntry(entryIdToRemove string) {
+	if len(p.EntryIds) == 0 {
+		return
+	}
+	newList := make([]string, 0, len(p.EntryIds)-1)
+	for _, entryId := range p.EntryIds {
+		if entryId == entryIdToRemove {
+			continue
+		}
+		newList = append(newList, entryId)
+	}
+	p.EntryIds = newList
+}
+
+type PlaybookEntry struct {
+	PlaybookId  string `json:"playbookid"`
+	EntryId     string `json:"entryid"`
+	Alias       string `json:"alias"`
+	CmdStr      string `json:"cmdstr"`
+	UpdatedTs   int64  `json:"updatedts"`
+	CreatedTs   int64  `json:"createdts"`
+	Description string `json:"description"`
+	Remove      bool   `json:"remove,omitempty"`
+}
+
+func CreatePlaybook(ctx context.Context, name string) (*PlaybookType, error) {
+	return sstore.WithTxRtn(ctx, func(tx *sstore.TxWrap) (*PlaybookType, error) {
+		query := `SELECT playbookid FROM playbook WHERE name = ?`
+		if tx.Exists(query, name) {
+			return nil, fmt.Errorf("playbook %q already exists", name)
+		}
+		rtn := &PlaybookType{}
+		rtn.PlaybookId = uuid.New().String()
+		rtn.PlaybookName = name
+		query = `INSERT INTO playbook ( playbookid, playbookname, description, entryids)
+                               VALUES (:playbookid,:playbookname,:description,:entryids)`
+		tx.Exec(query, rtn.ToMap())
+		return rtn, nil
+	})
+}
+
+func selectPlaybook(tx *sstore.TxWrap, playbookId string) *PlaybookType {
+	query := `SELECT * FROM playbook where playbookid = ?`
+	playbook := dbutil.GetMapGen[*PlaybookType](tx, query, playbookId)
+	return playbook
+}
+
+func AddPlaybookEntry(ctx context.Context, entry *PlaybookEntry) error {
+	if entry.EntryId == "" {
+		return fmt.Errorf("invalid entryid")
+	}
+	return sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		playbook := selectPlaybook(tx, entry.PlaybookId)
+		if playbook == nil {
+			return fmt.Errorf("cannot add entry, playbook does not exist")
+		}
+		query := `SELECT entryid FROM playbook_entry WHERE entryid = ?`
+		if tx.Exists(query, entry.EntryId) {
+			return fmt.Errorf("cannot add entry, entryid already exists")
+		}
+		query = `INSERT INTO playbook_entry ( entryid, playbookid, description, alias, cmdstr, createdts, updatedts)
+                                     VALUES (:entryid,:playbookid,:description,:alias,:cmdstr,:createdts,:updatedts)`
+		tx.Exec(query, entry)
+		playbook.EntryIds = append(playbook.EntryIds, entry.EntryId)
+		query = `UPDATE playbook SET entryids = ? WHERE playbookid = ?`
+		tx.Exec(query, dbutil.QuickJsonArr(playbook.EntryIds), entry.PlaybookId)
+		return nil
+	})
+}
+
+func RemovePlaybookEntry(ctx context.Context, playbookId string, entryId string) error {
+	return sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		playbook := selectPlaybook(tx, playbookId)
+		if playbook == nil {
+			return fmt.Errorf("cannot remove playbook entry, playbook does not exist")
+		}
+		query := `SELECT entryid FROM playbook_entry WHERE entryid = ?`
+		if !tx.Exists(query, entryId) {
+			return fmt.Errorf("cannot remove playbook entry, entry does not exist")
+		}
+		query = `DELETE FROM playbook_entry WHERE entryid = ?`
+		tx.Exec(query, entryId)
+		playbook.RemoveEntry(entryId)
+		query = `UPDATE playbook SET entryids = ? WHERE playbookid = ?`
+		tx.Exec(query, dbutil.QuickJsonArr(playbook.EntryIds), playbookId)
+		return nil
+	})
+}
+
+func GetPlaybookById(ctx context.Context, playbookId string) (*PlaybookType, error) {
+	return sstore.WithTxRtn(ctx, func(tx *sstore.TxWrap) (*PlaybookType, error) {
+		rtn := selectPlaybook(tx, playbookId)
+		if rtn == nil {
+			return nil, nil
+		}
+		query := `SELECT * FROM playbook_entry WHERE playbookid = ?`
+		tx.Select(&rtn.Entries, query, playbookId)
+		rtn.OrderEntries()
+		return rtn, nil
+	})
+}
@@ -39,6 +39,7 @@ import (
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/scbus"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/scpacket"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/sstore"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/telemetry"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/userinput"

 	"golang.org/x/crypto/ssh"
@@ -1419,8 +1420,8 @@ func getStateVarsFromInitPk(initPk *packet.InitPacketType) map[string]string {
 	return rtn
 }

-func makeReinitErrorUpdate(shellType string) sstore.ActivityUpdate {
-	rtn := sstore.ActivityUpdate{}
+func makeReinitErrorUpdate(shellType string) telemetry.ActivityUpdate {
+	rtn := telemetry.ActivityUpdate{}
 	if shellType == packet.ShellType_bash {
 		rtn.ReinitBashErrors = 1
 	} else if shellType == packet.ShellType_zsh {
@@ -1441,7 +1442,7 @@ func (msh *MShellProc) ReInit(ctx context.Context, ck base.CommandKey, shellType
 	}
 	defer func() {
 		if rtnErr != nil {
-			sstore.UpdateActivityWrap(ctx, makeReinitErrorUpdate(shellType), "reiniterror")
+			telemetry.UpdateActivityWrap(ctx, makeReinitErrorUpdate(shellType), "reiniterror")
 		}
 	}()
 	startTs := time.Now()
@@ -22,18 +22,21 @@ import (
 	"time"

 	"github.com/kevinburke/ssh_config"
+	"github.com/skeema/knownhosts"
 	"github.com/wavetermdev/waveterm/waveshell/pkg/base"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/scbus"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/sstore"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/userinput"
 	"golang.org/x/crypto/ssh"
-	"golang.org/x/crypto/ssh/knownhosts"
+	xknownhosts "golang.org/x/crypto/ssh/knownhosts"
 )

 type UserInputCancelError struct {
 	Err error
 }

+type HostKeyAlgorithmsFunction = func(hostWithPort string) (algos []string)
+
 func (uice UserInputCancelError) Error() string {
 	return uice.Err.Error()
 }
@@ -356,7 +359,7 @@ func lineContainsMatch(line []byte, matches [][]byte) bool {
 	return false
 }

-func createHostKeyCallback(opts *sstore.SSHOpts) (ssh.HostKeyCallback, error) {
+func createHostKeyCallback(opts *sstore.SSHOpts) (ssh.HostKeyCallback, HostKeyAlgorithmsFunction, error) {
 	rawUserKnownHostsFiles, _ := ssh_config.GetStrict(opts.SSHHost, "UserKnownHostsFile")
 	userKnownHostsFiles := strings.Fields(rawUserKnownHostsFiles) // TODO - smarter splitting escaped spaces and quotes
 	rawGlobalKnownHostsFiles, _ := ssh_config.GetStrict(opts.SSHHost, "GlobalKnownHostsFile")
@@ -364,7 +367,7 @@ func createHostKeyCallback(opts *sstore.SSHOpts) (ssh.HostKeyCallback, error) {

 	osUser, err := user.Current()
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	var unexpandedKnownHostsFiles []string
 	if osUser.Username == "root" {
@@ -380,7 +383,7 @@ func createHostKeyCallback(opts *sstore.SSHOpts) (ssh.HostKeyCallback, error) {

 	// there are no good known hosts files
 	if len(knownHostsFiles) == 0 {
-		return nil, fmt.Errorf("no known_hosts files provided by ssh. defaults are overridden")
+		return nil, nil, fmt.Errorf("no known_hosts files provided by ssh. defaults are overridden")
 	}

 	var unreadableFiles []string
@@ -389,9 +392,10 @@ func createHostKeyCallback(opts *sstore.SSHOpts) (ssh.HostKeyCallback, error) {
 	// incorrectly. if a problem file is found, it is removed from our list
 	// and we try again
 	var basicCallback ssh.HostKeyCallback
+	var keyDb *knownhosts.HostKeyDB
 	for basicCallback == nil && len(knownHostsFiles) > 0 {
 		var err error
-		basicCallback, err = knownhosts.New(knownHostsFiles...)
+		keyDb, err = knownhosts.NewDB(knownHostsFiles...)
 		if serr, ok := err.(*os.PathError); ok {
 			badFile := serr.Path
 			unreadableFiles = append(unreadableFiles, badFile)
@@ -402,35 +406,41 @@ func createHostKeyCallback(opts *sstore.SSHOpts) (ssh.HostKeyCallback, error) {
 				}
 			}
 			if len(okFiles) >= len(knownHostsFiles) {
-				return nil, fmt.Errorf("problem file (%s) doesn't exist. this should not be possible", badFile)
+				return nil, nil, fmt.Errorf("problem file (%s) doesn't exist. this should not be possible", badFile)
 			}
 			knownHostsFiles = okFiles
 		} else if err != nil {
 			// TODO handle obscure problems if possible
-			return nil, fmt.Errorf("known_hosts formatting error: %+v", err)
+			return nil, nil, fmt.Errorf("known_hosts formatting error: %+v", err)
+		} else {
+			basicCallback = keyDb.HostKeyCallback()
 		}
 	}

+	if basicCallback == nil {
+		return nil, nil, fmt.Errorf("no valid knownhosts files exist")
+	}
+
 	waveHostKeyCallback := func(hostname string, remote net.Addr, key ssh.PublicKey) error {
 		err := basicCallback(hostname, remote, key)
 		if err == nil {
 			// success
 			return nil
-		} else if _, ok := err.(*knownhosts.RevokedError); ok {
+		} else if _, ok := err.(*xknownhosts.RevokedError); ok {
 			// revoked credentials are refused outright
 			return err
-		} else if _, ok := err.(*knownhosts.KeyError); !ok {
+		} else if _, ok := err.(*xknownhosts.KeyError); !ok {
 			// this is an unknown error (note the !ok is opposite of usual)
 			return err
 		}
-		serr, _ := err.(*knownhosts.KeyError)
+		serr, _ := err.(*xknownhosts.KeyError)
 		if len(serr.Want) == 0 {
 			// the key was not found

 			// try to write to a file that could be parsed
 			var err error
 			for _, filename := range knownHostsFiles {
-				newLine := knownhosts.Line([]string{knownhosts.Normalize(hostname)}, key)
+				newLine := xknownhosts.Line([]string{xknownhosts.Normalize(hostname)}, key)
 				getUserVerification := createUnknownKeyVerifier(filename, hostname, remote.String(), key)
 				err = writeToKnownHosts(filename, newLine, getUserVerification)
 				if err == nil {
@@ -445,7 +455,7 @@ func createHostKeyCallback(opts *sstore.SSHOpts) (ssh.HostKeyCallback, error) {
 			// should catch cases where there is no known_hosts file
 			if err != nil {
 				for _, filename := range unreadableFiles {
-					newLine := knownhosts.Line([]string{knownhosts.Normalize(hostname)}, key)
+					newLine := xknownhosts.Line([]string{xknownhosts.Normalize(hostname)}, key)
 					getUserVerification := createMissingKnownHostsVerifier(filename, hostname, remote.String(), key)
 					err = writeToKnownHosts(filename, newLine, getUserVerification)
 					if err == nil {
@@ -495,7 +505,7 @@ func createHostKeyCallback(opts *sstore.SSHOpts) (ssh.HostKeyCallback, error) {
 			return fmt.Errorf("remote host identification has changed")
 		}

-		updatedCallback, err := knownhosts.New(knownHostsFiles...)
+		updatedCallback, err := xknownhosts.New(knownHostsFiles...)
 		if err != nil {
 			return err
 		}
@@ -503,7 +513,7 @@ func createHostKeyCallback(opts *sstore.SSHOpts) (ssh.HostKeyCallback, error) {
 		return updatedCallback(hostname, remote, key)
 	}

-	return waveHostKeyCallback, nil
+	return waveHostKeyCallback, keyDb.HostKeyAlgorithms, nil
 }

 func DialContext(ctx context.Context, network string, addr string, config *ssh.ClientConfig) (*ssh.Client, error) {
@@ -569,17 +579,18 @@ func ConnectToClient(connCtx context.Context, opts *sstore.SSHOpts, remoteDispla
 		authMethods = append(authMethods, authMethod)
 	}

-	hostKeyCallback, err := createHostKeyCallback(opts)
+	hostKeyCallback, hostKeyAlgorithms, err := createHostKeyCallback(opts)
 	if err != nil {
 		return nil, err
 	}

-	clientConfig := &ssh.ClientConfig{
-		User:            sshKeywords.User,
-		Auth:            authMethods,
-		HostKeyCallback: hostKeyCallback,
-	}
 	networkAddr := sshKeywords.HostName + ":" + sshKeywords.Port
+	clientConfig := &ssh.ClientConfig{
+		User:              sshKeywords.User,
+		Auth:              authMethods,
+		HostKeyCallback:   hostKeyCallback,
+		HostKeyAlgorithms: hostKeyAlgorithms(networkAddr),
+	}
 	return DialContext(connCtx, "tcp", networkAddr, clientConfig)
 }

@@ -110,6 +110,7 @@ const (
 	SSHConfigSrcTypeImport = "sshconfig-import"
 )

+// TODO: move to webshare package once sstore code is more modular
 const (
 	ShareModeLocal = "local"
 	ShareModeWeb   = "web"
@@ -154,8 +155,6 @@ const (
 	UpdateType_PtyPos             = "pty:pos"
 )

-const MaxTzNameLen = 50
-
 var globalDBLock = &sync.Mutex{}
 var globalDB *sqlx.DB
 var globalDBErr error
@@ -233,56 +232,6 @@ type ClientWinSizeType struct {
 	FullScreen bool `json:"fullscreen,omitempty"`
 }

-type ActivityUpdate struct {
-	FgMinutes        int
-	ActiveMinutes    int
-	OpenMinutes      int
-	NumCommands      int
-	ClickShared      int
-	HistoryView      int
-	BookmarksView    int
-	NumConns         int
-	WebShareLimit    int
-	ReinitBashErrors int
-	ReinitZshErrors  int
-	BuildTime        string
-}
-
-type ActivityType struct {
-	Day           string        `json:"day"`
-	Uploaded      bool          `json:"-"`
-	TData         TelemetryData `json:"tdata"`
-	TzName        string        `json:"tzname"`
-	TzOffset      int           `json:"tzoffset"`
-	ClientVersion string        `json:"clientversion"`
-	ClientArch    string        `json:"clientarch"`
-	BuildTime     string        `json:"buildtime"`
-	DefaultShell  string        `json:"defaultshell"`
-	OSRelease     string        `json:"osrelease"`
-}
-
-type TelemetryData struct {
-	NumCommands      int `json:"numcommands"`
-	ActiveMinutes    int `json:"activeminutes"`
-	FgMinutes        int `json:"fgminutes"`
-	OpenMinutes      int `json:"openminutes"`
-	ClickShared      int `json:"clickshared,omitempty"`
-	HistoryView      int `json:"historyview,omitempty"`
-	BookmarksView    int `json:"bookmarksview,omitempty"`
-	NumConns         int `json:"numconns"`
-	WebShareLimit    int `json:"websharelimit,omitempty"`
-	ReinitBashErrors int `json:"reinitbasherrors,omitempty"`
-	ReinitZshErrors  int `json:"reinitzsherrors,omitempty"`
-}
-
-func (tdata TelemetryData) Value() (driver.Value, error) {
-	return quickValueJson(tdata)
-}
-
-func (tdata *TelemetryData) Scan(val interface{}) error {
-	return quickScanJson(tdata, val)
-}
-
 type SidebarValueType struct {
 	Collapsed bool `json:"collapsed"`
 	Width     int  `json:"width"`
@@ -398,52 +347,6 @@ type SessionStatsType struct {
 	DiskStats          SessionDiskSizeType `json:"diskstats"`
 }

-func (h *HistoryItemType) ToMap() map[string]interface{} {
-	rtn := make(map[string]interface{})
-	rtn["historyid"] = h.HistoryId
-	rtn["ts"] = h.Ts
-	rtn["userid"] = h.UserId
-	rtn["sessionid"] = h.SessionId
-	rtn["screenid"] = h.ScreenId
-	rtn["lineid"] = h.LineId
-	rtn["linenum"] = h.LineNum
-	rtn["haderror"] = h.HadError
-	rtn["cmdstr"] = h.CmdStr
-	rtn["remoteownerid"] = h.Remote.OwnerId
-	rtn["remoteid"] = h.Remote.RemoteId
-	rtn["remotename"] = h.Remote.Name
-	rtn["ismetacmd"] = h.IsMetaCmd
-	rtn["exitcode"] = h.ExitCode
-	rtn["durationms"] = h.DurationMs
-	rtn["festate"] = quickJson(h.FeState)
-	rtn["tags"] = quickJson(h.Tags)
-	rtn["status"] = h.Status
-	return rtn
-}
-
-func (h *HistoryItemType) FromMap(m map[string]interface{}) bool {
-	quickSetStr(&h.HistoryId, m, "historyid")
-	quickSetInt64(&h.Ts, m, "ts")
-	quickSetStr(&h.UserId, m, "userid")
-	quickSetStr(&h.SessionId, m, "sessionid")
-	quickSetStr(&h.ScreenId, m, "screenid")
-	quickSetStr(&h.LineId, m, "lineid")
-	quickSetBool(&h.HadError, m, "haderror")
-	quickSetStr(&h.CmdStr, m, "cmdstr")
-	quickSetStr(&h.Remote.OwnerId, m, "remoteownerid")
-	quickSetStr(&h.Remote.RemoteId, m, "remoteid")
-	quickSetStr(&h.Remote.Name, m, "remotename")
-	quickSetBool(&h.IsMetaCmd, m, "ismetacmd")
-	quickSetStr(&h.HistoryNum, m, "historynum")
-	quickSetInt64(&h.LineNum, m, "linenum")
-	dbutil.QuickSetNullableInt64(&h.ExitCode, m, "exitcode")
-	dbutil.QuickSetNullableInt64(&h.DurationMs, m, "durationms")
-	quickSetJson(&h.FeState, m, "festate")
-	quickSetJson(&h.Tags, m, "tags")
-	quickSetStr(&h.Status, m, "status")
-	return true
-}
-
 type ScreenOptsType struct {
 	TabColor string `json:"tabcolor,omitempty"`
 	TabIcon  string `json:"tabicon,omitempty"`
@@ -619,55 +522,6 @@ type ScreenAnchorType struct {
 	AnchorOffset int `json:"anchoroffset,omitempty"`
 }

-type HistoryItemType struct {
-	HistoryId  string          `json:"historyid"`
-	Ts         int64           `json:"ts"`
-	UserId     string          `json:"userid"`
-	SessionId  string          `json:"sessionid"`
-	ScreenId   string          `json:"screenid"`
-	LineId     string          `json:"lineid"`
-	HadError   bool            `json:"haderror"`
-	CmdStr     string          `json:"cmdstr"`
-	Remote     RemotePtrType   `json:"remote"`
-	IsMetaCmd  bool            `json:"ismetacmd"`
-	ExitCode   *int64          `json:"exitcode,omitempty"`
-	DurationMs *int64          `json:"durationms,omitempty"`
-	FeState    FeStateType     `json:"festate,omitempty"`
-	Tags       map[string]bool `json:"tags,omitempty"`
-	LineNum    int64           `json:"linenum" dbmap:"-"`
-	Status     string          `json:"status"`
-
-	// only for updates
-	Remove bool `json:"remove" dbmap:"-"`
-
-	// transient (string because of different history orderings)
-	HistoryNum string `json:"historynum" dbmap:"-"`
-}
-
-type HistoryQueryOpts struct {
-	Offset     int
-	MaxItems   int
-	FromTs     int64
-	SearchText string
-	SessionId  string
-	RemoteId   string
-	ScreenId   string
-	NoMeta     bool
-	RawOffset  int
-	FilterFn   func(*HistoryItemType) bool
-}
-
-type HistoryQueryResult struct {
-	MaxItems      int
-	Items         []*HistoryItemType
-	Offset        int // the offset shown to user
-	RawOffset     int // internal offset
-	HasMore       bool
-	NextRawOffset int // internal offset used by pager for next query
-
-	prevItems int // holds number of items skipped by RawOffset
-}
-
 type TermOpts struct {
 	Rows       int64 `json:"rows"`
 	Cols       int64 `json:"cols"`
@@ -850,114 +704,6 @@ type OpenAIResponse struct {
 	Choices []OpenAIChoiceType `json:"choices,omitempty"`
 }

-type PlaybookType struct {
-	PlaybookId   string   `json:"playbookid"`
-	PlaybookName string   `json:"playbookname"`
-	Description  string   `json:"description"`
-	EntryIds     []string `json:"entryids"`
-
-	// this is not persisted to DB, just for transport to FE
-	Entries []*PlaybookEntry `json:"entries"`
-}
-
-func (p *PlaybookType) ToMap() map[string]interface{} {
-	rtn := make(map[string]interface{})
-	rtn["playbookid"] = p.PlaybookId
-	rtn["playbookname"] = p.PlaybookName
-	rtn["description"] = p.Description
-	rtn["entryids"] = quickJsonArr(p.EntryIds)
-	return rtn
-}
-
-func (p *PlaybookType) FromMap(m map[string]interface{}) bool {
-	quickSetStr(&p.PlaybookId, m, "playbookid")
-	quickSetStr(&p.PlaybookName, m, "playbookname")
-	quickSetStr(&p.Description, m, "description")
-	quickSetJsonArr(&p.Entries, m, "entries")
-	return true
-}
-
-// reorders p.Entries to match p.EntryIds
-func (p *PlaybookType) OrderEntries() {
-	if len(p.Entries) == 0 {
-		return
-	}
-	m := make(map[string]*PlaybookEntry)
-	for _, entry := range p.Entries {
-		m[entry.EntryId] = entry
-	}
-	newList := make([]*PlaybookEntry, 0, len(p.EntryIds))
-	for _, entryId := range p.EntryIds {
-		entry := m[entryId]
-		if entry != nil {
-			newList = append(newList, entry)
-		}
-	}
-	p.Entries = newList
-}
-
-// removes from p.EntryIds (not from p.Entries)
-func (p *PlaybookType) RemoveEntry(entryIdToRemove string) {
-	if len(p.EntryIds) == 0 {
-		return
-	}
-	newList := make([]string, 0, len(p.EntryIds)-1)
-	for _, entryId := range p.EntryIds {
-		if entryId == entryIdToRemove {
-			continue
-		}
-		newList = append(newList, entryId)
-	}
-	p.EntryIds = newList
-}
-
-type PlaybookEntry struct {
-	PlaybookId  string `json:"playbookid"`
-	EntryId     string `json:"entryid"`
-	Alias       string `json:"alias"`
-	CmdStr      string `json:"cmdstr"`
-	UpdatedTs   int64  `json:"updatedts"`
-	CreatedTs   int64  `json:"createdts"`
-	Description string `json:"description"`
-	Remove      bool   `json:"remove,omitempty"`
-}
-
-type BookmarkType struct {
-	BookmarkId  string   `json:"bookmarkid"`
-	CreatedTs   int64    `json:"createdts"`
-	CmdStr      string   `json:"cmdstr"`
-	Alias       string   `json:"alias,omitempty"`
-	Tags        []string `json:"tags"`
-	Description string   `json:"description"`
-	OrderIdx    int64    `json:"orderidx"`
-	Remove      bool     `json:"remove,omitempty"`
-}
-
-func (bm *BookmarkType) GetSimpleKey() string {
-	return bm.BookmarkId
-}
-
-func (bm *BookmarkType) ToMap() map[string]interface{} {
-	rtn := make(map[string]interface{})
-	rtn["bookmarkid"] = bm.BookmarkId
-	rtn["createdts"] = bm.CreatedTs
-	rtn["cmdstr"] = bm.CmdStr
-	rtn["alias"] = bm.Alias
-	rtn["description"] = bm.Description
-	rtn["tags"] = quickJsonArr(bm.Tags)
-	return rtn
-}
-
-func (bm *BookmarkType) FromMap(m map[string]interface{}) bool {
-	quickSetStr(&bm.BookmarkId, m, "bookmarkid")
-	quickSetInt64(&bm.CreatedTs, m, "createdts")
-	quickSetStr(&bm.Alias, m, "alias")
-	quickSetStr(&bm.CmdStr, m, "cmdstr")
-	quickSetStr(&bm.Description, m, "description")
-	quickSetJsonArr(&bm.Tags, m, "tags")
-	return true
-}
-
 type ResolveItem struct {
 	Name   string
 	Num    int
@@ -91,18 +91,6 @@ func (ClearInfoUpdate) GetType() string {
 	return "clearinfo"
 }

-type HistoryInfoType struct {
-	HistoryType string             `json:"historytype"`
-	SessionId   string             `json:"sessionid,omitempty"`
-	ScreenId    string             `json:"screenid,omitempty"`
-	Items       []*HistoryItemType `json:"items"`
-	Show        bool               `json:"show"`
-}
-
-func (HistoryInfoType) GetType() string {
-	return "history"
-}
-
 type InteractiveUpdate bool

 func (InteractiveUpdate) GetType() string {
@@ -122,43 +110,6 @@ func (ConnectUpdate) GetType() string {
 	return "connect"
 }

-type MainViewUpdate struct {
-	MainView      string           `json:"mainview"`
-	HistoryView   *HistoryViewData `json:"historyview,omitempty"`
-	BookmarksView *BookmarksUpdate `json:"bookmarksview,omitempty"`
-}
-
-func (MainViewUpdate) GetType() string {
-	return "mainview"
-}
-
-type BookmarksUpdate struct {
-	Bookmarks        []*BookmarkType `json:"bookmarks"`
-	SelectedBookmark string          `json:"selectedbookmark,omitempty"`
-}
-
-func (BookmarksUpdate) GetType() string {
-	return "bookmarks"
-}
-
-func AddBookmarksUpdate(update *scbus.ModelUpdatePacketType, bookmarks []*BookmarkType, selectedBookmark *string) {
-	if selectedBookmark == nil {
-		update.AddUpdate(BookmarksUpdate{Bookmarks: bookmarks})
-	} else {
-		update.AddUpdate(BookmarksUpdate{Bookmarks: bookmarks, SelectedBookmark: *selectedBookmark})
-	}
-}
-
-type HistoryViewData struct {
-	Items         []*HistoryItemType `json:"items"`
-	Offset        int                `json:"offset"`
-	RawOffset     int                `json:"rawoffset"`
-	NextRawOffset int                `json:"nextrawoffset"`
-	HasMore       bool               `json:"hasmore"`
-	Lines         []*LineType        `json:"lines"`
-	Cmds          []*CmdType         `json:"cmds"`
-}
-
 type RemoteEditType struct {
 	RemoteEdit  bool   `json:"remoteedit"`
 	RemoteId    string `json:"remoteid,omitempty"`
@@ -0,0 +1,153 @@
+// Copyright 2024, Command Line Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package telemetry
+
+import (
+	"context"
+	"database/sql/driver"
+	"log"
+	"time"
+
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/dbutil"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/scbase"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/sstore"
+)
+
+const MaxTzNameLen = 50
+
+type ActivityUpdate struct {
+	FgMinutes        int
+	ActiveMinutes    int
+	OpenMinutes      int
+	NumCommands      int
+	ClickShared      int
+	HistoryView      int
+	BookmarksView    int
+	NumConns         int
+	WebShareLimit    int
+	ReinitBashErrors int
+	ReinitZshErrors  int
+	BuildTime        string
+}
+
+type ActivityType struct {
+	Day           string        `json:"day"`
+	Uploaded      bool          `json:"-"`
+	TData         TelemetryData `json:"tdata"`
+	TzName        string        `json:"tzname"`
+	TzOffset      int           `json:"tzoffset"`
+	ClientVersion string        `json:"clientversion"`
+	ClientArch    string        `json:"clientarch"`
+	BuildTime     string        `json:"buildtime"`
+	DefaultShell  string        `json:"defaultshell"`
+	OSRelease     string        `json:"osrelease"`
+}
+
+type TelemetryData struct {
+	NumCommands      int `json:"numcommands"`
+	ActiveMinutes    int `json:"activeminutes"`
+	FgMinutes        int `json:"fgminutes"`
+	OpenMinutes      int `json:"openminutes"`
+	ClickShared      int `json:"clickshared,omitempty"`
+	HistoryView      int `json:"historyview,omitempty"`
+	BookmarksView    int `json:"bookmarksview,omitempty"`
+	NumConns         int `json:"numconns"`
+	WebShareLimit    int `json:"websharelimit,omitempty"`
+	ReinitBashErrors int `json:"reinitbasherrors,omitempty"`
+	ReinitZshErrors  int `json:"reinitzsherrors,omitempty"`
+}
+
+func (tdata TelemetryData) Value() (driver.Value, error) {
+	return dbutil.QuickValueJson(tdata)
+}
+
+func (tdata *TelemetryData) Scan(val interface{}) error {
+	return dbutil.QuickScanJson(tdata, val)
+}
+
+// Wraps UpdateCurrentActivity, but ignores errors
+func UpdateActivityWrap(ctx context.Context, update ActivityUpdate, debugStr string) {
+	err := UpdateCurrentActivity(ctx, update)
+	if err != nil {
+		// ignore error, just log, since this is not critical
+		log.Printf("error updating current activity (%s): %v\n", debugStr, err)
+	}
+}
+
+func GetCurDayStr() string {
+	now := time.Now()
+	dayStr := now.Format("2006-01-02")
+	return dayStr
+}
+
+func UpdateCurrentActivity(ctx context.Context, update ActivityUpdate) error {
+	now := time.Now()
+	dayStr := GetCurDayStr()
+	txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		var tdata TelemetryData
+		query := `SELECT tdata FROM activity WHERE day = ?`
+		found := tx.Get(&tdata, query, dayStr)
+		if !found {
+			query = `INSERT INTO activity (day, uploaded, tdata, tzname, tzoffset, clientversion, clientarch, buildtime, osrelease)
+                                   VALUES (?,   0,        ?,     ?,      ?,        ?,             ?         , ?        , ?)`
+			tzName, tzOffset := now.Zone()
+			if len(tzName) > MaxTzNameLen {
+				tzName = tzName[0:MaxTzNameLen]
+			}
+			tx.Exec(query, dayStr, tdata, tzName, tzOffset, scbase.WaveVersion, scbase.ClientArch(), scbase.BuildTime, scbase.UnameKernelRelease())
+		}
+		tdata.NumCommands += update.NumCommands
+		tdata.FgMinutes += update.FgMinutes
+		tdata.ActiveMinutes += update.ActiveMinutes
+		tdata.OpenMinutes += update.OpenMinutes
+		tdata.ClickShared += update.ClickShared
+		tdata.HistoryView += update.HistoryView
+		tdata.BookmarksView += update.BookmarksView
+		tdata.ReinitBashErrors += update.ReinitBashErrors
+		tdata.ReinitZshErrors += update.ReinitZshErrors
+		if update.NumConns > 0 {
+			tdata.NumConns = update.NumConns
+		}
+		query = `UPDATE activity
+                 SET tdata = ?,
+                     clientversion = ?,
+                     buildtime = ?
+                 WHERE day = ?`
+		tx.Exec(query, tdata, scbase.WaveVersion, scbase.BuildTime, dayStr)
+		return nil
+	})
+	if txErr != nil {
+		return txErr
+	}
+	return nil
+}
+
+func GetNonUploadedActivity(ctx context.Context) ([]*ActivityType, error) {
+	var rtn []*ActivityType
+	txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		query := `SELECT * FROM activity WHERE uploaded = 0 ORDER BY day DESC LIMIT 30`
+		tx.Select(&rtn, query)
+		return nil
+	})
+	if txErr != nil {
+		return nil, txErr
+	}
+	return rtn, nil
+}
+
+// note, will not mark the current day as uploaded
+func MarkActivityAsUploaded(ctx context.Context, activityArr []*ActivityType) error {
+	dayStr := GetCurDayStr()
+	txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error {
+		query := `UPDATE activity SET uploaded = 1 WHERE day = ?`
+		for _, activity := range activityArr {
+			if activity.Day == dayStr {
+				continue
+			}
+			tx.Exec(query, activity.Day)
+		}
+		return nil
+	})
+	return txErr
+}
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Sylvia Crowe	01c9a3444a	fix: integrate skeema knownhosts fix This fix makes it possible to differentiate between keys when multiple are provided by the remote server. It does not solve the case of multiple keys of the same type being shared, but it handles multiple keys of different types being shared, which is much more common.	2024-08-28 07:14:35 -07:00
Cole Lashley	69b9ce33b3	User input keybindings fix (#494 ) * added user input fix and focusVal optoinal in footer * small fixes and removed focusVal for now	2024-03-26 15:05:18 -07:00
Evan Simkowitz	a121bd4bb5	Move bookmarks, history, playbook, and telemetry code out of sstore (#493 ) * break out telemetry and playbook * break out bookmarks * add license disclaimers	2024-03-25 20:20:52 -07:00
dependabot[bot]	dcc7b2943e	Bump webpack-dev-middleware from 5.3.3 to 5.3.4 (#490 ) Bumps [webpack-dev-middleware](https://github.com/webpack/webpack-dev-middleware) from 5.3.3 to 5.3.4. - [Release notes](https://github.com/webpack/webpack-dev-middleware/releases) - [Changelog](https://github.com/webpack/webpack-dev-middleware/blob/v5.3.4/CHANGELOG.md) - [Commits](https://github.com/webpack/webpack-dev-middleware/compare/v5.3.3...v5.3.4) --- updated-dependencies: - dependency-name: webpack-dev-middleware dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2024-03-25 09:55:15 -07:00