From f8c675c3e76e8224fd77fda7672f15a4ad0ce5ed Mon Sep 17 00:00:00 2001 From: sawka Date: Mon, 23 Jan 2023 12:54:32 -0800 Subject: [PATCH] updates to telemetry, separate telemetry commands, use json data --- cmd/main-server.go | 13 ++- db/migrations/000002_activity.up.sql | 5 +- pkg/cmdrunner/cmdrunner.go | 123 +++++++++++++++++++-------- pkg/pcloud/pcloud.go | 12 ++- pkg/sstore/dbops.go | 36 +++++--- pkg/sstore/sstore.go | 32 ++++--- 6 files changed, 156 insertions(+), 65 deletions(-) diff --git a/cmd/main-server.go b/cmd/main-server.go index bf66c9d8..d3fef53d 100644 --- a/cmd/main-server.go +++ b/cmd/main-server.go @@ -407,9 +407,18 @@ func test() error { } func sendTelemetryWrapper() { + defer func() { + r := recover() + if r == nil { + return + } + log.Printf("[error] in sendTelemetryWrapper: %v\n", r) + debug.PrintStack() + return + }() ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second) defer cancelFn() - err := pcloud.SendTelemetry(ctx) + err := pcloud.SendTelemetry(ctx, false) if err != nil { log.Printf("[error] sending telemetry: %v\n", err) } @@ -498,7 +507,7 @@ func main() { go func() { log.Printf("PCLOUD_ENDPOINT=%s\n", pcloud.GetEndpoint()) - time.Sleep(1 * time.Minute) + time.Sleep(30 * time.Second) for { sendTelemetryWrapper() // send new telemetry every 8-hours diff --git a/db/migrations/000002_activity.up.sql b/db/migrations/000002_activity.up.sql index dbd82104..d6a84aa3 100644 --- a/db/migrations/000002_activity.up.sql +++ b/db/migrations/000002_activity.up.sql @@ -1,10 +1,7 @@ CREATE TABLE activity ( day varchar(20) PRIMARY KEY, uploaded boolean NOT NULL, - numcommands int NOT NULL, - activeminutes int NOT NULL, - fgminutes int NOT NULL, - openminutes int NOT NULL, + tdata json NOT NULL, tzname varchar(50) NOT NULL, tzoffset int NOT NULL, clientversion varchar(20) NOT NULL, diff --git a/pkg/cmdrunner/cmdrunner.go b/pkg/cmdrunner/cmdrunner.go index 456aa29c..40ca7038 100644 --- a/pkg/cmdrunner/cmdrunner.go +++ b/pkg/cmdrunner/cmdrunner.go @@ -53,15 +53,14 @@ var RemoteSetArgs = []string{"alias", "connectmode", "key", "password", "autoins var WindowCmds = []string{"run", "comment", "cd", "cr", "clear", "sw", "reset", "signal"} var NoHistCmds = []string{"_compgen", "line", "history", "_killserver"} -var GlobalCmds = []string{"session", "screen", "remote", "set", "client"} +var GlobalCmds = []string{"session", "screen", "remote", "set", "client", "telemetry"} var SetVarNameMap map[string]string = map[string]string{ - "tabcolor": "screen.tabcolor", - "pterm": "window.pterm", - "anchor": "sw.anchor", - "focus": "sw.focus", - "line": "sw.line", - "telemetry": "client.telemetry", + "tabcolor": "screen.tabcolor", + "pterm": "window.pterm", + "anchor": "sw.anchor", + "focus": "sw.focus", + "line": "sw.line", } var SetVarScopes = []SetVarScope{ @@ -159,9 +158,14 @@ func init() { registerCmdFn("line:purge", LinePurgeCommand) registerCmdFn("client", ClientCommand) - registerCmdFn("client:set", ClientSetCommand) registerCmdFn("client:show", ClientShowCommand) + registerCmdFn("telemetry", TelemetryCommand) + registerCmdFn("telemetry:on", TelemetryOnCommand) + registerCmdFn("telemetry:off", TelemetryOffCommand) + registerCmdFn("telemetry:send", TelemetrySendCommand) + registerCmdFn("telemetry:show", TelemetryShowCommand) + registerCmdFn("history", HistoryCommand) registerCmdFn("_killserver", KillServerCommand) @@ -2085,7 +2089,7 @@ func KillServerCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (s } func ClientCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (sstore.UpdatePacket, error) { - return nil, fmt.Errorf("/client requires a subcommand: %s", formatStrs([]string{"set", "show"}, "or", false)) + return nil, fmt.Errorf("/client requires a subcommand: %s", formatStrs([]string{"show"}, "or", false)) } func boolToStr(v bool, trueStr string, falseStr string) string { @@ -2113,45 +2117,94 @@ func ClientShowCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (s return update, nil } -func ClientSetCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (sstore.UpdatePacket, error) { +func TelemetryCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (sstore.UpdatePacket, error) { + return nil, fmt.Errorf("/telemetry requires a subcommand: %s", formatStrs([]string{"show", "on", "off", "send"}, "or", false)) +} + +func setNoTelemetry(ctx context.Context, clientData *sstore.ClientData, noTelemetryVal bool) error { + clientOpts := clientData.ClientOpts + clientOpts.NoTelemetry = noTelemetryVal + err := sstore.SetClientOpts(ctx, clientOpts) + if err != nil { + return fmt.Errorf("error trying to update client telemetry: %v", err) + } + log.Printf("client no-telemetry setting updated to %v\n", noTelemetryVal) + err = pcloud.SendNoTelemetryUpdate(ctx, clientOpts.NoTelemetry) + if err != nil { + // ignore error, just log + log.Printf("[error] sending no-telemetry update: %v\n", err) + log.Printf("note that telemetry update has still taken effect locally, and will be respected by the client\n") + } + return nil +} + +func TelemetryOnCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (sstore.UpdatePacket, error) { clientData, err := sstore.EnsureClientData(ctx) if err != nil { return nil, fmt.Errorf("cannot retrieve client data: %v\n", err) } - var varsUpdated []string - if pk.Kwargs["telemetry"] != "" { - noTelemetry := !resolveBool(pk.Kwargs["telemetry"], true) - if clientData.ClientOpts.NoTelemetry != noTelemetry { - clientOpts := clientData.ClientOpts - clientOpts.NoTelemetry = noTelemetry - err = sstore.SetClientOpts(ctx, clientOpts) - if err != nil { - return nil, fmt.Errorf("error trying to update client telemetry: %v", err) - } - log.Printf("client telemetry setting updated to %v\n", !noTelemetry) - err = pcloud.SendNoTelemetryUpdate(ctx, noTelemetry) - if err != nil { - // ignore error, just log - log.Printf("[error] sending no-telemetry update: %v\n", err) - log.Printf("note that telemetry update has still taken effect locally, and will be respected by the client\n") - } - } else { - log.Printf("client telemetry setting unchanged, is %v\n", !noTelemetry) - } - varsUpdated = append(varsUpdated, "telemetry") + if !clientData.ClientOpts.NoTelemetry { + return sstore.InfoMsgUpdate("telemetry is already on"), nil } - if len(varsUpdated) == 0 { - return nil, fmt.Errorf("/client:set no updates, can set %s", formatStrs([]string{"telemetry"}, "or", false)) + err = setNoTelemetry(ctx, clientData, false) + if err != nil { + return nil, err } + err = pcloud.SendTelemetry(ctx, false) + if err != nil { + // ignore error, but log + log.Printf("[error] sending telemetry update (in /telemetry:on): %v\n", err) + } + return sstore.InfoMsgUpdate("telemetry is now on"), nil +} + +func TelemetryOffCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (sstore.UpdatePacket, error) { + clientData, err := sstore.EnsureClientData(ctx) + if err != nil { + return nil, fmt.Errorf("cannot retrieve client data: %v\n", err) + } + if clientData.ClientOpts.NoTelemetry { + return sstore.InfoMsgUpdate("telemetry is already off"), nil + } + err = setNoTelemetry(ctx, clientData, true) + if err != nil { + return nil, err + } + return sstore.InfoMsgUpdate("telemetry is now off"), nil +} + +func TelemetryShowCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (sstore.UpdatePacket, error) { + clientData, err := sstore.EnsureClientData(ctx) + if err != nil { + return nil, fmt.Errorf("cannot retrieve client data: %v\n", err) + } + var buf bytes.Buffer + buf.WriteString(fmt.Sprintf(" %-15s %s\n", "telemetry", boolToStr(clientData.ClientOpts.NoTelemetry, "off", "on"))) update := sstore.ModelUpdate{ Info: &sstore.InfoMsgType{ - InfoMsg: fmt.Sprintf("client updated %s", formatStrs(varsUpdated, "and", false)), - TimeoutMs: 2000, + InfoTitle: fmt.Sprintf("telemetry info"), + InfoLines: splitLinesForInfo(buf.String()), }, } return update, nil } +func TelemetrySendCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (sstore.UpdatePacket, error) { + clientData, err := sstore.EnsureClientData(ctx) + if err != nil { + return nil, fmt.Errorf("cannot retrieve client data: %v\n", err) + } + force := resolveBool(pk.Kwargs["force"], false) + if clientData.ClientOpts.NoTelemetry && !force { + return nil, fmt.Errorf("cannot send telemetry, telemetry is off. pass force=1 to force the send, or turn on telemetry with /telemetry:on") + } + err = pcloud.SendTelemetry(ctx, force) + if err != nil { + return nil, fmt.Errorf("failed to send telemetry: %v", err) + } + return sstore.InfoMsgUpdate("telemetry sent"), nil +} + func formatTermOpts(termOpts sstore.TermOpts) string { if termOpts.Cols == 0 { return "???" diff --git a/pkg/pcloud/pcloud.go b/pkg/pcloud/pcloud.go index ffb39d2c..e55d4230 100644 --- a/pkg/pcloud/pcloud.go +++ b/pkg/pcloud/pcloud.go @@ -28,6 +28,7 @@ type NoTelemetryInputType struct { type TelemetryInputType struct { UserId string `json:"userid"` ClientId string `json:"clientid"` + CurDay string `json:"curday"` Activity []*sstore.ActivityType `json:"activity"` } @@ -87,12 +88,12 @@ func doRequest(req *http.Request, outputObj interface{}) (*http.Response, error) return resp, nil } -func SendTelemetry(ctx context.Context) error { +func SendTelemetry(ctx context.Context, force bool) error { clientData, err := sstore.EnsureClientData(ctx) if err != nil { return fmt.Errorf("cannot retrieve client data: %v", err) } - if clientData.ClientOpts.NoTelemetry { + if !force && clientData.ClientOpts.NoTelemetry { return nil } activity, err := sstore.GetNonUploadedActivity(ctx) @@ -103,7 +104,8 @@ func SendTelemetry(ctx context.Context) error { return nil } log.Printf("sending telemetry data\n") - input := TelemetryInputType{UserId: clientData.UserId, ClientId: clientData.ClientId, Activity: activity} + dayStr := sstore.GetCurDayStr() + input := TelemetryInputType{UserId: clientData.UserId, ClientId: clientData.ClientId, CurDay: dayStr, Activity: activity} req, err := makePostReq(ctx, "/telemetry", input) if err != nil { return err @@ -112,6 +114,10 @@ func SendTelemetry(ctx context.Context) error { if err != nil { return err } + err = sstore.MarkActivityAsUploaded(ctx, activity) + if err != nil { + return fmt.Errorf("error marking activity as uploaded: %v", err) + } return nil } diff --git a/pkg/sstore/dbops.go b/pkg/sstore/dbops.go index 2a501904..c1feacfe 100644 --- a/pkg/sstore/dbops.go +++ b/pkg/sstore/dbops.go @@ -1838,28 +1838,37 @@ func GetRIsForWindow(ctx context.Context, sessionId string, windowId string) ([] return rtn, nil } -func UpdateCurrentActivity(ctx context.Context, update ActivityUpdate) error { +func GetCurDayStr() string { now := time.Now() dayStr := now.Format("2006-01-02") + return dayStr +} + +func UpdateCurrentActivity(ctx context.Context, update ActivityUpdate) error { + now := time.Now() + dayStr := GetCurDayStr() txErr := WithTx(ctx, func(tx *TxWrap) error { - query := `SELECT day FROM activity WHERE day = ?` - if !tx.Exists(query, dayStr) { - query = `INSERT INTO activity (day, uploaded, numcommands, fgminutes, activeminutes, openminutes, tzname, tzoffset, clientversion, clientarch) - VALUES (?, 0, 0, 0, 0, 0, ?, ?, ?, ?)` + var tdata TelemetryData + query := `SELECT tdata FROM activity WHERE day = ?` + found := tx.GetWrap(&tdata, query, dayStr) + if !found { + query = `INSERT INTO activity (day, uploaded, tdata, tzname, tzoffset, clientversion, clientarch) + VALUES (?, 0, ?, ?, ?, ?, ?)` tzName, tzOffset := now.Zone() if len(tzName) > MaxTzNameLen { tzName = tzName[0:MaxTzNameLen] } - tx.ExecWrap(query, dayStr, tzName, tzOffset, scbase.PromptVersion, scbase.ClientArch()) + tx.ExecWrap(query, dayStr, tdata, tzName, tzOffset, scbase.PromptVersion, scbase.ClientArch()) } + tdata.NumCommands += update.NumCommands + tdata.FgMinutes += update.FgMinutes + tdata.ActiveMinutes += update.ActiveMinutes + tdata.OpenMinutes += update.OpenMinutes query = `UPDATE activity - SET numcommands = numcommands + ?, - fgminutes = fgminutes + ?, - activeminutes = activeminutes + ?, - openminutes = openminutes + ?, + SET tdata = ?, clientversion = ? WHERE day = ?` - tx.ExecWrap(query, update.NumCommands, update.FgMinutes, update.ActiveMinutes, update.OpenMinutes, scbase.PromptVersion, dayStr) + tx.ExecWrap(query, tdata, scbase.PromptVersion, dayStr) return nil }) if txErr != nil { @@ -1881,10 +1890,15 @@ func GetNonUploadedActivity(ctx context.Context) ([]*ActivityType, error) { return rtn, nil } +// note, will not mark the current day as uploaded func MarkActivityAsUploaded(ctx context.Context, activityArr []*ActivityType) error { + dayStr := GetCurDayStr() txErr := WithTx(ctx, func(tx *TxWrap) error { query := `UPDATE activity SET uploaded = 1 WHERE day = ?` for _, activity := range activityArr { + if activity.Day == dayStr { + continue + } tx.ExecWrap(query, activity.Day) } return nil diff --git a/pkg/sstore/sstore.go b/pkg/sstore/sstore.go index cff8f208..d75db565 100644 --- a/pkg/sstore/sstore.go +++ b/pkg/sstore/sstore.go @@ -120,16 +120,28 @@ type ActivityUpdate struct { } type ActivityType struct { - Day string `json:"day"` - Uploaded bool `json:"-"` - NumCommands int `json:"numcommands"` - ActiveMinutes int `json:"activeminutes"` - FgMinutes int `json:"fgminutes"` - OpenMinutes int `json:"openminutes"` - TzName string `json:"tzname"` - TzOffset int `json:"tzoffset"` - ClientVersion string `json:"clientversion"` - ClientArch string `json:"clientarch"` + Day string `json:"day"` + Uploaded bool `json:"-"` + TData TelemetryData `json:"tdata"` + TzName string `json:"tzname"` + TzOffset int `json:"tzoffset"` + ClientVersion string `json:"clientversion"` + ClientArch string `json:"clientarch"` +} + +type TelemetryData struct { + NumCommands int `json:"numcommands"` + ActiveMinutes int `json:"activeminutes"` + FgMinutes int `json:"fgminutes"` + OpenMinutes int `json:"openminutes"` +} + +func (tdata TelemetryData) Value() (driver.Value, error) { + return quickValueJson(tdata) +} + +func (tdata *TelemetryData) Scan(val interface{}) error { + return quickScanJson(tdata, val) } type ClientOptsType struct {