merge 'main' into use-ssh-library--add-user-input

I had to reimplement a few front end changes but nothing too major aside
from that.

src/app/appconst.ts

@@ -8,6 +8,7 @@ export const SESSION_SETTINGS = "sessionSettings";
 export const LINE_SETTINGS = "lineSettings";
 export const CLIENT_SETTINGS = "clientSettings";
 export const TAB_SWITCHER = "tabSwitcher";
+export const USER_INPUT = "userInput";
 
 export const LineContainer_Main = "main";
 export const LineContainer_History = "history";

src/app/common/elements/markdown.less

@@ -5,6 +5,7 @@
     margin-bottom: 10px;
     font-family: @markdown-font;
     font-size: 14px;
+    overflow-wrap: break-word;
 
     code {
         background-color: @markdown-highlight;

src/app/common/elements/passwordfield.tsx

@@ -60,7 +60,13 @@ class PasswordField extends TextField {
         };
 
         return (
-            <div className={cn(`wave-textfield wave-password ${className || ""}`, { focused: focused, error: error })}>
+            <div
+                className={cn(`wave-textfield wave-password ${className || ""}`, {
+                    focused: focused,
+                    error: error,
+                    "no-label": !label,
+                })}
+            >
                 {decoration?.startDecoration && <>{decoration.startDecoration}</>}
                 <div className="wave-textfield-inner">
                     <label

src/app/common/modals/index.tsx

@@ -9,3 +9,4 @@ export { TabSwitcherModal } from "./tabswitcher";
 export { SessionSettingsModal } from "./sessionsettings";
 export { ScreenSettingsModal } from "./screensettings";
 export { LineSettingsModal } from "./linesettings";
+export { UserInputModal } from "./userinput";

src/app/common/modals/provider.tsx

@@ -17,7 +17,7 @@ class ModalsProvider extends React.Component {
         for (let i = 0; i < store.length; i++) {
             let entry = store[i];
             let Comp = entry.component;
-            rtn.push(<Comp key={entry.uniqueKey} />);
+            rtn.push(<Comp key={entry.uniqueKey} {...entry.props} />);
         }
         return <>{rtn}</>;
     }

src/app/common/modals/registry.tsx

@@ -12,19 +12,21 @@ import {
     SessionSettingsModal,
     ScreenSettingsModal,
     LineSettingsModal,
+    UserInputModal,
 } from "../modals";
 import * as constants from "../../appconst";
 
-const modalsRegistry: { [key: string]: () => React.ReactElement } = {
-    [constants.ABOUT]: () => <AboutModal />,
-    [constants.CREATE_REMOTE]: () => <CreateRemoteConnModal />,
-    [constants.VIEW_REMOTE]: () => <ViewRemoteConnDetailModal />,
-    [constants.EDIT_REMOTE]: () => <EditRemoteConnModal />,
-    [constants.ALERT]: () => <AlertModal />,
-    [constants.SCREEN_SETTINGS]: () => <ScreenSettingsModal />,
-    [constants.SESSION_SETTINGS]: () => <SessionSettingsModal />,
-    [constants.LINE_SETTINGS]: () => <LineSettingsModal />,
-    [constants.TAB_SWITCHER]: () => <TabSwitcherModal />,
+const modalsRegistry: { [key: string]: React.ComponentType } = {
+    [constants.ABOUT]: AboutModal,
+    [constants.CREATE_REMOTE]: CreateRemoteConnModal,
+    [constants.VIEW_REMOTE]: ViewRemoteConnDetailModal,
+    [constants.EDIT_REMOTE]: EditRemoteConnModal,
+    [constants.ALERT]: AlertModal,
+    [constants.SCREEN_SETTINGS]: ScreenSettingsModal,
+    [constants.SESSION_SETTINGS]: SessionSettingsModal,
+    [constants.LINE_SETTINGS]: LineSettingsModal,
+    [constants.TAB_SWITCHER]: TabSwitcherModal,
+    [constants.USER_INPUT]: UserInputModal,
 };
 
 export { modalsRegistry };

src/app/common/modals/userinput.less

@@ -0,0 +1,15 @@
+@import "../../../app/common/themes/themes.less";
+
+.userinput-modal {
+    width: 500px;
+
+    .wave-modal-content {
+        .wave-modal-body {
+            padding: 20px 20px;
+
+            .userinput-query {
+                margin-bottom: 10px;
+            }
+        }
+    }
+}

src/app/common/modals/userinput.tsx

@@ -0,0 +1,88 @@
+import * as React from "react";
+import { GlobalModel } from "../../../models";
+import { Choose, When, If } from "tsx-control-statements";
+import { Modal, PasswordField, Markdown } from "../elements";
+import { UserInputRequest } from "../../../types/types";
+
+import "./userinput.less";
+
+export const UserInputModal = (userInputRequest: UserInputRequest) => {
+    const [responseText, setResponseText] = React.useState(null);
+    const [countdown, setCountdown] = React.useState(Math.floor(userInputRequest.timeoutms / 1000));
+
+    const closeModal = React.useCallback(() => {
+        GlobalModel.sendUserInput({
+            type: "userinputresp",
+            requestid: userInputRequest.requestid,
+            errormsg: "Canceled by the user",
+        });
+        GlobalModel.remotesModel.closeModal();
+    }, [responseText, userInputRequest]);
+
+    const handleSendText = React.useCallback(() => {
+        GlobalModel.sendUserInput({
+            type: "userinputresp",
+            requestid: userInputRequest.requestid,
+            text: responseText,
+        });
+        GlobalModel.remotesModel.closeModal();
+    }, [responseText, userInputRequest]);
+
+    const handleSendConfirm = React.useCallback(
+        (response: boolean) => {
+            GlobalModel.sendUserInput({
+                type: "userinputresp",
+                requestid: userInputRequest.requestid,
+                confirm: response,
+            });
+            GlobalModel.remotesModel.closeModal();
+        },
+        [userInputRequest]
+    );
+
+    React.useEffect(() => {
+        let timeout: ReturnType<typeof setTimeout>;
+        if (countdown == 0) {
+            timeout = setTimeout(() => {
+                GlobalModel.remotesModel.closeModal();
+            }, 300);
+        } else {
+            timeout = setTimeout(() => {
+                setCountdown(countdown - 1);
+            }, 1000);
+        }
+        return () => clearTimeout(timeout);
+    }, [countdown]);
+
+    return (
+        <Modal className="userinput-modal">
+            <Modal.Header onClose={closeModal} title={userInputRequest.title + ` (${countdown})`} />
+            <div className="wave-modal-body">
+                <div className="userinput-query">
+                    <If condition={userInputRequest.markdown}>
+                        <Markdown text={userInputRequest.querytext} />
+                    </If>
+                    <If condition={!userInputRequest.markdown}>{userInputRequest.querytext}</If>
+                </div>
+                <Choose>
+                    <When condition={userInputRequest.responsetype == "text"}>
+                        <PasswordField onChange={setResponseText} value={responseText} maxLength={400} />
+                    </When>
+                </Choose>
+            </div>
+            <Choose>
+                <When condition={userInputRequest.responsetype == "text"}>
+                    <Modal.Footer onCancel={closeModal} onOk={handleSendText} okLabel="Continue" />
+                </When>
+                <When condition={userInputRequest.responsetype == "confirm"}>
+                    <Modal.Footer
+                        onCancel={() => handleSendConfirm(false)}
+                        onOk={() => handleSendConfirm(true)}
+                        okLabel="Yes"
+                        cancelLabel="No"
+                    />
+                </When>
+            </Choose>
+        </Modal>
+    );
+};

src/models/modals.ts

@@ -8,14 +8,14 @@ import { modalsRegistry } from "../app/common/modals/registry";
 import { OArr } from "../types/types";
 
 class ModalsModel {
-    store: OArr<ModalStoreEntry> = mobx.observable.array([], { name: "ModalsModel-store" });
+    store: OArr<ModalStoreEntry> = mobx.observable.array([], { name: "ModalsModel-store", deep: false });
 
-    pushModal(modalId: string) {
+    pushModal(modalId: string, props?: any) {
         const modalFactory = modalsRegistry[modalId];
 
         if (modalFactory && !this.store.some((modal) => modal.id === modalId)) {
             mobx.action(() => {
-                this.store.push({ id: modalId, component: modalFactory, uniqueKey: uuidv4() });
+                this.store.push({ id: modalId, component: modalFactory, uniqueKey: uuidv4(), props });
             })();
         }
     }

src/models/model.ts

@@ -31,6 +31,8 @@ import {
     RendererContext,
     ClientDataType,
     AlertMessageType,
+    UserInputRequest,
+    UserInputResponsePacket,
     ScreenLinesType,
     RemoteViewType,
     CommandRtnType,
@@ -902,6 +904,10 @@ class Model {
                 this.getScreenById_single(snc.screenid)?.setNumRunningCmds(snc.num);
             }
         }
+        if ("userinputrequest" in update) {
+            let userInputRequest: UserInputRequest = update.userinputrequest;
+            this.modalsModel.pushModal(appconst.USER_INPUT, userInputRequest);
+        }
     }
 
     updateRemotes(remotes: RemoteType[]): void {
@@ -1309,6 +1315,10 @@ class Model {
         this.ws.pushMessage(inputPacket);
     }
 
+    sendUserInput(userInputResponsePacket: UserInputResponsePacket) {
+        this.ws.pushMessage(userInputResponsePacket);
+    }
+
     sendCmdInputText(screenId: string, sp: StrWithPos) {
         let pk: CmdInputTextPacketType = {
             type: "cmdinputtext",

src/models/model_old.ts

@@ -51,6 +51,8 @@ import {
     HistoryViewDataType,
     AlertMessageType,
     HistorySearchParams,
+    UserInputRequest,
+    UserInputResponsePacket,
     FocusTypeStrs,
     ScreenLinesType,
     HistoryTypeStrs,
@@ -3376,14 +3378,14 @@ class RemotesModel {
 }
 
 class ModalsModel {
-    store: OArr<T.ModalStoreEntry> = mobx.observable.array([], { name: "ModalsModel-store" });
+    store: OArr<T.ModalStoreEntry> = mobx.observable.array([], { name: "ModalsModel-store", deep: false });
 
-    pushModal(modalId: string) {
+    pushModal(modalId: string, props?: any) {
         const modalFactory = modalsRegistry[modalId];
 
         if (modalFactory && !this.store.some((modal) => modal.id === modalId)) {
             mobx.action(() => {
-                this.store.push({ id: modalId, component: modalFactory, uniqueKey: uuidv4() });
+                this.store.push({ id: modalId, component: modalFactory, uniqueKey: uuidv4(), props: props });
             })();
         }
     }
@@ -4184,6 +4186,10 @@ class Model {
                 this.getScreenById_single(snc.screenid)?.setNumRunningCmds(snc.num);
             }
         }
+        if ("userinputrequest" in update) {
+            let userInputRequest: UserInputRequest = update.userinputrequest;
+            this.modalsModel.pushModal(appconst.USER_INPUT, userInputRequest);
+        }
     }
 
     updateRemotes(remotes: RemoteType[]): void {
@@ -4591,6 +4597,10 @@ class Model {
         this.ws.pushMessage(inputPacket);
     }
 
+    sendUserInput(userInputResponsePacket: UserInputResponsePacket) {
+        this.ws.pushMessage(userInputResponsePacket);
+    }
+
     sendCmdInputText(screenId: string, sp: T.StrWithPos) {
         let pk: T.CmdInputTextPacketType = {
             type: "cmdinputtext",

src/types/types.ts

@@ -333,6 +333,7 @@ type ModelUpdateType = {
     alertmessage?: AlertMessageType;
     screenstatusindicators?: ScreenStatusIndicatorUpdateType[];
     screennumrunningcommands?: ScreenNumRunningCommandsUpdateType[];
+    userinputrequest?: UserInputRequest;
 };
 
 type HistoryViewDataType = {
@@ -595,6 +596,23 @@ type HistorySearchParams = {
     filterCmds?: boolean;
 };
 
+type UserInputRequest = {
+    requestid: string;
+    querytext: string;
+    responsetype: string;
+    title: string;
+    markdown: boolean;
+    timeoutms: number;
+};
+
+type UserInputResponsePacket = {
+    type: string;
+    requestid: string;
+    text?: string;
+    confirm?: boolean;
+    errormsg?: string;
+};
+
 type RenderModeType = "normal" | "collapsed" | "expanded";
 
 type WebScreen = {
@@ -730,6 +748,7 @@ type ModalStoreEntry = {
     id: string;
     component: React.ComponentType;
     uniqueKey: string;
+    props?: any;
 };
 
 type StrWithPos = {
@@ -809,6 +828,8 @@ export type {
     RenderModeType,
     AlertMessageType,
     HistorySearchParams,
+    UserInputRequest,
+    UserInputResponsePacket,
     ScreenLinesType,
     FocusTypeStrs,
     HistoryTypeStrs,

waveshell/pkg/packet/parser.go

@@ -156,16 +156,12 @@ func (p *PacketParser) trySendRpcResponse(pk PacketType) bool {
 		return false
 	}
 	p.Lock.Lock()
-	defer p.Lock.Unlock()
 	entry := p.RpcMap[respId]
+	p.Lock.Unlock()
 	if entry == nil {
 		return false
 	}
-	// nonblocking send
-	select {
-	case entry.RespCh <- respPk:
-	default:
-	}
+	entry.RespCh <- respPk
 	return true
 }
 

wavesrv/pkg/cmdrunner/resolver.go

@@ -24,6 +24,11 @@ const (
 	R_RemoteConnected = 16
 )
 
+const (
+	ConnectedRemote = "connected"
+	LocalRemote     = "local"
+)
+
 type resolvedIds struct {
 	SessionId string
 	ScreenId  string

wavesrv/pkg/remote/remote.go

@@ -40,7 +40,7 @@ import (
 	"golang.org/x/mod/semver"
 )
 
-const UseSshLibrary = false
+const UseSshLibrary = true
 
 const RemoteTypeMShell = "mshell"
 const DefaultTerm = "xterm-256color"
@@ -50,6 +50,7 @@ const RemoteTermRows = 8
 const RemoteTermCols = 80
 const PtyReadBufSize = 100
 const RemoteConnectTimeout = 15 * time.Second
+const RpcIterChannelSize = 100
 
 var envVarsToStrip map[string]bool = map[string]bool{
 	"PROMPT":               true,
@@ -665,7 +666,12 @@ func (msh *MShellProc) GetRemoteRuntimeState() RemoteRuntimeState {
 	if vars["remoteuser"] == "root" || vars["sudo"] == "1" {
 		vars["isroot"] = "1"
 	}
-	state.RemoteVars = vars
+	varsCopy := make(map[string]string)
+	// deep copy so that concurrent calls don't collide on this data
+	for key, value := range vars {
+		varsCopy[key] = value
+	}
+	state.RemoteVars = varsCopy
 	state.ActiveShells = msh.StateMap.GetShells()
 	return state
 }
@@ -1203,6 +1209,10 @@ func (msh *MShellProc) ReInit(ctx context.Context, shellType string) (*packet.Sh
 	return ssPk, nil
 }
 
+func (msh *MShellProc) WriteFile(ctx context.Context, writePk *packet.WriteFilePacketType) (*packet.RpcResponseIter, error) {
+	return msh.PacketRpcIter(ctx, writePk)
+}
+
 func (msh *MShellProc) StreamFile(ctx context.Context, streamPk *packet.StreamFilePacketType) (*packet.RpcResponseIter, error) {
 	return msh.PacketRpcIter(ctx, streamPk)
 }
@@ -1886,7 +1896,6 @@ func RunCommand(ctx context.Context, rcOpts RunCommandOpts, runPacket *packet.Ru
 		RunPacket: runPacket,
 	})
 
-	go pushNumRunningCmdsUpdate(&runPacket.CK, 1)
 	return cmd, func() { removeCmdWait(runPacket.CK) }, nil
 }
 
@@ -1925,7 +1934,7 @@ func (msh *MShellProc) PacketRpcIter(ctx context.Context, pk packet.RpcPacketTyp
 		return nil, fmt.Errorf("PacketRpc passed nil packet")
 	}
 	reqId := pk.GetReqId()
-	msh.ServerProc.Output.RegisterRpc(reqId)
+	msh.ServerProc.Output.RegisterRpcSz(reqId, RpcIterChannelSize)
 	err := msh.ServerProc.Input.SendPacketCtx(ctx, pk)
 	if err != nil {
 		return nil, err
@@ -2064,8 +2073,6 @@ func (msh *MShellProc) handleCmdDonePacket(donePk *packet.CmdDonePacketType) {
 			// fall-through (nothing to do)
 		}
 	}
-
-	go pushNumRunningCmdsUpdate(&donePk.CK, -1)
 	sstore.MainBus.SendUpdate(update)
 	return
 }

wavesrv/pkg/remote/sshclient.go

@@ -1,43 +1,117 @@
-// Copyright 2024, Command Line Inc.
+// Copyright 2023-2024, Command Line Inc.
 // SPDX-License-Identifier: Apache-2.0
 
 package remote
 
 import (
-	"errors"
+	"bytes"
+	"context"
+	"crypto/x509"
+	"encoding/base64"
 	"fmt"
+	"log"
+	"net"
 	"os"
 	"os/user"
+	"path/filepath"
 	"strconv"
 	"strings"
+	"sync"
+	"time"
 
 	"github.com/kevinburke/ssh_config"
 	"github.com/wavetermdev/waveterm/waveshell/pkg/base"
+	"github.com/wavetermdev/waveterm/wavesrv/pkg/scpacket"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/sstore"
 	"golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/knownhosts"
 )
 
-func createPublicKeyAuth(identityFile string, passphrase string) (ssh.AuthMethod, error) {
+type UserInputCancelError struct {
+	Err error
+}
+
+func (uice UserInputCancelError) Error() string {
+	return uice.Err.Error()
+}
+
+func createPublicKeyAuth(identityFile string, passphrase string) (ssh.Signer, error) {
 	privateKey, err := os.ReadFile(base.ExpandHomeDir(identityFile))
 	if err != nil {
 		return nil, fmt.Errorf("failed to read ssh key file. err: %+v", err)
 	}
 	signer, err := ssh.ParsePrivateKey(privateKey)
-	if err != nil {
-		if errors.Is(err, &ssh.PassphraseMissingError{}) {
-			signer, err = ssh.ParsePrivateKeyWithPassphrase(privateKey, []byte(passphrase))
-			if err != nil {
-				return nil, fmt.Errorf("failed to parse private ssh key with passphrase. err: %+v", err)
-			}
-		} else {
-			return nil, fmt.Errorf("failed to parse private ssh key. err: %+v", err)
-		}
+	if err == nil {
+		return signer, err
 	}
-	return ssh.PublicKeys(signer), nil
+	if _, ok := err.(*ssh.PassphraseMissingError); !ok {
+		return nil, fmt.Errorf("failed to parse private ssh key. err: %+v", err)
+	}
+
+	signer, err = ssh.ParsePrivateKeyWithPassphrase(privateKey, []byte(passphrase))
+	if err == nil {
+		return signer, err
+	}
+	if err != x509.IncorrectPasswordError && err.Error() != "bcrypt_pbkdf: empty password" {
+		log.Printf("qwerty: %+v", err)
+		return nil, fmt.Errorf("failed to parse private ssh key. err: %+v", err)
+	}
+	request := &sstore.UserInputRequestType{
+		ResponseType: "text",
+		QueryText:    fmt.Sprintf("Enter passphrase for the SSH key: %s", identityFile),
+		Title:        "Publickey Auth + Passphrase",
+	}
+	ctx, cancelFn := context.WithTimeout(context.Background(), 15*time.Second)
+	defer cancelFn()
+	response, err := sstore.MainBus.GetUserInput(ctx, request)
+	if err != nil {
+		return nil, UserInputCancelError{Err: err}
+	}
+	return ssh.ParsePrivateKeyWithPassphrase(privateKey, []byte(response.Text))
 }
 
-func createKeyboardInteractiveAuth(password string) ssh.AuthMethod {
-	challenge := func(name, instruction string, questions []string, echos []bool) (answers []string, err error) {
+func createDefaultPasswordCallbackPrompt(password string) func() (secret string, err error) {
+	return func() (secret string, err error) {
+		// this should be modified to return an error if no password is stored
+		// but an empty password is not sufficient because some systems allow
+		// empty passwords
+		return password, nil
+	}
+}
+
+func createInteractivePasswordCallbackPrompt() func() (secret string, err error) {
+	return func() (secret string, err error) {
+		// limited to 15 seconds for some reason. this should be investigated more
+		// in the future
+		ctx, cancelFn := context.WithTimeout(context.Background(), 15*time.Second)
+		defer cancelFn()
+		request := &sstore.UserInputRequestType{
+			ResponseType: "text",
+			QueryText:    "Password:",
+			Title:        "Password Authentication",
+		}
+		response, err := sstore.MainBus.GetUserInput(ctx, request)
+		if err != nil {
+			return "", err
+		}
+		return response.Text, nil
+	}
+}
+
+func createCombinedPasswordCallbackPrompt(password string) func() (secret string, err error) {
+	var once sync.Once
+	return func() (secret string, err error) {
+		var prompt func() (secret string, err error)
+		once.Do(func() { prompt = createDefaultPasswordCallbackPrompt(password) })
+		if prompt == nil {
+			prompt = createInteractivePasswordCallbackPrompt()
+		}
+		return prompt()
+	}
+}
+
+func createNaiveKbdInteractiveChallenge(password string) func(name, instruction string, questions []string, echos []bool) (answers []string, err error) {
+	return func(name, instruction string, questions []string, echos []bool) (answers []string, err error) {
 		for _, q := range questions {
 			if strings.Contains(strings.ToLower(q), "password") {
 				answers = append(answers, password)
@@ -47,7 +121,291 @@ func createKeyboardInteractiveAuth(password string) ssh.AuthMethod {
 		}
 		return answers, nil
 	}
-	return ssh.KeyboardInteractive(challenge)
+}
+
+func createInteractiveKbdInteractiveChallenge() func(name, instruction string, questions []string, echos []bool) (answers []string, err error) {
+	return func(name, instruction string, questions []string, echos []bool) (answers []string, err error) {
+		if len(questions) != len(echos) {
+			return nil, fmt.Errorf("bad response from server: questions has len %d, echos has len %d", len(questions), len(echos))
+		}
+		for i, question := range questions {
+			echo := echos[i]
+			answer, err := promptChallengeQuestion(question, echo)
+			if err != nil {
+				return nil, err
+			}
+			answers = append(answers, answer)
+		}
+		return answers, nil
+	}
+}
+
+func promptChallengeQuestion(question string, echo bool) (answer string, err error) {
+	// limited to 15 seconds for some reason. this should be investigated more
+	// in the future
+	ctx, cancelFn := context.WithTimeout(context.Background(), 15*time.Second)
+	defer cancelFn()
+	request := &sstore.UserInputRequestType{
+		ResponseType: "text",
+		QueryText:    question,
+		Title:        "Keyboard Interactive Authentication",
+	}
+	response, err := sstore.MainBus.GetUserInput(ctx, request)
+	if err != nil {
+		return "", err
+	}
+	return response.Text, nil
+}
+
+func createCombinedKbdInteractiveChallenge(password string) ssh.KeyboardInteractiveChallenge {
+	var once sync.Once
+	return func(name, instruction string, questions []string, echos []bool) (answers []string, err error) {
+		var challenge ssh.KeyboardInteractiveChallenge
+		once.Do(func() { challenge = createNaiveKbdInteractiveChallenge(password) })
+		if challenge == nil {
+			challenge = createInteractiveKbdInteractiveChallenge()
+		}
+		return challenge(name, instruction, questions, echos)
+	}
+}
+
+func openKnownHostsForEdit(knownHostsFilename string) (*os.File, error) {
+	path, _ := filepath.Split(knownHostsFilename)
+	err := os.MkdirAll(path, 0700)
+	if err != nil {
+		return nil, err
+	}
+	return os.OpenFile(knownHostsFilename, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
+}
+
+func writeToKnownHosts(knownHostsFile string, newLine string, getUserVerification func() (*scpacket.UserInputResponsePacketType, error)) error {
+	if getUserVerification == nil {
+		getUserVerification = func() (*scpacket.UserInputResponsePacketType, error) {
+			return &scpacket.UserInputResponsePacketType{
+				Type:    "confirm",
+				Confirm: true,
+			}, nil
+		}
+	}
+
+	path, _ := filepath.Split(knownHostsFile)
+	err := os.MkdirAll(path, 0700)
+	if err != nil {
+		return err
+	}
+	f, err := os.OpenFile(knownHostsFile, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
+	if err != nil {
+		return err
+	}
+	// do not close writeable files with defer
+
+	// this file works, so let's ask the user for permission
+	response, err := getUserVerification()
+	if err != nil {
+		f.Close()
+		return UserInputCancelError{Err: err}
+	}
+	if !response.Confirm {
+		f.Close()
+		return UserInputCancelError{Err: fmt.Errorf("Canceled by the user")}
+	}
+
+	_, err = f.WriteString(newLine)
+	return f.Close()
+}
+
+func createUnknownKeyVerifier(knownHostsFile string, hostname string, remote string, key ssh.PublicKey) func() (*scpacket.UserInputResponsePacketType, error) {
+	base64Key := base64.StdEncoding.EncodeToString(key.Marshal())
+	queryText := fmt.Sprintf(
+		"The authenticity of host '%s (%s)' can't be established "+
+			"as it **does not exist in any checked known_hosts files**. "+
+			"The host you are attempting to connect to provides this %s key:  \n"+
+			"%s.\n\n"+
+			"**Would you like to continue connecting?** If so, the key will be permanently "+
+			"added to the file %s "+
+			"to protect from future man-in-the-middle attacks.", hostname, remote, key.Type(), base64Key, knownHostsFile)
+	request := &sstore.UserInputRequestType{
+		ResponseType: "confirm",
+		QueryText:    queryText,
+		Markdown:     true,
+		Title:        "Known Hosts Key Missing",
+	}
+	return func() (*scpacket.UserInputResponsePacketType, error) {
+		ctx, cancelFn := context.WithTimeout(context.Background(), 60*time.Second)
+		defer cancelFn()
+		return sstore.MainBus.GetUserInput(ctx, request)
+	}
+}
+
+func createMissingKnownHostsVerifier(knownHostsFile string, hostname string, remote string, key ssh.PublicKey) func() (*scpacket.UserInputResponsePacketType, error) {
+	base64Key := base64.StdEncoding.EncodeToString(key.Marshal())
+	queryText := fmt.Sprintf(
+		"The authenticity of host '%s (%s)' can't be established "+
+			"as **no known_hosts files could be found**. "+
+			"The host you are attempting to connect to provides this %s key:  \n"+
+			"%s.\n\n"+
+			"**Would you like to continue connecting?** If so:  \n"+
+			"- %s will be created  \n"+
+			"- the key will be added to %s\n\n"+
+			"This will protect from future man-in-the-middle attacks.", hostname, remote, key.Type(), base64Key, knownHostsFile, knownHostsFile)
+	request := &sstore.UserInputRequestType{
+		ResponseType: "confirm",
+		QueryText:    queryText,
+		Markdown:     true,
+		Title:        "Known Hosts File Missing",
+	}
+	return func() (*scpacket.UserInputResponsePacketType, error) {
+		ctx, cancelFn := context.WithTimeout(context.Background(), 60*time.Second)
+		defer cancelFn()
+		return sstore.MainBus.GetUserInput(ctx, request)
+	}
+}
+
+func lineContainsMatch(line []byte, matches [][]byte) bool {
+	for _, match := range matches {
+		if bytes.Contains(line, match) {
+			return true
+		}
+	}
+	return false
+}
+
+func createHostKeyCallback(opts *sstore.SSHOpts) (ssh.HostKeyCallback, error) {
+	rawUserKnownHostsFiles, _ := ssh_config.GetStrict(opts.SSHHost, "UserKnownHostsFile")
+	userKnownHostsFiles := strings.Fields(rawUserKnownHostsFiles) // TODO - smarter splitting escaped spaces and quotes
+	rawGlobalKnownHostsFiles, _ := ssh_config.GetStrict(opts.SSHHost, "GlobalKnownHostsFile")
+	globalKnownHostsFiles := strings.Fields(rawGlobalKnownHostsFiles) // TODO - smarter splitting escaped spaces and quotes
+	unexpandedKnownHostsFiles := append(userKnownHostsFiles, globalKnownHostsFiles...)
+	var knownHostsFiles []string
+	for _, filename := range unexpandedKnownHostsFiles {
+		knownHostsFiles = append(knownHostsFiles, base.ExpandHomeDir(filename))
+	}
+
+	// there are no good known hosts files
+	if len(knownHostsFiles) == 0 {
+		return nil, fmt.Errorf("no known_hosts files provided by ssh. defaults are overridden")
+	}
+
+	var unreadableFiles []string
+
+	// the library we use isn't very forgiving about files that are formatted
+	// incorrectly. if a problem file is found, it is removed from our list
+	// and we try again
+	var basicCallback ssh.HostKeyCallback
+	for basicCallback == nil && len(knownHostsFiles) > 0 {
+		var err error
+		basicCallback, err = knownhosts.New(knownHostsFiles...)
+		if serr, ok := err.(*os.PathError); ok {
+			badFile := serr.Path
+			unreadableFiles = append(unreadableFiles, badFile)
+			var okFiles []string
+			for _, filename := range knownHostsFiles {
+				if filename != badFile {
+					okFiles = append(okFiles, filename)
+				}
+			}
+			if len(okFiles) >= len(knownHostsFiles) {
+				return nil, fmt.Errorf("problem file (%s) doesn't exist. this should not be possible", badFile)
+			}
+			knownHostsFiles = okFiles
+		} else if err != nil {
+			// TODO handle obscure problems if possible
+			return nil, fmt.Errorf("known_hosts formatting error: %+v", err)
+		}
+	}
+
+	waveHostKeyCallback := func(hostname string, remote net.Addr, key ssh.PublicKey) error {
+		err := basicCallback(hostname, remote, key)
+		if err == nil {
+			// success
+			return nil
+		} else if _, ok := err.(*knownhosts.RevokedError); ok {
+			// revoked credentials are refused outright
+			return err
+		} else if _, ok := err.(*knownhosts.KeyError); !ok {
+			// this is an unknown error (note the !ok is opposite of usual)
+			return err
+		}
+		serr, _ := err.(*knownhosts.KeyError)
+		if len(serr.Want) == 0 {
+			// the key was not found
+
+			// try to write to a file that could be parsed
+			var err error
+			for _, filename := range knownHostsFiles {
+				newLine := knownhosts.Line([]string{knownhosts.Normalize(hostname)}, key)
+				getUserVerification := createUnknownKeyVerifier(filename, hostname, remote.String(), key)
+				err = writeToKnownHosts(filename, newLine, getUserVerification)
+				if err == nil {
+					break
+				}
+				if serr, ok := err.(UserInputCancelError); ok {
+					return serr
+				}
+			}
+
+			// try to write to a file that could not be read (file likely doesn't exist)
+			// should catch cases where there is no known_hosts file
+			if err != nil {
+				for _, filename := range unreadableFiles {
+					newLine := knownhosts.Line([]string{knownhosts.Normalize(hostname)}, key)
+					getUserVerification := createMissingKnownHostsVerifier(filename, hostname, remote.String(), key)
+					err = writeToKnownHosts(filename, newLine, getUserVerification)
+					if err == nil {
+						knownHostsFiles = []string{filename}
+						break
+					}
+					if serr, ok := err.(UserInputCancelError); ok {
+						return serr
+					}
+				}
+			}
+			if err != nil {
+				return err
+			}
+		} else {
+			// the key changed
+			correctKeyFingerprint := base64.StdEncoding.EncodeToString(key.Marshal())
+			var bulletListKnownHosts []string
+			for _, knownHostName := range knownHostsFiles {
+				withBulletPoint := "- " + knownHostName
+				bulletListKnownHosts = append(bulletListKnownHosts, withBulletPoint)
+			}
+			var offendingKeysFmt []string
+			for _, badKey := range serr.Want {
+				formattedKey := "- " + base64.StdEncoding.EncodeToString(badKey.Key.Marshal())
+				offendingKeysFmt = append(offendingKeysFmt, formattedKey)
+			}
+			alertText := fmt.Sprintf("**WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED!**\n\n"+
+				"If this is not expected, it is possible that someone could be trying to "+
+				"eavesdrop on you via a man-in-the-middle attack. "+
+				"Alternatively, the host you are connecting to may have changed its key. "+
+				"The %s key sent by the remote hist has the fingerprint:  \n"+
+				"%s\n\n"+
+				"If you are sure this is correct, please update your known_hosts files to "+
+				"remove the lines with the offending before trying to connect again.  \n"+
+				"**Known Hosts Files**  \n"+
+				"%s\n\n"+
+				"**Offending Keys**  \n"+
+				"%s", key.Type(), correctKeyFingerprint, strings.Join(bulletListKnownHosts, "  \n"), strings.Join(offendingKeysFmt, "  \n"))
+			update := &sstore.ModelUpdate{AlertMessage: &sstore.AlertMessageType{
+				Markdown: true,
+				Title:    "Known Hosts Key Changed",
+				Message:  alertText,
+			}}
+			sstore.MainBus.SendUpdate(update)
+			return fmt.Errorf("remote host identification has changed")
+		}
+
+		updatedCallback, err := knownhosts.New(knownHostsFiles...)
+		if err != nil {
+			return err
+		}
+		// try one final time
+		return updatedCallback(hostname, remote, key)
+	}
+
+	return waveHostKeyCallback, nil
 }
 
 func ConnectToClient(opts *sstore.SSHOpts) (*ssh.Client, error) {
@@ -60,14 +418,17 @@ func ConnectToClient(opts *sstore.SSHOpts) (*ssh.Client, error) {
 		identityFile = configIdentity
 	}
 
-	hostKeyCallback := ssh.InsecureIgnoreHostKey()
-	var authMethods []ssh.AuthMethod
-	publicKeyAuth, err := createPublicKeyAuth(identityFile, opts.SSHPassword)
-	if err == nil {
-		authMethods = append(authMethods, publicKeyAuth)
+	hostKeyCallback, err := createHostKeyCallback(opts)
+	if err != nil {
+		return nil, err
 	}
-	authMethods = append(authMethods, createKeyboardInteractiveAuth(opts.SSHPassword))
-	authMethods = append(authMethods, ssh.Password(opts.SSHPassword))
+	var authMethods []ssh.AuthMethod
+	publicKeySigner, err := createPublicKeyAuth(identityFile, opts.SSHPassword)
+	if err == nil {
+		authMethods = append(authMethods, ssh.PublicKeys(publicKeySigner))
+	}
+	authMethods = append(authMethods, ssh.RetryableAuthMethod(ssh.KeyboardInteractive(createCombinedKbdInteractiveChallenge(opts.SSHPassword)), 2))
+	authMethods = append(authMethods, ssh.RetryableAuthMethod(ssh.PasswordCallback(createCombinedPasswordCallbackPrompt(opts.SSHPassword)), 2))
 
 	configUser, _ := ssh_config.GetStrict(opts.SSHHost, "User")
 	configHostName, _ := ssh_config.GetStrict(opts.SSHHost, "HostName")

wavesrv/pkg/scpacket/scpacket.go

@@ -6,20 +6,84 @@ package scpacket
 import (
 	"fmt"
 	"reflect"
+	"regexp"
 	"strings"
 
 	"github.com/alessio/shellescape"
+	"github.com/google/uuid"
 	"github.com/wavetermdev/waveterm/waveshell/pkg/base"
 	"github.com/wavetermdev/waveterm/waveshell/pkg/packet"
 	"github.com/wavetermdev/waveterm/waveshell/pkg/utilfn"
-	"github.com/wavetermdev/waveterm/wavesrv/pkg/sstore"
 )
 
+var RemoteNameRe = regexp.MustCompile("^\\*?[a-zA-Z0-9_-]+$")
+
+type RemotePtrType struct {
+	OwnerId  string `json:"ownerid"`
+	RemoteId string `json:"remoteid"`
+	Name     string `json:"name"`
+}
+
+func (r RemotePtrType) IsSessionScope() bool {
+	return strings.HasPrefix(r.Name, "*")
+}
+
+func (rptr *RemotePtrType) GetDisplayName(baseDisplayName string) string {
+	name := baseDisplayName
+	if rptr == nil {
+		return name
+	}
+	if rptr.Name != "" {
+		name = name + ":" + rptr.Name
+	}
+	if rptr.OwnerId != "" {
+		name = "@" + rptr.OwnerId + ":" + name
+	}
+	return name
+}
+
+func (r RemotePtrType) Validate() error {
+	if r.OwnerId != "" {
+		if _, err := uuid.Parse(r.OwnerId); err != nil {
+			return fmt.Errorf("invalid ownerid format: %v", err)
+		}
+	}
+	if r.RemoteId != "" {
+		if _, err := uuid.Parse(r.RemoteId); err != nil {
+			return fmt.Errorf("invalid remoteid format: %v", err)
+		}
+	}
+	if r.Name != "" {
+		ok := RemoteNameRe.MatchString(r.Name)
+		if !ok {
+			return fmt.Errorf("invalid remote name")
+		}
+	}
+	return nil
+}
+
+func (r RemotePtrType) MakeFullRemoteRef() string {
+	if r.RemoteId == "" {
+		return ""
+	}
+	if r.OwnerId == "" && r.Name == "" {
+		return r.RemoteId
+	}
+	if r.OwnerId != "" && r.Name == "" {
+		return fmt.Sprintf("@%s:%s", r.OwnerId, r.RemoteId)
+	}
+	if r.OwnerId == "" && r.Name != "" {
+		return fmt.Sprintf("%s:%s", r.RemoteId, r.Name)
+	}
+	return fmt.Sprintf("@%s:%s:%s", r.OwnerId, r.RemoteId, r.Name)
+}
+
 const FeCommandPacketStr = "fecmd"
 const WatchScreenPacketStr = "watchscreen"
 const FeInputPacketStr = "feinput"
 const RemoteInputPacketStr = "remoteinput"
 const CmdInputTextPacketStr = "cmdinputtext"
+const UserInputResponsePacketStr = "userinputresp"
 
 type FeCommandPacketType struct {
 	Type        string            `json:"type"`
@@ -55,20 +119,20 @@ func (pk *FeCommandPacketType) GetRawStr() string {
 }
 
 type UIContextType struct {
-	SessionId string                `json:"sessionid"`
-	ScreenId  string                `json:"screenid"`
-	Remote    *sstore.RemotePtrType `json:"remote,omitempty"`
-	WinSize   *packet.WinSize       `json:"winsize,omitempty"`
-	Build     string                `json:"build,omitempty"`
+	SessionId string          `json:"sessionid"`
+	ScreenId  string          `json:"screenid"`
+	Remote    *RemotePtrType  `json:"remote,omitempty"`
+	WinSize   *packet.WinSize `json:"winsize,omitempty"`
+	Build     string          `json:"build,omitempty"`
 }
 
 type FeInputPacketType struct {
-	Type        string               `json:"type"`
-	CK          base.CommandKey      `json:"ck"`
-	Remote      sstore.RemotePtrType `json:"remote"`
-	InputData64 string               `json:"inputdata64"`
-	SigName     string               `json:"signame,omitempty"`
-	WinSize     *packet.WinSize      `json:"winsize,omitempty"`
+	Type        string          `json:"type"`
+	CK          base.CommandKey `json:"ck"`
+	Remote      RemotePtrType   `json:"remote"`
+	InputData64 string          `json:"inputdata64"`
+	SigName     string          `json:"signame,omitempty"`
+	WinSize     *packet.WinSize `json:"winsize,omitempty"`
 }
 
 type RemoteInputPacketType struct {
@@ -92,12 +156,21 @@ type CmdInputTextPacketType struct {
 	Text     utilfn.StrWithPos `json:"text"`
 }
 
+type UserInputResponsePacketType struct {
+	Type      string `json:"type"`
+	RequestId string `json:"requestid"`
+	Text      string `json:"text,omitempty"`
+	Confirm   bool   `json:"confirm,omitempty"`
+	ErrorMsg  string `json:"errormsg,omitempty"`
+}
+
 func init() {
 	packet.RegisterPacketType(FeCommandPacketStr, reflect.TypeOf(FeCommandPacketType{}))
 	packet.RegisterPacketType(WatchScreenPacketStr, reflect.TypeOf(WatchScreenPacketType{}))
 	packet.RegisterPacketType(FeInputPacketStr, reflect.TypeOf(FeInputPacketType{}))
 	packet.RegisterPacketType(RemoteInputPacketStr, reflect.TypeOf(RemoteInputPacketType{}))
 	packet.RegisterPacketType(CmdInputTextPacketStr, reflect.TypeOf(CmdInputTextPacketType{}))
+	packet.RegisterPacketType(UserInputResponsePacketStr, reflect.TypeOf(UserInputResponsePacketType{}))
 }
 
 func (*CmdInputTextPacketType) GetType() string {
@@ -139,3 +212,7 @@ func MakeRemoteInputPacket() *RemoteInputPacketType {
 func (*RemoteInputPacketType) GetType() string {
 	return RemoteInputPacketStr
 }
+
+func (*UserInputResponsePacketType) GetType() string {
+	return UserInputResponsePacketStr
+}

wavesrv/pkg/scws/scws.go

@@ -281,6 +281,18 @@ func (ws *WSState) processMessage(msgBytes []byte) error {
 		sstore.ScreenMemSetCmdInputText(cmdInputPk.ScreenId, cmdInputPk.Text, cmdInputPk.SeqNum)
 		return nil
 	}
+	if pk.GetType() == scpacket.UserInputResponsePacketStr {
+		userInputRespPk := pk.(*scpacket.UserInputResponsePacketType)
+		uich, ok := sstore.MainBus.GetUserInputChannel(userInputRespPk.RequestId)
+		if !ok {
+			return fmt.Errorf("received User Input Response with invalid Id (%s): %v\n", userInputRespPk.RequestId, err)
+		}
+		select {
+		case uich <- userInputRespPk:
+		default:
+		}
+		return nil
+	}
 	return fmt.Errorf("got ws bad message: %v", pk.GetType())
 }
 

wavesrv/pkg/sstore/dbops.go

@@ -950,6 +950,7 @@ func UpdateCmdDoneInfo(ctx context.Context, ck base.CommandKey, donePk *packet.C
 		// This is not a fatal error, so just log it
 		log.Printf("error setting status indicator level after done packet: %v\n", err)
 	}
+	IncrementNumRunningCmds_Update(update, screenId, -1)
 
 	return update, nil
 }