new wshrpc mechanism (#112)

lots of changes. new wshrpc implementation. unify websocket, web,
blockcontroller, domain sockets, and terminal inputs to all use the new
rpc system.

lots of moving files around to deal with circular dependencies

use new wshrpc as a client in wsh cmd

pkg/wshrpc/rpc_client.go

@@ -1,264 +0,0 @@
-// Copyright 2024, Command Line Inc.
-// SPDX-License-Identifier: Apache-2.0
-
-package wshprc
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"log"
-	"runtime/debug"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/google/uuid"
-)
-
-// there is a single go-routine that reads from RecvCh
-type RpcClient struct {
-	CVar               *sync.Cond
-	NextSeqNum         *atomic.Int64
-	ReqPacketsInFlight map[int64]string // seqnum -> rpcId
-	AckList            []int64
-	RpcReqs            map[string]*RpcInfo
-	SendCh             chan *RpcPacket
-	RecvCh             chan *RpcPacket
-}
-
-type RpcInfo struct {
-	CloseSync       *sync.Once
-	RpcId           string
-	PacketsInFlight map[int64]bool  // seqnum -> bool (for clients this is for requests, for servers it is for responses)
-	PkCh            chan *RpcPacket // for clients this is for responses, for servers it is for requests
-}
-
-func MakeRpcClient(sendCh chan *RpcPacket, recvCh chan *RpcPacket) *RpcClient {
-	if cap(sendCh) < MaxInFlightPackets {
-		panic(fmt.Errorf("sendCh buffer size must be at least MaxInFlightPackets(%d)", MaxInFlightPackets))
-	}
-	rtn := &RpcClient{
-		CVar:               sync.NewCond(&sync.Mutex{}),
-		NextSeqNum:         &atomic.Int64{},
-		ReqPacketsInFlight: make(map[int64]string),
-		AckList:            nil,
-		RpcReqs:            make(map[string]*RpcInfo),
-		SendCh:             sendCh,
-		RecvCh:             recvCh,
-	}
-	go rtn.runRecvLoop()
-	return rtn
-}
-
-func (c *RpcClient) runRecvLoop() {
-	defer func() {
-		if r := recover(); r != nil {
-			log.Printf("RpcClient.runRecvLoop() panic: %v", r)
-			debug.PrintStack()
-		}
-	}()
-	for pk := range c.RecvCh {
-		if pk.RpcType == RpcType_Resp {
-			c.handleResp(pk)
-			continue
-		}
-		log.Printf("RpcClient.runRecvLoop() bad packet type: %v", pk)
-	}
-	log.Printf("RpcClient.runRecvLoop() normal exit")
-}
-
-func (c *RpcClient) getRpcInfo(rpcId string) *RpcInfo {
-	c.CVar.L.Lock()
-	defer c.CVar.L.Unlock()
-	return c.RpcReqs[rpcId]
-}
-
-func (c *RpcClient) handleResp(pk *RpcPacket) {
-	c.handleAcks(pk.Acks)
-	if pk.RpcId == "" {
-		c.ackResp(pk.SeqNum)
-		log.Printf("RpcClient.handleResp() missing rpcId: %v", pk)
-		return
-	}
-	rpcInfo := c.getRpcInfo(pk.RpcId)
-	if rpcInfo == nil {
-		c.ackResp(pk.SeqNum)
-		log.Printf("RpcClient.handleResp() unknown rpcId: %v", pk)
-		return
-	}
-	select {
-	case rpcInfo.PkCh <- pk:
-	default:
-		log.Printf("RpcClient.handleResp() respCh full, dropping packet")
-	}
-	if pk.RespDone {
-		c.removeReqInfo(pk.RpcId, false)
-	}
-}
-
-func (c *RpcClient) grabAcks() []int64 {
-	c.CVar.L.Lock()
-	defer c.CVar.L.Unlock()
-	acks := c.AckList
-	c.AckList = nil
-	return acks
-}
-
-func (c *RpcClient) ackResp(seqNum int64) {
-	if seqNum == 0 {
-		return
-	}
-	c.CVar.L.Lock()
-	defer c.CVar.L.Unlock()
-	c.AckList = append(c.AckList, seqNum)
-}
-
-func (c *RpcClient) waitForReq(ctx context.Context, req *RpcPacket) (*RpcInfo, error) {
-	c.CVar.L.Lock()
-	defer c.CVar.L.Unlock()
-	// issue with ctx timeout sync -- we need the cvar to be signaled fairly regularly so we can check ctx.Err()
-	for {
-		if ctx.Err() != nil {
-			return nil, ctx.Err()
-		}
-		if len(c.RpcReqs) >= MaxOpenRpcs {
-			c.CVar.Wait()
-			continue
-		}
-		if len(c.ReqPacketsInFlight) >= MaxOpenRpcs {
-			c.CVar.Wait()
-			continue
-		}
-		if rpcInfo, ok := c.RpcReqs[req.RpcId]; ok {
-			if len(rpcInfo.PacketsInFlight) >= MaxUnackedPerRpc {
-				c.CVar.Wait()
-				continue
-			}
-		}
-		break
-	}
-	select {
-	case c.SendCh <- req:
-	default:
-		return nil, errors.New("SendCh Full")
-	}
-	c.ReqPacketsInFlight[req.SeqNum] = req.RpcId
-	rpcInfo := c.RpcReqs[req.RpcId]
-	if rpcInfo == nil {
-		rpcInfo = &RpcInfo{
-			CloseSync:       &sync.Once{},
-			RpcId:           req.RpcId,
-			PacketsInFlight: make(map[int64]bool),
-			PkCh:            make(chan *RpcPacket, MaxUnackedPerRpc),
-		}
-		rpcInfo.PacketsInFlight[req.SeqNum] = true
-		c.RpcReqs[req.RpcId] = rpcInfo
-	}
-	return rpcInfo, nil
-}
-
-func (c *RpcClient) handleAcks(acks []int64) {
-	if len(acks) == 0 {
-		return
-	}
-	c.CVar.L.Lock()
-	defer c.CVar.L.Unlock()
-	for _, ack := range acks {
-		rpcId, ok := c.ReqPacketsInFlight[ack]
-		if !ok {
-			continue
-		}
-		rpcInfo := c.RpcReqs[rpcId]
-		if rpcInfo != nil {
-			delete(rpcInfo.PacketsInFlight, ack)
-		}
-		delete(c.ReqPacketsInFlight, ack)
-	}
-	c.CVar.Broadcast()
-}
-
-func (c *RpcClient) removeReqInfo(rpcId string, clearSend bool) {
-	c.CVar.L.Lock()
-	defer c.CVar.L.Unlock()
-	rpcInfo := c.RpcReqs[rpcId]
-	delete(c.RpcReqs, rpcId)
-	if rpcInfo != nil {
-		if clearSend {
-			// unblock the recv loop if it happens to be waiting
-			// because the delete has already happens, it will not be able to send again on the channel
-			select {
-			case <-rpcInfo.PkCh:
-			default:
-			}
-		}
-		rpcInfo.CloseSync.Do(func() {
-			close(rpcInfo.PkCh)
-		})
-	}
-}
-
-func (c *RpcClient) SimpleReq(ctx context.Context, command string, data any) (any, error) {
-	rpcId := uuid.NewString()
-	seqNum := c.NextSeqNum.Add(1)
-	var timeoutInfo *TimeoutInfo
-	deadline, ok := ctx.Deadline()
-	if ok {
-		timeoutInfo = &TimeoutInfo{Deadline: deadline.UnixMilli()}
-	}
-	req := &RpcPacket{
-		Command: command,
-		RpcId:   rpcId,
-		RpcType: RpcType_Req,
-		SeqNum:  seqNum,
-		ReqDone: true,
-		Acks:    c.grabAcks(),
-		Timeout: timeoutInfo,
-		Data:    data,
-	}
-	rpcInfo, err := c.waitForReq(ctx, req)
-	if err != nil {
-		return nil, err
-	}
-	defer c.removeReqInfo(rpcId, true)
-	var rtnPacket *RpcPacket
-	select {
-	case <-ctx.Done():
-		return nil, ctx.Err()
-	case rtnPacket = <-rpcInfo.PkCh:
-		// fallthrough
-	}
-	if rtnPacket.Error != "" {
-		return nil, errors.New(rtnPacket.Error)
-	}
-	return rtnPacket.Data, nil
-}
-
-func (c *RpcClient) StreamReq(ctx context.Context, command string, data any, respTimeout time.Duration) (chan *RpcPacket, error) {
-	rpcId := uuid.NewString()
-	seqNum := c.NextSeqNum.Add(1)
-	var timeoutInfo *TimeoutInfo = &TimeoutInfo{RespPacketTimeout: respTimeout.Milliseconds()}
-	deadline, ok := ctx.Deadline()
-	if ok {
-		timeoutInfo.Deadline = deadline.UnixMilli()
-	}
-	req := &RpcPacket{
-		Command: command,
-		RpcId:   rpcId,
-		RpcType: RpcType_Req,
-		SeqNum:  seqNum,
-		ReqDone: true,
-		Acks:    c.grabAcks(),
-		Timeout: timeoutInfo,
-		Data:    data,
-	}
-	rpcInfo, err := c.waitForReq(ctx, req)
-	if err != nil {
-		return nil, err
-	}
-	return rpcInfo.PkCh, nil
-}
-
-func (c *RpcClient) EndStreamReq(rpcId string) {
-	c.removeReqInfo(rpcId, true)
-}

pkg/wshrpc/rpc_server.go

@@ -1,299 +0,0 @@
-// Copyright 2024, Command Line Inc.
-// SPDX-License-Identifier: Apache-2.0
-
-package wshprc
-
-import (
-	"context"
-	"fmt"
-	"log"
-	"runtime/debug"
-	"sync"
-	"sync/atomic"
-	"time"
-)
-
-type SimpleCommandHandlerFn func(context.Context, *RpcServer, string, any) (any, error)
-type StreamCommandHandlerFn func(context.Context, *RpcServer, *RpcPacket) error
-
-type RpcServer struct {
-	CVar                  *sync.Cond
-	NextSeqNum            *atomic.Int64
-	RespPacketsInFlight   map[int64]string // seqnum -> rpcId
-	AckList               []int64
-	RpcReqs               map[string]*RpcInfo
-	SendCh                chan *RpcPacket
-	RecvCh                chan *RpcPacket
-	SimpleCommandHandlers map[string]SimpleCommandHandlerFn
-	StreamCommandHandlers map[string]StreamCommandHandlerFn
-}
-
-func MakeRpcServer(sendCh chan *RpcPacket, recvCh chan *RpcPacket) *RpcServer {
-	if cap(sendCh) < MaxInFlightPackets {
-		panic(fmt.Errorf("sendCh buffer size must be at least MaxInFlightPackets(%d)", MaxInFlightPackets))
-	}
-	rtn := &RpcServer{
-		CVar:                  sync.NewCond(&sync.Mutex{}),
-		NextSeqNum:            &atomic.Int64{},
-		RespPacketsInFlight:   make(map[int64]string),
-		AckList:               nil,
-		RpcReqs:               make(map[string]*RpcInfo),
-		SendCh:                sendCh,
-		RecvCh:                recvCh,
-		SimpleCommandHandlers: make(map[string]SimpleCommandHandlerFn),
-		StreamCommandHandlers: make(map[string]StreamCommandHandlerFn),
-	}
-	go rtn.runRecvLoop()
-	return rtn
-}
-
-func (s *RpcServer) shouldUseStreamHandler(command string) bool {
-	s.CVar.L.Lock()
-	defer s.CVar.L.Unlock()
-	_, ok := s.StreamCommandHandlers[command]
-	return ok
-}
-
-func (s *RpcServer) getStreamHandler(command string) StreamCommandHandlerFn {
-	s.CVar.L.Lock()
-	defer s.CVar.L.Unlock()
-	return s.StreamCommandHandlers[command]
-}
-
-func (s *RpcServer) getSimpleHandler(command string) SimpleCommandHandlerFn {
-	s.CVar.L.Lock()
-	defer s.CVar.L.Unlock()
-	return s.SimpleCommandHandlers[command]
-}
-
-func (s *RpcServer) RegisterSimpleCommandHandler(command string, handler SimpleCommandHandlerFn) {
-	s.CVar.L.Lock()
-	defer s.CVar.L.Unlock()
-	if s.StreamCommandHandlers[command] != nil {
-		panic(fmt.Errorf("command %q already registered as a stream handler", command))
-	}
-	s.SimpleCommandHandlers[command] = handler
-}
-
-func (s *RpcServer) RegisterStreamCommandHandler(command string, handler StreamCommandHandlerFn) {
-	s.CVar.L.Lock()
-	defer s.CVar.L.Unlock()
-	if s.SimpleCommandHandlers[command] != nil {
-		panic(fmt.Errorf("command %q already registered as a simple handler", command))
-	}
-	s.StreamCommandHandlers[command] = handler
-}
-
-func (s *RpcServer) runRecvLoop() {
-	defer func() {
-		if r := recover(); r != nil {
-			log.Printf("RpcServer.runRecvLoop() panic: %v", r)
-			debug.PrintStack()
-		}
-	}()
-	for pk := range s.RecvCh {
-		s.handleAcks(pk.Acks)
-		if pk.RpcType == RpcType_Req {
-			if s.shouldUseStreamHandler(pk.Command) {
-				s.handleStreamReq(pk)
-			} else {
-				s.handleSimpleReq(pk)
-			}
-			continue
-		}
-		log.Printf("RpcClient.runRecvLoop() bad packet type: %v", pk)
-	}
-	log.Printf("RpcServer.runRecvLoop() normal exit")
-}
-
-func (s *RpcServer) ackResp(seqNum int64) {
-	if seqNum == 0 {
-		return
-	}
-	s.CVar.L.Lock()
-	defer s.CVar.L.Unlock()
-	s.AckList = append(s.AckList, seqNum)
-}
-
-func makeContextFromTimeout(timeout *TimeoutInfo) (context.Context, context.CancelFunc) {
-	if timeout == nil {
-		return context.Background(), func() {}
-	}
-	return context.WithDeadline(context.Background(), time.UnixMilli(timeout.Deadline))
-}
-
-func (s *RpcServer) SendResponse(ctx context.Context, pk *RpcPacket) error {
-	return s.waitForSend(ctx, pk)
-}
-
-func (s *RpcServer) waitForSend(ctx context.Context, pk *RpcPacket) error {
-	s.CVar.L.Lock()
-	defer s.CVar.L.Unlock()
-	for {
-		if ctx.Err() != nil {
-			return ctx.Err()
-		}
-		if len(s.RespPacketsInFlight) >= MaxInFlightPackets {
-			s.CVar.Wait()
-			continue
-		}
-		rpcInfo := s.RpcReqs[pk.RpcId]
-		if rpcInfo != nil {
-			if len(rpcInfo.PacketsInFlight) >= MaxUnackedPerRpc {
-				s.CVar.Wait()
-				continue
-			}
-		}
-		break
-	}
-	s.RespPacketsInFlight[pk.SeqNum] = pk.RpcId
-	pk.Acks = s.grabAcks_nolock()
-	s.SendCh <- pk
-	rpcInfo := s.RpcReqs[pk.RpcId]
-	if !pk.RespDone && rpcInfo != nil {
-		rpcInfo = &RpcInfo{
-			CloseSync:       &sync.Once{},
-			RpcId:           pk.RpcId,
-			PkCh:            make(chan *RpcPacket, MaxUnackedPerRpc),
-			PacketsInFlight: make(map[int64]bool),
-		}
-		s.RpcReqs[pk.RpcId] = rpcInfo
-	}
-	if rpcInfo != nil {
-		rpcInfo.PacketsInFlight[pk.SeqNum] = true
-	}
-	if pk.RespDone {
-		delete(s.RpcReqs, pk.RpcId)
-	}
-	return nil
-}
-
-func (s *RpcServer) handleAcks(acks []int64) {
-	if len(acks) == 0 {
-		return
-	}
-	s.CVar.L.Lock()
-	defer s.CVar.L.Unlock()
-	for _, ack := range acks {
-		rpcId, ok := s.RespPacketsInFlight[ack]
-		if !ok {
-			continue
-		}
-		rpcInfo := s.RpcReqs[rpcId]
-		if rpcInfo != nil {
-			delete(rpcInfo.PacketsInFlight, ack)
-		}
-		delete(s.RespPacketsInFlight, ack)
-	}
-	s.CVar.Broadcast()
-}
-
-func (s *RpcServer) handleSimpleReq(pk *RpcPacket) {
-	s.ackResp(pk.SeqNum)
-	handler := s.getSimpleHandler(pk.Command)
-	if handler == nil {
-		s.sendErrorResp(pk, fmt.Errorf("unknown command: %s", pk.Command))
-		log.Printf("RpcServer.handleReq() unknown command: %s", pk.Command)
-		return
-	}
-	go func() {
-		defer func() {
-			if r := recover(); r != nil {
-				log.Printf("RpcServer.handleReq(%q) panic: %v", pk.Command, r)
-				debug.PrintStack()
-			}
-		}()
-		ctx, cancelFn := makeContextFromTimeout(pk.Timeout)
-		defer cancelFn()
-		data, err := handler(ctx, s, pk.Command, pk.Data)
-		seqNum := s.NextSeqNum.Add(1)
-		respPk := &RpcPacket{
-			Command:  pk.Command,
-			RpcId:    pk.RpcId,
-			RpcType:  RpcType_Resp,
-			SeqNum:   seqNum,
-			RespDone: true,
-		}
-		if err != nil {
-			respPk.Error = err.Error()
-		} else {
-			respPk.Data = data
-		}
-		s.waitForSend(ctx, respPk)
-	}()
-}
-
-func (s *RpcServer) grabAcks_nolock() []int64 {
-	acks := s.AckList
-	s.AckList = nil
-	return acks
-}
-
-func (s *RpcServer) sendErrorResp(pk *RpcPacket, err error) {
-	respPk := &RpcPacket{
-		Command:  pk.Command,
-		RpcId:    pk.RpcId,
-		RpcType:  RpcType_Resp,
-		SeqNum:   s.NextSeqNum.Add(1),
-		RespDone: true,
-		Error:    err.Error(),
-	}
-	s.waitForSend(context.Background(), respPk)
-}
-
-func (s *RpcServer) makeRespPk(pk *RpcPacket, data any, done bool) *RpcPacket {
-	return &RpcPacket{
-		Command:  pk.Command,
-		RpcId:    pk.RpcId,
-		RpcType:  RpcType_Resp,
-		SeqNum:   s.NextSeqNum.Add(1),
-		RespDone: done,
-		Data:     data,
-	}
-}
-
-func (s *RpcServer) handleStreamReq(pk *RpcPacket) {
-	s.ackResp(pk.SeqNum)
-	handler := s.getStreamHandler(pk.Command)
-	if handler == nil {
-		s.ackResp(pk.SeqNum)
-		s.sendErrorResp(pk, fmt.Errorf("unknown command: %s", pk.Command))
-		log.Printf("RpcServer.handleStreamReq() unknown command: %s", pk.Command)
-		return
-	}
-	go func() {
-		defer func() {
-			r := recover()
-			if r == nil {
-				return
-			}
-			log.Printf("RpcServer.handleStreamReq(%q) panic: %v", pk.Command, r)
-			debug.PrintStack()
-			respPk := &RpcPacket{
-				Command:  pk.Command,
-				RpcId:    pk.RpcId,
-				RpcType:  RpcType_Resp,
-				SeqNum:   s.NextSeqNum.Add(1),
-				RespDone: true,
-				Error:    fmt.Sprintf("panic: %v", r),
-			}
-			s.waitForSend(context.Background(), respPk)
-		}()
-		ctx, cancelFn := makeContextFromTimeout(pk.Timeout)
-		defer cancelFn()
-		err := handler(ctx, s, pk)
-		if err != nil {
-			respPk := &RpcPacket{
-				Command:  pk.Command,
-				RpcId:    pk.RpcId,
-				RpcType:  RpcType_Resp,
-				SeqNum:   s.NextSeqNum.Add(1),
-				RespDone: true,
-				Error:    err.Error(),
-			}
-			s.waitForSend(ctx, respPk)
-			return
-		}
-		// check if RespDone has been set, if not, send it here
-	}()
-}

pkg/wshrpc/rpc_test.go

@@ -1,194 +0,0 @@
-// Copyright 2024, Command Line Inc.
-// SPDX-License-Identifier: Apache-2.0
-
-package wshprc
-
-import (
-	"context"
-	"fmt"
-	"log"
-	"sync"
-	"testing"
-	"time"
-)
-
-func TestSimple(t *testing.T) {
-	sendCh := make(chan *RpcPacket, MaxInFlightPackets)
-	recvCh := make(chan *RpcPacket, MaxInFlightPackets)
-	client := MakeRpcClient(sendCh, recvCh)
-	ctx, cancelFn := context.WithTimeout(context.Background(), 2*time.Second)
-	defer cancelFn()
-	var wg sync.WaitGroup
-	wg.Add(2)
-	go func() {
-		defer wg.Done()
-		resp, err := client.SimpleReq(ctx, "test", "hello")
-		if err != nil {
-			t.Errorf("SimpleReq() failed: %v", err)
-			return
-		}
-		if resp != "world" {
-			t.Errorf("SimpleReq() failed: expected 'world', got '%s'", resp)
-		}
-	}()
-	go func() {
-		defer wg.Done()
-		req := <-sendCh
-		if req.Command != "test" {
-			t.Errorf("expected 'test', got '%s'", req.Command)
-		}
-		if req.Data != "hello" {
-			t.Errorf("expected 'hello', got '%s'", req.Data)
-		}
-		resp := &RpcPacket{
-			Command:  "test",
-			RpcId:    req.RpcId,
-			RpcType:  RpcType_Resp,
-			SeqNum:   1,
-			RespDone: true,
-			Acks:     []int64{req.SeqNum},
-			Data:     "world",
-		}
-		recvCh <- resp
-	}()
-	wg.Wait()
-}
-
-func makeRpcResp(req *RpcPacket, data any, seqNum int64, done bool) *RpcPacket {
-	return &RpcPacket{
-		Command:  req.Command,
-		RpcId:    req.RpcId,
-		RpcType:  RpcType_Resp,
-		SeqNum:   seqNum,
-		RespDone: done,
-		Data:     data,
-	}
-}
-
-func TestStream(t *testing.T) {
-	sendCh := make(chan *RpcPacket, MaxInFlightPackets)
-	recvCh := make(chan *RpcPacket, MaxInFlightPackets)
-	client := MakeRpcClient(sendCh, recvCh)
-	ctx, cancelFn := context.WithTimeout(context.Background(), 2*time.Second)
-	defer cancelFn()
-	var wg sync.WaitGroup
-	wg.Add(2)
-	go func() {
-		defer wg.Done()
-		respCh, err := client.StreamReq(ctx, "test", "hello", 1000)
-		if err != nil {
-			t.Errorf("StreamReq() failed: %v", err)
-			return
-		}
-		var output []string
-		for resp := range respCh {
-			if resp.Error != "" {
-				t.Errorf("StreamReq() failed: %v", resp.Error)
-				return
-			}
-			output = append(output, resp.Data.(string))
-		}
-		if len(output) != 3 {
-			t.Errorf("expected 3 responses, got %d (%v)", len(output), output)
-			return
-		}
-		if output[0] != "one" || output[1] != "two" || output[2] != "three" {
-			t.Errorf("expected 'one', 'two', 'three', got %v", output)
-			return
-		}
-	}()
-	go func() {
-		defer wg.Done()
-		req := <-sendCh
-		if req.Command != "test" {
-			t.Errorf("expected 'test', got '%s'", req.Command)
-		}
-		if req.Data != "hello" {
-			t.Errorf("expected 'hello', got '%s'", req.Data)
-		}
-		resp := makeRpcResp(req, "one", 1, false)
-		recvCh <- resp
-		resp = makeRpcResp(req, "two", 2, false)
-		recvCh <- resp
-		resp = makeRpcResp(req, "three", 3, true)
-		recvCh <- resp
-	}()
-	wg.Wait()
-}
-
-func TestSimpleClientServer(t *testing.T) {
-	sendCh := make(chan *RpcPacket, MaxInFlightPackets)
-	recvCh := make(chan *RpcPacket, MaxInFlightPackets)
-	client := MakeRpcClient(sendCh, recvCh)
-	server := MakeRpcServer(recvCh, sendCh)
-	ctx, cancelFn := context.WithTimeout(context.Background(), 2*time.Second)
-	defer cancelFn()
-	server.RegisterSimpleCommandHandler("test", func(ctx context.Context, s *RpcServer, cmd string, data any) (any, error) {
-		if data != "hello" {
-			return nil, fmt.Errorf("expected 'hello', got '%s'", data)
-		}
-		return "world", nil
-	})
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		resp, err := client.SimpleReq(ctx, "test", "hello")
-		if err != nil {
-			t.Errorf("SimpleReq() failed: %v", err)
-			return
-		}
-		if resp != "world" {
-			t.Errorf("SimpleReq() failed: expected 'world', got '%s'", resp)
-		}
-	}()
-	wg.Wait()
-
-}
-
-func TestStreamClientServer(t *testing.T) {
-	sendCh := make(chan *RpcPacket, MaxInFlightPackets)
-	recvCh := make(chan *RpcPacket, MaxInFlightPackets)
-	client := MakeRpcClient(sendCh, recvCh)
-	server := MakeRpcServer(recvCh, sendCh)
-	ctx, cancelFn := context.WithTimeout(context.Background(), 2*time.Second)
-	defer cancelFn()
-	server.RegisterStreamCommandHandler("test", func(ctx context.Context, s *RpcServer, req *RpcPacket) error {
-		pk1 := s.makeRespPk(req, "one", false)
-		pk2 := s.makeRespPk(req, "two", false)
-		pk3 := s.makeRespPk(req, "three", true)
-		s.SendResponse(ctx, pk1)
-		s.SendResponse(ctx, pk2)
-		s.SendResponse(ctx, pk3)
-		return nil
-	})
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		respCh, err := client.StreamReq(ctx, "test", "hello", 2*time.Second)
-		if err != nil {
-			t.Errorf("StreamReq() failed: %v", err)
-			return
-		}
-		var result []string
-		for respPk := range respCh {
-			if respPk.Error != "" {
-				t.Errorf("StreamReq() failed: %v", respPk.Error)
-				return
-			}
-			log.Printf("got response: %#v", respPk)
-			result = append(result, respPk.Data.(string))
-		}
-		if len(result) != 3 {
-			t.Errorf("expected 3 responses, got %d", len(result))
-			return
-		}
-		if result[0] != "one" || result[1] != "two" || result[2] != "three" {
-			t.Errorf("expected 'one', 'two', 'three', got %v", result)
-			return
-		}
-	}()
-	wg.Wait()
-
-}

pkg/wshrpc/wshclient/wshclient.go

@@ -0,0 +1,74 @@
+// Copyright 2024, Command Line Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+// generated by cmd/generatewshclient/main-generatewshclient.go
+
+package wshclient
+
+import (
+	"github.com/wavetermdev/thenextwave/pkg/wshutil"
+	"github.com/wavetermdev/thenextwave/pkg/wshrpc"
+	"github.com/wavetermdev/thenextwave/pkg/waveobj"
+)
+
+// command "controller:input", wshserver.BlockInputCommand
+func BlockInputCommand(w *wshutil.WshRpc, data wshrpc.CommandBlockInputData, opts *wshrpc.WshRpcCommandOpts) error {
+    _, err := sendRpcRequestHelper[any](w, "controller:input", data, opts)
+    return err
+}
+
+// command "controller:restart", wshserver.BlockRestartCommand
+func BlockRestartCommand(w *wshutil.WshRpc, data wshrpc.CommandBlockRestartData, opts *wshrpc.WshRpcCommandOpts) error {
+    _, err := sendRpcRequestHelper[any](w, "controller:restart", data, opts)
+    return err
+}
+
+// command "createblock", wshserver.CreateBlockCommand
+func CreateBlockCommand(w *wshutil.WshRpc, data wshrpc.CommandCreateBlockData, opts *wshrpc.WshRpcCommandOpts) (*waveobj.ORef, error) {
+    resp, err := sendRpcRequestHelper[*waveobj.ORef](w, "createblock", data, opts)
+    return resp, err
+}
+
+// command "file:append", wshserver.AppendFileCommand
+func AppendFileCommand(w *wshutil.WshRpc, data wshrpc.CommandAppendFileData, opts *wshrpc.WshRpcCommandOpts) error {
+    _, err := sendRpcRequestHelper[any](w, "file:append", data, opts)
+    return err
+}
+
+// command "file:appendijson", wshserver.AppendIJsonCommand
+func AppendIJsonCommand(w *wshutil.WshRpc, data wshrpc.CommandAppendIJsonData, opts *wshrpc.WshRpcCommandOpts) error {
+    _, err := sendRpcRequestHelper[any](w, "file:appendijson", data, opts)
+    return err
+}
+
+// command "getmeta", wshserver.GetMetaCommand
+func GetMetaCommand(w *wshutil.WshRpc, data wshrpc.CommandGetMetaData, opts *wshrpc.WshRpcCommandOpts) (map[string]interface {}, error) {
+    resp, err := sendRpcRequestHelper[map[string]interface {}](w, "getmeta", data, opts)
+    return resp, err
+}
+
+// command "message", wshserver.MessageCommand
+func MessageCommand(w *wshutil.WshRpc, data wshrpc.CommandMessageData, opts *wshrpc.WshRpcCommandOpts) error {
+    _, err := sendRpcRequestHelper[any](w, "message", data, opts)
+    return err
+}
+
+// command "resolveids", wshserver.ResolveIdsCommand
+func ResolveIdsCommand(w *wshutil.WshRpc, data wshrpc.CommandResolveIdsData, opts *wshrpc.WshRpcCommandOpts) (wshrpc.CommandResolveIdsRtnData, error) {
+    resp, err := sendRpcRequestHelper[wshrpc.CommandResolveIdsRtnData](w, "resolveids", data, opts)
+    return resp, err
+}
+
+// command "setmeta", wshserver.SetMetaCommand
+func SetMetaCommand(w *wshutil.WshRpc, data wshrpc.CommandSetMetaData, opts *wshrpc.WshRpcCommandOpts) error {
+    _, err := sendRpcRequestHelper[any](w, "setmeta", data, opts)
+    return err
+}
+
+// command "setview", wshserver.BlockSetViewCommand
+func BlockSetViewCommand(w *wshutil.WshRpc, data wshrpc.CommandBlockSetViewData, opts *wshrpc.WshRpcCommandOpts) error {
+    _, err := sendRpcRequestHelper[any](w, "setview", data, opts)
+    return err
+}
+
+

pkg/wshrpc/wshclient/wshclientutil.go

@@ -0,0 +1,30 @@
+// Copyright 2024, Command Line Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package wshclient
+
+import (
+	"github.com/wavetermdev/thenextwave/pkg/util/utilfn"
+	"github.com/wavetermdev/thenextwave/pkg/wshrpc"
+	"github.com/wavetermdev/thenextwave/pkg/wshutil"
+)
+
+func sendRpcRequestHelper[T any](w *wshutil.WshRpc, command string, data interface{}, opts *wshrpc.WshRpcCommandOpts) (T, error) {
+	var respData T
+	if opts.NoResponse {
+		err := w.SendCommand(command, data)
+		if err != nil {
+			return respData, err
+		}
+		return respData, nil
+	}
+	resp, err := w.SendRpcRequest(command, data, opts.Timeout)
+	if err != nil {
+		return respData, err
+	}
+	err = utilfn.ReUnmarshal(&respData, resp)
+	if err != nil {
+		return respData, err
+	}
+	return respData, nil
+}

pkg/wshrpc/wshrpc.go

@@ -1,58 +0,0 @@
-// Copyright 2024, Command Line Inc.
-// SPDX-License-Identifier: Apache-2.0
-
-package wshprc
-
-import (
-	"context"
-)
-
-const (
-	MaxOpenRpcs        = 10
-	MaxUnackedPerRpc   = 10
-	MaxInFlightPackets = MaxOpenRpcs * MaxUnackedPerRpc
-)
-
-const (
-	RpcType_Req  = "req"
-	RpcType_Resp = "resp"
-)
-
-const (
-	CommandType_Ack     = ":ack"
-	CommandType_Ping    = ":ping"
-	CommandType_Cancel  = ":cancel"
-	CommandType_Timeout = ":timeout"
-)
-
-var rpcClientContextKey = struct{}{}
-
-type TimeoutInfo struct {
-	Deadline          int64 `json:"deadline,omitempty"`
-	ReqPacketTimeout  int64 `json:"reqpackettimeout,omitempty"`  // for streaming requests
-	RespPacketTimeout int64 `json:"resppackettimeout,omitempty"` // for streaming responses
-}
-
-type RpcPacket struct {
-	Command  string       `json:"command"`
-	RpcId    string       `json:"rpcid"`
-	RpcType  string       `json:"rpctype"`
-	SeqNum   int64        `json:"seqnum"`
-	ReqDone  bool         `json:"reqdone"`
-	RespDone bool         `json:"resdone"`
-	Acks     []int64      `json:"acks,omitempty"`    // seqnums acked
-	Timeout  *TimeoutInfo `json:"timeout,omitempty"` // for initial request only
-	Data     any          `json:"data"`              // json data for command
-	Error    string       `json:"error,omitempty"`
-}
-
-func GetRpcClient(ctx context.Context) *RpcClient {
-	if ctx == nil {
-		return nil
-	}
-	val := ctx.Value(rpcClientContextKey)
-	if val == nil {
-		return nil
-	}
-	return val.(*RpcClient)
-}

pkg/wshrpc/wshrpctypes.go

@@ -0,0 +1,130 @@
+// Copyright 2024, Command Line Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+// types and methods for wsh rpc calls
+package wshrpc
+
+import (
+	"reflect"
+
+	"github.com/wavetermdev/thenextwave/pkg/ijson"
+	"github.com/wavetermdev/thenextwave/pkg/shellexec"
+	"github.com/wavetermdev/thenextwave/pkg/waveobj"
+	"github.com/wavetermdev/thenextwave/pkg/wshutil"
+	"github.com/wavetermdev/thenextwave/pkg/wstore"
+)
+
+const (
+	Command_Message     = "message"
+	Command_SetView     = "setview"
+	Command_SetMeta     = "setmeta"
+	Command_GetMeta     = "getmeta"
+	Command_BlockInput  = "controller:input"
+	Command_Restart     = "controller:restart"
+	Command_AppendFile  = "file:append"
+	Command_AppendIJson = "file:appendijson"
+	Command_ResolveIds  = "resolveids"
+	Command_CreateBlock = "createblock"
+)
+
+type MetaDataType = map[string]any
+
+var DataTypeMap = map[string]reflect.Type{
+	"meta":          reflect.TypeOf(MetaDataType{}),
+	"resolveidsrtn": reflect.TypeOf(CommandResolveIdsRtnData{}),
+	"oref":          reflect.TypeOf(waveobj.ORef{}),
+}
+
+// for frontend
+type WshServerCommandMeta struct {
+	CommandType string `json:"commandtype"`
+}
+
+type WshRpcCommandOpts struct {
+	Timeout    int  `json:"timeout"`
+	NoResponse bool `json:"noresponse"`
+}
+
+func HackRpcContextIntoData(dataPtr any, rpcContext wshutil.RpcContext) {
+	dataVal := reflect.ValueOf(dataPtr).Elem()
+	dataType := dataVal.Type()
+	for i := 0; i < dataVal.NumField(); i++ {
+		field := dataVal.Field(i)
+		if !field.IsZero() {
+			continue
+		}
+		fieldType := dataType.Field(i)
+		tag := fieldType.Tag.Get("wshcontext")
+		if tag == "" {
+			continue
+		}
+		switch tag {
+		case "BlockId":
+			field.SetString(rpcContext.BlockId)
+		case "TabId":
+			field.SetString(rpcContext.TabId)
+		case "WindowId":
+			field.SetString(rpcContext.WindowId)
+		case "BlockORef":
+			if rpcContext.BlockId != "" {
+				field.Set(reflect.ValueOf(waveobj.MakeORef(wstore.OType_Block, rpcContext.BlockId)))
+			}
+		}
+	}
+}
+
+type CommandMessageData struct {
+	ORef    waveobj.ORef `json:"oref" wshcontext:"BlockORef"`
+	Message string       `json:"message"`
+}
+
+type CommandGetMetaData struct {
+	ORef waveobj.ORef `json:"oref" wshcontext:"BlockORef"`
+}
+
+type CommandSetMetaData struct {
+	ORef waveobj.ORef `json:"oref" wshcontext:"BlockORef"`
+	Meta MetaDataType `json:"meta"`
+}
+
+type CommandResolveIdsData struct {
+	Ids []string `json:"ids"`
+}
+
+type CommandResolveIdsRtnData struct {
+	ResolvedIds map[string]waveobj.ORef `json:"resolvedids"`
+}
+
+type CommandCreateBlockData struct {
+	TabId    string              `json:"tabid" wshcontext:"TabId"`
+	BlockDef *wstore.BlockDef    `json:"blockdef"`
+	RtOpts   *wstore.RuntimeOpts `json:"rtopts"`
+}
+
+type CommandBlockSetViewData struct {
+	BlockId string `json:"blockid" wshcontext:"BlockId"`
+	View    string `json:"view"`
+}
+
+type CommandBlockRestartData struct {
+	BlockId string `json:"blockid" wshcontext:"BlockId"`
+}
+
+type CommandBlockInputData struct {
+	BlockId     string              `json:"blockid" wshcontext:"BlockId"`
+	InputData64 string              `json:"inputdata64,omitempty"`
+	SigName     string              `json:"signame,omitempty"`
+	TermSize    *shellexec.TermSize `json:"termsize,omitempty"`
+}
+
+type CommandAppendFileData struct {
+	ZoneId   string `json:"zoneid" wshcontext:"BlockId"`
+	FileName string `json:"filename"`
+	Data64   string `json:"data64"`
+}
+
+type CommandAppendIJsonData struct {
+	ZoneId   string        `json:"zoneid" wshcontext:"BlockId"`
+	FileName string        `json:"filename"`
+	Data     ijson.Command `json:"data"`
+}

pkg/wshrpc/wshserver/wshserver.go

@@ -0,0 +1,417 @@
+// Copyright 2024, Command Line Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package wshserver
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"io/fs"
+	"log"
+	"net"
+	"os"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/wavetermdev/thenextwave/pkg/blockcontroller"
+	"github.com/wavetermdev/thenextwave/pkg/eventbus"
+	"github.com/wavetermdev/thenextwave/pkg/filestore"
+	"github.com/wavetermdev/thenextwave/pkg/util/utilfn"
+	"github.com/wavetermdev/thenextwave/pkg/wavebase"
+	"github.com/wavetermdev/thenextwave/pkg/waveobj"
+	"github.com/wavetermdev/thenextwave/pkg/wshrpc"
+	"github.com/wavetermdev/thenextwave/pkg/wshutil"
+	"github.com/wavetermdev/thenextwave/pkg/wstore"
+)
+
+const (
+	DefaultOutputChSize = 32
+	DefaultInputChSize  = 32
+)
+
+type WshServer struct{}
+
+var WshServerImpl = WshServer{}
+var contextRType = reflect.TypeOf((*context.Context)(nil)).Elem()
+
+type WshServerMethodDecl struct {
+	Command                 string
+	CommandType             string
+	MethodName              string
+	Method                  reflect.Value
+	CommandDataType         reflect.Type
+	DefaultResponseDataType reflect.Type
+	RequestDataTypes        []reflect.Type // for streaming requests
+	ResponseDataTypes       []reflect.Type // for streaming responses
+}
+
+var WshServerCommandToDeclMap = map[string]*WshServerMethodDecl{
+	wshrpc.Command_Message:     GetWshServerMethod(wshrpc.Command_Message, wshutil.RpcType_Call, "MessageCommand", WshServerImpl.MessageCommand),
+	wshrpc.Command_SetView:     GetWshServerMethod(wshrpc.Command_SetView, wshutil.RpcType_Call, "BlockSetViewCommand", WshServerImpl.BlockSetViewCommand),
+	wshrpc.Command_SetMeta:     GetWshServerMethod(wshrpc.Command_SetMeta, wshutil.RpcType_Call, "SetMetaCommand", WshServerImpl.SetMetaCommand),
+	wshrpc.Command_GetMeta:     GetWshServerMethod(wshrpc.Command_GetMeta, wshutil.RpcType_Call, "GetMetaCommand", WshServerImpl.GetMetaCommand),
+	wshrpc.Command_ResolveIds:  GetWshServerMethod(wshrpc.Command_ResolveIds, wshutil.RpcType_Call, "ResolveIdsCommand", WshServerImpl.ResolveIdsCommand),
+	wshrpc.Command_CreateBlock: GetWshServerMethod(wshrpc.Command_CreateBlock, wshutil.RpcType_Call, "CreateBlockCommand", WshServerImpl.CreateBlockCommand),
+	wshrpc.Command_Restart:     GetWshServerMethod(wshrpc.Command_Restart, wshutil.RpcType_Call, "BlockRestartCommand", WshServerImpl.BlockRestartCommand),
+	wshrpc.Command_BlockInput:  GetWshServerMethod(wshrpc.Command_BlockInput, wshutil.RpcType_Call, "BlockInputCommand", WshServerImpl.BlockInputCommand),
+	wshrpc.Command_AppendFile:  GetWshServerMethod(wshrpc.Command_AppendFile, wshutil.RpcType_Call, "AppendFileCommand", WshServerImpl.AppendFileCommand),
+	wshrpc.Command_AppendIJson: GetWshServerMethod(wshrpc.Command_AppendIJson, wshutil.RpcType_Call, "AppendIJsonCommand", WshServerImpl.AppendIJsonCommand),
+}
+
+func GetWshServerMethod(command string, commandType string, methodName string, methodFunc any) *WshServerMethodDecl {
+	methodVal := reflect.ValueOf(methodFunc)
+	methodType := methodVal.Type()
+	if methodType.Kind() != reflect.Func {
+		panic(fmt.Sprintf("methodVal must be a function got [%v]", methodType))
+	}
+	if methodType.In(0) != contextRType {
+		panic(fmt.Sprintf("methodVal must have a context as the first argument %v", methodType))
+	}
+	var defResponseType reflect.Type
+	if methodType.NumOut() > 1 {
+		defResponseType = methodType.Out(0)
+	}
+	rtn := &WshServerMethodDecl{
+		Command:                 command,
+		CommandType:             commandType,
+		MethodName:              methodName,
+		Method:                  methodVal,
+		CommandDataType:         methodType.In(1),
+		DefaultResponseDataType: defResponseType,
+	}
+	return rtn
+}
+
+func (ws *WshServer) MessageCommand(ctx context.Context, data wshrpc.CommandMessageData) error {
+	log.Printf("MESSAGE: %s | %q\n", data.ORef, data.Message)
+	return nil
+}
+
+func (ws *WshServer) GetMetaCommand(ctx context.Context, data wshrpc.CommandGetMetaData) (wshrpc.MetaDataType, error) {
+	log.Printf("calling meta: %s\n", data.ORef)
+	obj, err := wstore.DBGetORef(ctx, data.ORef)
+	if err != nil {
+		return nil, fmt.Errorf("error getting object: %w", err)
+	}
+	if obj == nil {
+		return nil, fmt.Errorf("object not found: %s", data.ORef)
+	}
+	return waveobj.GetMeta(obj), nil
+}
+
+func (ws *WshServer) SetMetaCommand(ctx context.Context, data wshrpc.CommandSetMetaData) error {
+	oref := data.ORef
+	if oref.IsEmpty() {
+		return fmt.Errorf("no oref")
+	}
+	log.Printf("SETMETA: %s | %v\n", oref, data.Meta)
+	obj, err := wstore.DBGetORef(ctx, oref)
+	if err != nil {
+		return fmt.Errorf("error getting object: %w", err)
+	}
+	if obj == nil {
+		return nil
+	}
+	meta := waveobj.GetMeta(obj)
+	if meta == nil {
+		meta = make(map[string]any)
+	}
+	for k, v := range data.Meta {
+		if v == nil {
+			delete(meta, k)
+			continue
+		}
+		meta[k] = v
+	}
+	waveobj.SetMeta(obj, meta)
+	err = wstore.DBUpdate(ctx, obj)
+	if err != nil {
+		return fmt.Errorf("error updating block: %w", err)
+	}
+	sendWaveObjUpdate(oref)
+	return nil
+}
+
+func sendWaveObjUpdate(oref waveobj.ORef) {
+	ctx, cancelFn := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancelFn()
+	// send a waveobj:update event
+	waveObj, err := wstore.DBGetORef(ctx, oref)
+	if err != nil {
+		log.Printf("error getting object for update event: %v", err)
+		return
+	}
+	eventbus.SendEvent(eventbus.WSEventType{
+		EventType: eventbus.WSEvent_WaveObjUpdate,
+		ORef:      oref.String(),
+		Data: wstore.WaveObjUpdate{
+			UpdateType: wstore.UpdateType_Update,
+			OType:      waveObj.GetOType(),
+			OID:        waveobj.GetOID(waveObj),
+			Obj:        waveObj,
+		},
+	})
+}
+
+func resolveSimpleId(ctx context.Context, simpleId string) (*waveobj.ORef, error) {
+	if strings.Contains(simpleId, ":") {
+		rtn, err := waveobj.ParseORef(simpleId)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing simple id: %w", err)
+		}
+		return &rtn, nil
+	}
+	return wstore.DBResolveEasyOID(ctx, simpleId)
+}
+
+func (ws *WshServer) ResolveIdsCommand(ctx context.Context, data wshrpc.CommandResolveIdsData) (wshrpc.CommandResolveIdsRtnData, error) {
+	rtn := wshrpc.CommandResolveIdsRtnData{}
+	rtn.ResolvedIds = make(map[string]waveobj.ORef)
+	for _, simpleId := range data.Ids {
+		oref, err := resolveSimpleId(ctx, simpleId)
+		if err != nil || oref == nil {
+			continue
+		}
+		rtn.ResolvedIds[simpleId] = *oref
+	}
+	return rtn, nil
+}
+
+func sendWStoreUpdatesToEventBus(updates wstore.UpdatesRtnType) {
+	for _, update := range updates {
+		eventbus.SendEvent(eventbus.WSEventType{
+			EventType: eventbus.WSEvent_WaveObjUpdate,
+			ORef:      waveobj.MakeORef(update.OType, update.OID).String(),
+			Data:      update,
+		})
+	}
+}
+
+func (ws *WshServer) CreateBlockCommand(ctx context.Context, data wshrpc.CommandCreateBlockData) (*waveobj.ORef, error) {
+	ctx = wstore.ContextWithUpdates(ctx)
+	tabId := data.TabId
+	if data.TabId != "" {
+		tabId = data.TabId
+	}
+	blockData, err := wstore.CreateBlock(ctx, tabId, data.BlockDef, data.RtOpts)
+	if err != nil {
+		return nil, fmt.Errorf("error creating block: %w", err)
+	}
+	if blockData.Controller != "" {
+		// TODO
+		err = blockcontroller.StartBlockController(ctx, data.TabId, blockData.OID)
+		if err != nil {
+			return nil, fmt.Errorf("error starting block controller: %w", err)
+		}
+	}
+	updates := wstore.ContextGetUpdatesRtn(ctx)
+	sendWStoreUpdatesToEventBus(updates)
+	windowId, err := wstore.DBFindWindowForTabId(ctx, tabId)
+	if err != nil {
+		return nil, fmt.Errorf("error finding window for tab: %w", err)
+	}
+	if windowId == "" {
+		return nil, fmt.Errorf("no window found for tab")
+	}
+	eventbus.SendEventToWindow(windowId, eventbus.WSEventType{
+		EventType: eventbus.WSEvent_LayoutAction,
+		Data: &eventbus.WSLayoutActionData{
+			ActionType: "insert",
+			TabId:      tabId,
+			BlockId:    blockData.OID,
+		},
+	})
+	return &waveobj.ORef{OType: wstore.OType_Block, OID: blockData.OID}, nil
+}
+
+func (ws *WshServer) BlockSetViewCommand(ctx context.Context, data wshrpc.CommandBlockSetViewData) error {
+	log.Printf("SETVIEW: %s | %q\n", data.BlockId, data.View)
+	ctx = wstore.ContextWithUpdates(ctx)
+	block, err := wstore.DBGet[*wstore.Block](ctx, data.BlockId)
+	if err != nil {
+		return fmt.Errorf("error getting block: %w", err)
+	}
+	block.View = data.View
+	err = wstore.DBUpdate(ctx, block)
+	if err != nil {
+		return fmt.Errorf("error updating block: %w", err)
+	}
+	updates := wstore.ContextGetUpdatesRtn(ctx)
+	sendWStoreUpdatesToEventBus(updates)
+	return nil
+}
+
+func (ws *WshServer) BlockRestartCommand(ctx context.Context, data wshrpc.CommandBlockRestartData) error {
+	bc := blockcontroller.GetBlockController(data.BlockId)
+	if bc == nil {
+		return fmt.Errorf("block controller not found for block %q", data.BlockId)
+	}
+	return bc.RestartController()
+}
+
+func (ws *WshServer) BlockInputCommand(ctx context.Context, data wshrpc.CommandBlockInputData) error {
+	bc := blockcontroller.GetBlockController(data.BlockId)
+	if bc == nil {
+		return fmt.Errorf("block controller not found for block %q", data.BlockId)
+	}
+	inputUnion := &blockcontroller.BlockInputUnion{
+		SigName:  data.SigName,
+		TermSize: data.TermSize,
+	}
+	if len(data.InputData64) > 0 {
+		inputBuf := make([]byte, base64.StdEncoding.DecodedLen(len(data.InputData64)))
+		nw, err := base64.StdEncoding.Decode(inputBuf, []byte(data.InputData64))
+		if err != nil {
+			return fmt.Errorf("error decoding input data: %w", err)
+		}
+		inputUnion.InputData = inputBuf[:nw]
+	}
+	return bc.SendInput(inputUnion)
+}
+
+func (ws *WshServer) AppendFileCommand(ctx context.Context, data wshrpc.CommandAppendFileData) error {
+	dataBuf, err := base64.StdEncoding.DecodeString(data.Data64)
+	if err != nil {
+		return fmt.Errorf("error decoding data64: %w", err)
+	}
+	err = filestore.WFS.AppendData(ctx, data.ZoneId, data.FileName, dataBuf)
+	if err != nil {
+		return fmt.Errorf("error appending to blockfile: %w", err)
+	}
+	eventbus.SendEvent(eventbus.WSEventType{
+		EventType: eventbus.WSEvent_BlockFile,
+		ORef:      waveobj.MakeORef(wstore.OType_Block, data.ZoneId).String(),
+		Data: &eventbus.WSFileEventData{
+			ZoneId:   data.ZoneId,
+			FileName: data.FileName,
+			FileOp:   eventbus.FileOp_Append,
+			Data64:   base64.StdEncoding.EncodeToString(dataBuf),
+		},
+	})
+	return nil
+}
+
+func (ws *WshServer) AppendIJsonCommand(ctx context.Context, data wshrpc.CommandAppendIJsonData) error {
+	tryCreate := true
+	if data.FileName == blockcontroller.BlockFile_Html && tryCreate {
+		err := filestore.WFS.MakeFile(ctx, data.ZoneId, data.FileName, nil, filestore.FileOptsType{MaxSize: blockcontroller.DefaultHtmlMaxFileSize, IJson: true})
+		if err != nil && err != fs.ErrExist {
+			return fmt.Errorf("error creating blockfile[html]: %w", err)
+		}
+	}
+	err := filestore.WFS.AppendIJson(ctx, data.ZoneId, data.FileName, data.Data)
+	if err != nil {
+		return fmt.Errorf("error appending to blockfile(ijson): %w", err)
+	}
+	eventbus.SendEvent(eventbus.WSEventType{
+		EventType: eventbus.WSEvent_BlockFile,
+		ORef:      waveobj.MakeORef(wstore.OType_Block, data.ZoneId).String(),
+		Data: &eventbus.WSFileEventData{
+			ZoneId:   data.ZoneId,
+			FileName: data.FileName,
+			FileOp:   eventbus.FileOp_Append,
+			Data64:   base64.StdEncoding.EncodeToString([]byte("{}")),
+		},
+	})
+	return nil
+}
+
+func decodeRtnVals(rtnVals []reflect.Value) (any, error) {
+	switch len(rtnVals) {
+	case 0:
+		return nil, nil
+	case 1:
+		errIf := rtnVals[0].Interface()
+		if errIf == nil {
+			return nil, nil
+		}
+		return nil, errIf.(error)
+	case 2:
+		errIf := rtnVals[1].Interface()
+		if errIf == nil {
+			return rtnVals[0].Interface(), nil
+		}
+		return rtnVals[0].Interface(), errIf.(error)
+	default:
+		return nil, fmt.Errorf("too many return values: %d", len(rtnVals))
+	}
+}
+
+func mainWshServerHandler(handler *wshutil.RpcResponseHandler) {
+	command := handler.GetCommand()
+	methodDecl := WshServerCommandToDeclMap[command]
+	if methodDecl == nil {
+		handler.SendResponseError(fmt.Errorf("command %q not found", command))
+		return
+	}
+	var callParams []reflect.Value
+	callParams = append(callParams, reflect.ValueOf(handler.Context()))
+	if methodDecl.CommandDataType != nil {
+		commandData := reflect.New(methodDecl.CommandDataType).Interface()
+		err := utilfn.ReUnmarshal(commandData, handler.GetCommandRawData())
+		if err != nil {
+			handler.SendResponseError(fmt.Errorf("error re-marshalling command data: %w", err))
+			return
+		}
+		wshrpc.HackRpcContextIntoData(commandData, handler.GetRpcContext())
+		callParams = append(callParams, reflect.ValueOf(commandData).Elem())
+	}
+	rtnVals := methodDecl.Method.Call(callParams)
+	rtnData, rtnErr := decodeRtnVals(rtnVals)
+	if rtnErr != nil {
+		handler.SendResponseError(rtnErr)
+		return
+	} else {
+		handler.SendResponse(rtnData, true)
+	}
+}
+
+func MakeUnixListener(sockName string) (net.Listener, error) {
+	os.Remove(sockName) // ignore error
+	rtn, err := net.Listen("unix", sockName)
+	if err != nil {
+		return nil, fmt.Errorf("error creating listener at %v: %v", sockName, err)
+	}
+	os.Chmod(sockName, 0700)
+	log.Printf("Server listening on %s\n", sockName)
+	return rtn, nil
+}
+
+func runWshRpcWithStream(conn net.Conn) {
+	defer conn.Close()
+	inputCh := make(chan []byte, DefaultInputChSize)
+	outputCh := make(chan []byte, DefaultOutputChSize)
+	go wshutil.AdaptMsgChToStream(outputCh, conn)
+	go wshutil.AdaptStreamToMsgCh(conn, inputCh)
+	wshutil.MakeWshRpc(inputCh, outputCh, wshutil.RpcContext{}, mainWshServerHandler)
+}
+
+func RunWshRpcOverListener(listener net.Listener) {
+	go func() {
+		for {
+			conn, err := listener.Accept()
+			if err != nil {
+				log.Printf("error accepting connection: %v\n", err)
+				continue
+			}
+			go runWshRpcWithStream(conn)
+		}
+	}()
+}
+
+func RunDomainSocketWshServer() error {
+	sockName := wavebase.GetDomainSocketName()
+	listener, err := MakeUnixListener(sockName)
+	if err != nil {
+		return fmt.Errorf("error starging unix listener for wsh-server: %w", err)
+	}
+	defer listener.Close()
+	RunWshRpcOverListener(listener)
+	return nil
+}
+
+func MakeWshServer(inputCh chan []byte, outputCh chan []byte, initialCtx wshutil.RpcContext) {
+	wshutil.MakeWshRpc(inputCh, outputCh, initialCtx, mainWshServerHandler)
+}