Update lisafs to return ENOMEM on hitting max channel limit.

Also add check to ensure that at least 1 channel is created.

PiperOrigin-RevId: 433256702

pkg/lisafs/channel.go

@@ -102,9 +102,13 @@ func (c *Connection) createChannel(maxMessageSize uint32) (*channel, flipcall.Pa
 	c.channelsMu.Lock()
 	defer c.channelsMu.Unlock()
 	// If c.channels is nil, the connection has closed.
-	if c.channels == nil || len(c.channels) >= maxChannels() {
+	if c.channels == nil {
 		return nil, flipcall.PacketWindowDescriptor{}, -1, unix.ENOSYS
 	}
+	// Return ENOMEM to indicate that the server has hit its max channels limit.
+	if len(c.channels) >= maxChannels() {
+		return nil, flipcall.PacketWindowDescriptor{}, -1, unix.ENOMEM
+	}
 	ch := &channel{}
 
 	// Set up data channel.

pkg/lisafs/client.go

@@ -128,7 +128,11 @@ func NewClient(sock *unet.Socket) (*Client, Inode, error) {
 			defer channelsWg.Done()
 			ch, err := c.createChannel()
 			if err != nil {
-				log.Warningf("channel creation failed: %v", err)
+				if err == unix.ENOMEM {
+					log.Debugf("channel creation failed because server hit max channels limit")
+				} else {
+					log.Warningf("channel creation failed: %v", err)
+				}
 				return
 			}
 			c.channelsMu.Lock()
@@ -139,6 +143,16 @@ func NewClient(sock *unet.Socket) (*Client, Inode, error) {
 	}
 	channelsWg.Wait()
 
+	// Check that atleast 1 channel is created. This is not required by lisafs
+	// protocol. It exists to flag server side issues in channel creation.
+	c.channelsMu.Lock()
+	numChannels := len(c.channels)
+	c.channelsMu.Unlock()
+	if maxChans > 0 && numChannels == 0 {
+		log.Warningf("all channel RPCs failed")
+		return nil, Inode{}, unix.ENOMEM
+	}
+
 	cu.Release()
 	return c, mountResp.Root, nil
 }