From 2083e858ad8de9652b2e5625442899dd1c4cb459 Mon Sep 17 00:00:00 2001 From: Bhasker Hariharan Date: Thu, 27 Jan 2022 17:14:39 -0800 Subject: [PATCH] Add support to provide a sharedmem path to sharedmem. PiperOrigin-RevId: 424744312 --- pkg/tcpip/link/sharedmem/queuepair.go | 65 ++++++++++++------- .../link/sharedmem/sharedmem_server_test.go | 2 +- pkg/tcpip/link/sharedmem/sharedmem_test.go | 4 +- 3 files changed, 44 insertions(+), 27 deletions(-) diff --git a/pkg/tcpip/link/sharedmem/queuepair.go b/pkg/tcpip/link/sharedmem/queuepair.go index c289bb873..3d2ae696b 100644 --- a/pkg/tcpip/link/sharedmem/queuepair.go +++ b/pkg/tcpip/link/sharedmem/queuepair.go @@ -26,11 +26,11 @@ import ( ) const ( - // defaultQueueDataSize is the size of the shared memory data region that + // DefaultQueueDataSize is the size of the shared memory data region that // holds the scatter/gather buffers. - defaultQueueDataSize = 1 << 20 // 1MiB + DefaultQueueDataSize = 1 << 20 // 1MiB - // defaultQueuePipeSize is the size of the pipe that holds the packet descriptors. + // DefaultQueuePipeSize is the size of the pipe that holds the packet descriptors. // // Assuming each packet data is approximately 1280 bytes (IPv6 Minimum MTU) // then we can hold approximately 1024*1024/1280 ~ 819 packets in the data @@ -45,17 +45,21 @@ const ( // descriptors. We could go with a 32 KiB pipe but to give it some slack in // how the upper layer may make use of the scatter gather buffers we double // this to hold enough descriptors. - defaultQueuePipeSize = 64 << 10 // 64KiB + DefaultQueuePipeSize = 64 << 10 // 64KiB - // defaultSharedDataSize is the size of the sharedData region used to + // DefaultSharedDataSize is the size of the sharedData region used to // enable/disable notifications. - defaultSharedDataSize = 4 << 10 // 4KiB + DefaultSharedDataSize = 4 << 10 // 4KiB // DefaultBufferSize is the size of each individual buffer that the data // region is broken down into to hold packet data. Should be larger than // 1500 + 14 (Ethernet header) + 10 (VirtIO header) to fit each packet // in a single buffer. DefaultBufferSize = 2048 + + // DefaultTmpDir is the path used to create the memory files if a path + // is not provided. + DefaultTmpDir = "/dev/shm" ) // A QueuePair represents a pair of TX/RX queues. @@ -67,24 +71,34 @@ type QueuePair struct { rxCfg QueueConfig } +// QueueOptions allows queue specific configuration to be specified when +// creating a QueuePair. +type QueueOptions struct { + // sharedMemPath is the path to use to create the shared memory backing + // files for the queue. + // + // If unspecified it defaults to "/dev/shm". + sharedMemPath string +} + // NewQueuePair creates a shared memory QueuePair. -func NewQueuePair() (*QueuePair, error) { - txCfg, err := createQueueFDs(queueSizes{ - dataSize: defaultQueueDataSize, - txPipeSize: defaultQueuePipeSize, - rxPipeSize: defaultQueuePipeSize, - sharedDataSize: defaultSharedDataSize, +func NewQueuePair(opts QueueOptions) (*QueuePair, error) { + txCfg, err := createQueueFDs(opts.sharedMemPath, queueSizes{ + dataSize: DefaultQueueDataSize, + txPipeSize: DefaultQueuePipeSize, + rxPipeSize: DefaultQueuePipeSize, + sharedDataSize: DefaultSharedDataSize, }) if err != nil { return nil, fmt.Errorf("failed to create tx queue: %s", err) } - rxCfg, err := createQueueFDs(queueSizes{ - dataSize: defaultQueueDataSize, - txPipeSize: defaultQueuePipeSize, - rxPipeSize: defaultQueuePipeSize, - sharedDataSize: defaultSharedDataSize, + rxCfg, err := createQueueFDs(opts.sharedMemPath, queueSizes{ + dataSize: DefaultQueueDataSize, + txPipeSize: DefaultQueuePipeSize, + rxPipeSize: DefaultQueuePipeSize, + sharedDataSize: DefaultSharedDataSize, }) if err != nil { @@ -121,7 +135,7 @@ type queueSizes struct { sharedDataSize int64 } -func createQueueFDs(s queueSizes) (QueueConfig, error) { +func createQueueFDs(sharedMemPath string, s queueSizes) (QueueConfig, error) { success := false var eventFD eventfd.Eventfd var dataFD, txPipeFD, rxPipeFD, sharedDataFD int @@ -141,19 +155,19 @@ func createQueueFDs(s queueSizes) (QueueConfig, error) { if err != nil { return QueueConfig{}, fmt.Errorf("eventfd failed: %v", err) } - dataFD, err = createFile(s.dataSize, false) + dataFD, err = createFile(sharedMemPath, s.dataSize, false) if err != nil { return QueueConfig{}, fmt.Errorf("failed to create dataFD: %s", err) } - txPipeFD, err = createFile(s.txPipeSize, true) + txPipeFD, err = createFile(sharedMemPath, s.txPipeSize, true) if err != nil { return QueueConfig{}, fmt.Errorf("failed to create txPipeFD: %s", err) } - rxPipeFD, err = createFile(s.rxPipeSize, true) + rxPipeFD, err = createFile(sharedMemPath, s.rxPipeSize, true) if err != nil { return QueueConfig{}, fmt.Errorf("failed to create rxPipeFD: %s", err) } - sharedDataFD, err = createFile(s.sharedDataSize, false) + sharedDataFD, err = createFile(sharedMemPath, s.sharedDataSize, false) if err != nil { return QueueConfig{}, fmt.Errorf("failed to create sharedDataFD: %s", err) } @@ -167,8 +181,11 @@ func createQueueFDs(s queueSizes) (QueueConfig, error) { }, nil } -func createFile(size int64, initQueue bool) (fd int, err error) { - const tmpDir = "/dev/shm/" +func createFile(sharedMemPath string, size int64, initQueue bool) (fd int, err error) { + var tmpDir = DefaultTmpDir + if sharedMemPath != "" { + tmpDir = sharedMemPath + } f, err := ioutil.TempFile(tmpDir, "sharedmem_test") if err != nil { return -1, fmt.Errorf("TempFile failed: %v", err) diff --git a/pkg/tcpip/link/sharedmem/sharedmem_server_test.go b/pkg/tcpip/link/sharedmem/sharedmem_server_test.go index a13e753e3..26d234148 100644 --- a/pkg/tcpip/link/sharedmem/sharedmem_server_test.go +++ b/pkg/tcpip/link/sharedmem/sharedmem_server_test.go @@ -162,7 +162,7 @@ func newTestContext(t *testing.T) *testContext { if err != nil { t.Fatalf("failed to create peerFDs: %s", err) } - q, err := sharedmem.NewQueuePair() + q, err := sharedmem.NewQueuePair(sharedmem.QueueOptions{}) if err != nil { t.Fatalf("failed to create sharedmem queue: %s", err) } diff --git a/pkg/tcpip/link/sharedmem/sharedmem_test.go b/pkg/tcpip/link/sharedmem/sharedmem_test.go index 1abc0b91e..d57839303 100644 --- a/pkg/tcpip/link/sharedmem/sharedmem_test.go +++ b/pkg/tcpip/link/sharedmem/sharedmem_test.go @@ -104,7 +104,7 @@ func newTestContext(t *testing.T, mtu, bufferSize uint32, addr tcpip.LinkAddress t: t, packetCh: make(chan struct{}, 1000000), } - c.txCfg, err = createQueueFDs(queueSizes{ + c.txCfg, err = createQueueFDs("" /* sharedMemPath */, queueSizes{ dataSize: queueDataSize, txPipeSize: queuePipeSize, rxPipeSize: queuePipeSize, @@ -113,7 +113,7 @@ func newTestContext(t *testing.T, mtu, bufferSize uint32, addr tcpip.LinkAddress if err != nil { t.Fatalf("createQueueFDs for tx failed: %s", err) } - c.rxCfg, err = createQueueFDs(queueSizes{ + c.rxCfg, err = createQueueFDs("" /* sharedMemPath */, queueSizes{ dataSize: queueDataSize, txPipeSize: queuePipeSize, rxPipeSize: queuePipeSize,