Remote checker

Add a generic checker that serializes Points protos to a remote process.

More details here: https://docs.google.com/document/d/1RQQKzeFpO-zOoBHZLA-tr5Ed_bvAOLDqgGgKhqUff2A/

Updates #4805

PiperOrigin-RevId: 443690622
This commit is contained in:
Fabricio Voznika
2022-04-22 09:54:21 -07:00
committed by gVisor bot
parent 39790bd3a1
commit 2a238b23e7
5 changed files with 759 additions and 0 deletions
+14
View File
@@ -0,0 +1,14 @@
load("//tools:defs.bzl", "cc_binary")
package(licenses = ["notice"])
cc_binary(
name = "server_cc",
srcs = ["server.cc"],
visibility = ["//:sandbox"],
deps = [
# any_cc_proto placeholder,
"//pkg/sentry/seccheck/points:points_cc_proto",
"@com_google_absl//absl/strings",
],
)
+221
View File
@@ -0,0 +1,221 @@
// Copyright 2021 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <err.h>
#include <pthread.h>
#include <stdarg.h>
#include <stdio.h>
#include <sys/epoll.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <unistd.h>
#include <array>
#include <string>
#include <vector>
#include "google/protobuf/any.pb.h"
#include "absl/strings/string_view.h"
#include "pkg/sentry/seccheck/points/sentry.pb.h"
typedef std::function<void(const google::protobuf::Any& any)> Callback;
constexpr size_t prefixLen = sizeof("type.googleapis.com/") - 1;
constexpr size_t maxEventSize = 300 * 1024;
bool quiet = false;
#pragma pack(push, 1)
struct header {
uint16_t header_size;
uint32_t dropped_count;
};
#pragma pack(pop)
void log(const char* fmt, ...) {
if (!quiet) {
va_list ap;
va_start(ap, fmt);
vprintf(fmt, ap);
va_end(ap);
}
}
template <class T>
void unpack(const google::protobuf::Any& any) {
T evt;
if (!any.UnpackTo(&evt)) {
err(1, "UnpackTo(): %s", any.DebugString().c_str());
}
auto name = any.type_url().substr(prefixLen);
log("%.*s => %s\n", static_cast<int>(name.size()), name.data(),
evt.ShortDebugString().c_str());
}
std::map<std::string, Callback> dispatchers = {
{"gvisor.sentry.CloneInfo", unpack<::gvisor::sentry::CloneInfo>},
{"gvisor.sentry.ExecveInfo", unpack<::gvisor::sentry::ExecveInfo>},
{"gvisor.sentry.ExitNotifyParentInfo",
unpack<::gvisor::sentry::ExitNotifyParentInfo>},
};
void unpack(const absl::string_view buf) {
const header* hdr = reinterpret_cast<const header*>(&buf[0]);
size_t payload_size = buf.size() - hdr->header_size;
if (payload_size <= 0) {
printf("Header size (%u) is larger than message %lu\n", hdr->header_size,
buf.size());
return;
}
auto proto = buf.substr(hdr->header_size);
if (proto.size() < payload_size) {
printf("Message was truncated, size: %lu, expected: %zu\n", proto.size(),
payload_size);
return;
}
google::protobuf::Any any;
if (!any.ParseFromArray(proto.data(), proto.size())) {
err(1, "invalid proto message");
}
auto url = any.type_url();
if (url.size() <= prefixLen) {
printf("Invalid URL %s\n", any.type_url().data());
return;
}
const std::string name(url.substr(prefixLen));
Callback cb = dispatchers[name];
if (cb == nullptr) {
printf("No callback registered for %s. Skipping it...\n", name.c_str());
} else {
cb(any);
}
}
void* pollLoop(void* ptr) {
const int poll_fd = *reinterpret_cast<int*>(&ptr);
for (;;) {
epoll_event evts[64];
int nfds = epoll_wait(poll_fd, evts, 64, -1);
if (nfds < 0) {
if (errno == EINTR) {
continue;
}
err(1, "epoll_wait");
}
for (int i = 0; i < nfds; ++i) {
if (evts[i].events & EPOLLIN) {
int client = evts[i].data.fd;
std::array<char, maxEventSize> buf;
int bytes = read(client, buf.data(), buf.size());
if (bytes < 0) {
err(1, "read");
} else if (bytes > 0) {
unpack(absl::string_view(buf.data(), bytes));
}
}
if ((evts[i].events & (EPOLLRDHUP | EPOLLHUP)) != 0) {
int client = evts[i].data.fd;
close(client);
printf("Connection closed\n");
}
if (evts[i].events & EPOLLERR) {
printf("error\n");
}
}
}
}
void startPollThread(int poll_fd) {
pthread_t thread;
if (pthread_create(&thread, nullptr, pollLoop,
reinterpret_cast<void*>(poll_fd)) != 0) {
err(1, "pthread_create");
}
pthread_detach(thread);
}
extern "C" int main(int argc, char** argv) {
for (int c = 0; (c = getopt(argc, argv, "q")) != -1;) {
switch (c) {
case 'q':
quiet = true;
break;
default:
exit(1);
}
}
if (!quiet) {
setbuf(stdout, NULL);
setbuf(stderr, NULL);
}
std::string path("/tmp/gvisor_events.sock");
if (optind < argc) {
path = argv[optind];
}
if (path.empty()) {
err(1, "empty file name");
}
printf("Socket address %s\n", path.c_str());
unlink(path.c_str());
int sock = socket(AF_UNIX, SOCK_SEQPACKET, 0);
if (sock < 0) {
err(1, "socket");
}
struct sockaddr_un addr;
addr.sun_family = AF_UNIX;
strncpy(addr.sun_path, path.c_str(), path.size() + 1);
if (bind(sock, reinterpret_cast<struct sockaddr*>(&addr), sizeof(addr))) {
err(1, "bind");
}
if (listen(sock, 5) < 0) {
err(1, "listen");
}
int epoll_fd = epoll_create(1);
if (epoll_fd < 0) {
err(1, "epoll_create");
}
startPollThread(epoll_fd);
for (;;) {
int client = accept(sock, nullptr, nullptr);
if (client < 0) {
if (errno == EINTR) {
continue;
}
err(1, "accept");
}
printf("Connection accepted\n");
struct epoll_event evt;
evt.data.fd = client;
evt.events = EPOLLIN;
if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, client, &evt) < 0) {
err(1, "epoll_ctl(ADD)");
}
}
close(sock);
unlink(path.c_str());
return 0;
}
+43
View File
@@ -0,0 +1,43 @@
load("//tools:defs.bzl", "go_library", "go_test")
package(licenses = ["notice"])
go_library(
name = "remote",
srcs = ["remote.go"],
marshal = True,
visibility = ["//:sandbox"],
deps = [
"//pkg/cleanup",
"//pkg/context",
"//pkg/fd",
"//pkg/log",
"//pkg/sentry/seccheck",
"//pkg/sentry/seccheck/points:points_go_proto",
"@org_golang_google_protobuf//proto:go_default_library",
"@org_golang_google_protobuf//types/known/anypb:go_default_library",
"@org_golang_x_sys//unix:go_default_library",
],
)
go_test(
name = "remote_test",
size = "small",
srcs = ["remote_test.go"],
data = [
"//examples/seccheck:server_cc",
],
library = ":remote",
deps = [
"//pkg/cleanup",
"//pkg/fd",
"//pkg/sentry/seccheck",
"//pkg/sentry/seccheck/points:points_go_proto",
"//pkg/sync",
"//pkg/test/testutil",
"@com_github_cenkalti_backoff//:go_default_library",
"@org_golang_google_protobuf//proto:go_default_library",
"@org_golang_google_protobuf//types/known/anypb:go_default_library",
"@org_golang_x_sys//unix:go_default_library",
],
)
@@ -0,0 +1,152 @@
// Copyright 2021 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package remote defines a seccheck.Checker that serializes points to a remote
// process. Points are serialized using the protobuf format, asynchronously.
package remote
import (
"fmt"
"os"
"gvisor.dev/gvisor/pkg/log"
"golang.org/x/sys/unix"
"google.golang.org/protobuf/proto"
"gvisor.dev/gvisor/pkg/cleanup"
"gvisor.dev/gvisor/pkg/context"
"gvisor.dev/gvisor/pkg/fd"
"gvisor.dev/gvisor/pkg/sentry/seccheck"
"google.golang.org/protobuf/types/known/anypb"
pb "gvisor.dev/gvisor/pkg/sentry/seccheck/points/points_go_proto"
)
// Remote sends a serialized point to a remote process asynchronously over a
// SOCK_SEQPACKET Unix-domain socket. Each message corresponds to a single
// serialized point proto, preceded by a standard header. If the point cannot
// be sent, e.g. buffer full, the point is dropped on the floor to avoid
// delaying/hanging indefinitely the application.
type Remote struct {
seccheck.CheckerDefaults
endpoint *fd.FD
}
var _ seccheck.Checker = (*Remote)(nil)
func setup(path string) (*os.File, error) {
log.Debugf("Remote sink connecting to %q", path)
socket, err := unix.Socket(unix.AF_UNIX, unix.SOCK_SEQPACKET, 0)
if err != nil {
return nil, fmt.Errorf("socket(AF_UNIX, SOCK_SEQPACKET, 0): %w", err)
}
f := os.NewFile(uintptr(socket), path)
cu := cleanup.Make(func() {
_ = f.Close()
})
defer cu.Clean()
addr := unix.SockaddrUnix{Name: path}
if err := unix.Connect(int(f.Fd()), &addr); err != nil {
return nil, fmt.Errorf("connect(%q): %w", path, err)
}
cu.Release()
return f, nil
}
// New creates a new Remote checker.
func New(_ map[string]interface{}, endpoint *fd.FD) (seccheck.Checker, error) {
if endpoint == nil {
return nil, fmt.Errorf("remote sink requires an endpoint")
}
// TODO(gvisor.dev/issue/4805): perform version handshake with remote:
// 1. sentry and remote exchange versions
// 2. sentry continues if remote >= min(sentry)
// 3. remote continues if sentry >= min(remote).
// min() being the minimal supported version. Let's say current sentry
// supports batching but remote doesn't, sentry can chose to not batch or
// refuse the connection.
return &Remote{endpoint: endpoint}, nil
}
// Header is used to describe the message being sent to the remote process.
//
// +marshal
type Header struct {
// HeaderSize is the size of the header in bytes. The payload comes
// immediatelly after the header. The length is needed to allow the header to
// expand in the future without breaking remotes that do not yet understand
// the new fields.
HeaderSize uint16
_ uint16
// DroppedCount is the number of points that failed to be written and had to
// be dropped. It wraps around after max(uint32).
DroppedCount uint32
}
// headerStructSize size of header struct in bytes.
const headerStructSize = 8
// TODO(gvisor.dev/issue/4805) Any requires writing the full type URL to the
// message. We're not memory bandwidth bound, but having an enum event type in
// the header to identify the proto type would reduce message size and speed
// up event dispatch in the consumer.
func (r *Remote) writeAny(any *anypb.Any) error {
out, err := proto.Marshal(any)
if err != nil {
return err
}
hdr := Header{
HeaderSize: uint16(headerStructSize),
}
var hdrOut [headerStructSize]byte
hdr.MarshalUnsafe(hdrOut[:])
// TODO(gvisor.dev/issue/4805): Change to non-blocking write. Count as dropped
// if write fails.
_, err = unix.Writev(r.endpoint.FD(), [][]byte{hdrOut[:], out})
return err
}
func (r *Remote) write(msg proto.Message) {
any, err := anypb.New(msg)
if err != nil {
log.Debugf("anypd.New(%+v): %v", msg, err)
return
}
if err := r.writeAny(any); err != nil {
log.Debugf("writeAny(%+v): %v", any, err)
return
}
return
}
// Clone implements seccheck.Checker.
func (r *Remote) Clone(_ context.Context, _ seccheck.FieldSet, info *pb.CloneInfo) error {
r.write(info)
return nil
}
// Execve implements seccheck.Checker.
func (r *Remote) Execve(_ context.Context, _ seccheck.FieldSet, info *pb.ExecveInfo) error {
r.write(info)
return nil
}
// ExitNotifyParent implements seccheck.Checker.
func (r *Remote) ExitNotifyParent(_ context.Context, _ seccheck.FieldSet, info *pb.ExitNotifyParentInfo) error {
r.write(info)
return nil
}
@@ -0,0 +1,329 @@
// Copyright 2022 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package remote
import (
"bytes"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"github.com/cenkalti/backoff"
"golang.org/x/sys/unix"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
"gvisor.dev/gvisor/pkg/cleanup"
"gvisor.dev/gvisor/pkg/fd"
"gvisor.dev/gvisor/pkg/sentry/seccheck"
pb "gvisor.dev/gvisor/pkg/sentry/seccheck/points/points_go_proto"
"gvisor.dev/gvisor/pkg/sync"
"gvisor.dev/gvisor/pkg/test/testutil"
)
func waitForFile(path string) error {
return testutil.Poll(func() error {
if _, err := os.Stat(path); err != nil {
if os.IsNotExist(err) {
return err
}
return &backoff.PermanentError{Err: err}
}
return nil
}, 5*time.Second)
}
type exampleServer struct {
path string
cmd *exec.Cmd
out bytes.Buffer
}
func newExampleServer(quiet bool) (*exampleServer, error) {
exe, err := testutil.FindFile("examples/seccheck/server_cc")
if err != nil {
return nil, fmt.Errorf("error finding server_cc: %v", err)
}
dir, err := os.MkdirTemp(os.TempDir(), "remote")
if err != nil {
return nil, fmt.Errorf("Setup(%q): %v", dir, err)
}
server := &exampleServer{path: filepath.Join(dir, "remote.sock")}
server.cmd = exec.Command(exe, server.path)
if quiet {
server.cmd.Args = append(server.cmd.Args, "-q")
}
server.cmd.Stdout = &server.out
server.cmd.Stderr = &server.out
if err := server.cmd.Start(); err != nil {
os.RemoveAll(dir)
return nil, fmt.Errorf("error running %q: %v", exe, err)
}
if err := waitForFile(server.path); err != nil {
server.stop()
return nil, fmt.Errorf("error waiting for server file %q: %w", server.path, err)
}
return server, nil
}
func (s *exampleServer) stop() {
_ = s.cmd.Process.Kill()
_ = s.cmd.Wait()
_ = os.Remove(s.path)
}
type server struct {
path string
fd *fd.FD
stopCh chan struct{}
mu sync.Mutex
// +checklocks:mu
points []*anypb.Any
}
func newServer() (*server, error) {
dir, err := ioutil.TempDir(os.TempDir(), "remote")
if err != nil {
return nil, err
}
server, err := newServerPath(filepath.Join(dir, "remote.sock"))
if err != nil {
_ = os.RemoveAll(dir)
return nil, err
}
return server, nil
}
func newServerPath(path string) (*server, error) {
socket, err := unix.Socket(unix.AF_UNIX, unix.SOCK_SEQPACKET, 0)
if err != nil {
return nil, fmt.Errorf("socket(AF_UNIX, SOCK_SEQPACKET, 0): %w", err)
}
cu := cleanup.Make(func() {
_ = unix.Close(socket)
})
defer cu.Clean()
sa := &unix.SockaddrUnix{Name: path}
if err := unix.Bind(socket, sa); err != nil {
return nil, fmt.Errorf("bind(%q): %w", path, err)
}
if err := unix.Listen(socket, 5); err != nil {
return nil, fmt.Errorf("listen(): %w", err)
}
server := &server{
path: path,
fd: fd.New(socket),
stopCh: make(chan struct{}),
}
go server.run()
cu.Release()
return server, nil
}
func (s *server) run() {
defer func() {
s.stopCh <- struct{}{}
}()
for {
client, _, err := unix.Accept(s.fd.FD())
if err != nil {
panic(err)
}
go s.handleClient(client)
}
}
func (s *server) handleClient(client int) {
defer unix.Close(client)
var buf = make([]byte, 1024*1024)
for {
read, err := unix.Read(client, buf)
if err != nil {
return
}
if read == 0 {
return
}
if read <= headerStructSize {
panic("invalid message")
}
hdr := Header{}
hdr.UnmarshalUnsafe(buf[0:headerStructSize])
msg := &anypb.Any{}
if err := proto.Unmarshal(buf[hdr.HeaderSize:read], msg); err != nil {
panic("invalid proto")
}
s.mu.Lock()
s.points = append(s.points, msg)
s.mu.Unlock()
}
}
func (s *server) count() int {
s.mu.Lock()
defer s.mu.Unlock()
return len(s.points)
}
func (s *server) getPoints() []*anypb.Any {
s.mu.Lock()
defer s.mu.Unlock()
cpy := make([]*anypb.Any, len(s.points))
copy(cpy, s.points)
return cpy
}
func (s *server) wait() {
<-s.stopCh
}
func (s *server) close() {
_ = s.fd.Close()
_ = os.Remove(s.path)
}
func TestBasic(t *testing.T) {
server, err := newServer()
if err != nil {
t.Fatalf("newServer(): %v", err)
}
defer server.close()
endpoint, err := setup(server.path)
if err != nil {
t.Fatalf("setup(): %v", err)
}
endpointFD, err := fd.NewFromFile(endpoint)
if err != nil {
_ = endpoint.Close()
t.Fatalf("NewFromFile(): %v", err)
}
_ = endpoint.Close()
r, err := New(nil, endpointFD)
if err != nil {
t.Fatalf("New(): %v", err)
}
info := &pb.ExitNotifyParentInfo{ExitStatus: 123}
if err := r.ExitNotifyParent(nil, seccheck.FieldSet{}, info); err != nil {
t.Fatalf("ExitNotifyParent: %v", err)
}
testutil.Poll(func() error {
if server.count() == 0 {
return fmt.Errorf("waiting for points to arrive")
}
return nil
}, 5*time.Second)
if want, got := 1, server.count(); want != got {
t.Errorf("wrong number of points, want: %d, got: %d", want, got)
}
any := server.getPoints()[0]
got := &pb.ExitNotifyParentInfo{}
if err := any.UnmarshalTo(got); err != nil {
t.Errorf("any.UnmarshallTo(ExitNotifyParentInfo): %v", err)
}
if !proto.Equal(info, got) {
t.Errorf("Received point is different, want: %+v, got: %+v", info, got)
}
}
// Test that the example C++ server works. It's easier to test from here and
// also changes that can break it will likely originate here.
func TestExample(t *testing.T) {
server, err := newExampleServer(false)
if err != nil {
t.Fatalf("newExampleServer(): %v", err)
}
defer server.stop()
endpoint, err := setup(server.path)
if err != nil {
t.Fatalf("setup(): %v", err)
}
endpointFD, err := fd.NewFromFile(endpoint)
if err != nil {
_ = endpoint.Close()
t.Fatalf("NewFromFile(): %v", err)
}
_ = endpoint.Close()
r, err := New(nil, endpointFD)
if err != nil {
t.Fatalf("New(): %v", err)
}
info := pb.ExitNotifyParentInfo{ExitStatus: 123}
if err := r.ExitNotifyParent(nil, seccheck.FieldSet{}, &info); err != nil {
t.Fatalf("ExitNotifyParent: %v", err)
}
check := func() error {
if got := server.out.String(); !strings.Contains(got, "gvisor.sentry.ExitNotifyParentInfo => exit_status: 123") {
return fmt.Errorf("ExitNotifyParentInfo point didn't get to the server, out: %q", got)
}
return nil
}
if err := testutil.Poll(check, time.Second); err != nil {
t.Errorf(err.Error())
}
}
func BenchmarkSmall(t *testing.B) {
// Run server in a separate process just to isolate it as much as possible.
server, err := newExampleServer(false)
if err != nil {
t.Fatalf("newExampleServer(): %v", err)
}
defer server.stop()
endpoint, err := setup(server.path)
if err != nil {
t.Fatalf("setup(): %v", err)
}
endpointFD, err := fd.NewFromFile(endpoint)
if err != nil {
_ = endpoint.Close()
t.Fatalf("NewFromFile(): %v", err)
}
_ = endpoint.Close()
r, err := New(nil, endpointFD)
if err != nil {
t.Fatalf("New(): %v", err)
}
t.ResetTimer()
t.RunParallel(func(sub *testing.PB) {
for sub.Next() {
info := pb.ExitNotifyParentInfo{ExitStatus: 123}
if err := r.ExitNotifyParent(nil, seccheck.FieldSet{}, &info); err != nil {
t.Fatalf("ExitNotifyParent: %v", err)
}
}
})
}