diff --git a/examples/seccheck/BUILD b/examples/seccheck/BUILD new file mode 100644 index 000000000..ed4fbbabb --- /dev/null +++ b/examples/seccheck/BUILD @@ -0,0 +1,14 @@ +load("//tools:defs.bzl", "cc_binary") + +package(licenses = ["notice"]) + +cc_binary( + name = "server_cc", + srcs = ["server.cc"], + visibility = ["//:sandbox"], + deps = [ + # any_cc_proto placeholder, + "//pkg/sentry/seccheck/points:points_cc_proto", + "@com_google_absl//absl/strings", + ], +) diff --git a/examples/seccheck/server.cc b/examples/seccheck/server.cc new file mode 100644 index 000000000..38f91d37e --- /dev/null +++ b/examples/seccheck/server.cc @@ -0,0 +1,221 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "google/protobuf/any.pb.h" +#include "absl/strings/string_view.h" +#include "pkg/sentry/seccheck/points/sentry.pb.h" + +typedef std::function Callback; + +constexpr size_t prefixLen = sizeof("type.googleapis.com/") - 1; +constexpr size_t maxEventSize = 300 * 1024; + +bool quiet = false; + +#pragma pack(push, 1) +struct header { + uint16_t header_size; + uint32_t dropped_count; +}; +#pragma pack(pop) + +void log(const char* fmt, ...) { + if (!quiet) { + va_list ap; + va_start(ap, fmt); + vprintf(fmt, ap); + va_end(ap); + } +} + +template +void unpack(const google::protobuf::Any& any) { + T evt; + if (!any.UnpackTo(&evt)) { + err(1, "UnpackTo(): %s", any.DebugString().c_str()); + } + auto name = any.type_url().substr(prefixLen); + log("%.*s => %s\n", static_cast(name.size()), name.data(), + evt.ShortDebugString().c_str()); +} + +std::map dispatchers = { + {"gvisor.sentry.CloneInfo", unpack<::gvisor::sentry::CloneInfo>}, + {"gvisor.sentry.ExecveInfo", unpack<::gvisor::sentry::ExecveInfo>}, + {"gvisor.sentry.ExitNotifyParentInfo", + unpack<::gvisor::sentry::ExitNotifyParentInfo>}, +}; + +void unpack(const absl::string_view buf) { + const header* hdr = reinterpret_cast(&buf[0]); + size_t payload_size = buf.size() - hdr->header_size; + if (payload_size <= 0) { + printf("Header size (%u) is larger than message %lu\n", hdr->header_size, + buf.size()); + return; + } + + auto proto = buf.substr(hdr->header_size); + if (proto.size() < payload_size) { + printf("Message was truncated, size: %lu, expected: %zu\n", proto.size(), + payload_size); + return; + } + + google::protobuf::Any any; + if (!any.ParseFromArray(proto.data(), proto.size())) { + err(1, "invalid proto message"); + } + + auto url = any.type_url(); + if (url.size() <= prefixLen) { + printf("Invalid URL %s\n", any.type_url().data()); + return; + } + const std::string name(url.substr(prefixLen)); + Callback cb = dispatchers[name]; + if (cb == nullptr) { + printf("No callback registered for %s. Skipping it...\n", name.c_str()); + } else { + cb(any); + } +} + +void* pollLoop(void* ptr) { + const int poll_fd = *reinterpret_cast(&ptr); + for (;;) { + epoll_event evts[64]; + int nfds = epoll_wait(poll_fd, evts, 64, -1); + if (nfds < 0) { + if (errno == EINTR) { + continue; + } + err(1, "epoll_wait"); + } + + for (int i = 0; i < nfds; ++i) { + if (evts[i].events & EPOLLIN) { + int client = evts[i].data.fd; + std::array buf; + int bytes = read(client, buf.data(), buf.size()); + if (bytes < 0) { + err(1, "read"); + } else if (bytes > 0) { + unpack(absl::string_view(buf.data(), bytes)); + } + } + if ((evts[i].events & (EPOLLRDHUP | EPOLLHUP)) != 0) { + int client = evts[i].data.fd; + close(client); + printf("Connection closed\n"); + } + if (evts[i].events & EPOLLERR) { + printf("error\n"); + } + } + } +} + +void startPollThread(int poll_fd) { + pthread_t thread; + if (pthread_create(&thread, nullptr, pollLoop, + reinterpret_cast(poll_fd)) != 0) { + err(1, "pthread_create"); + } + pthread_detach(thread); +} + +extern "C" int main(int argc, char** argv) { + for (int c = 0; (c = getopt(argc, argv, "q")) != -1;) { + switch (c) { + case 'q': + quiet = true; + break; + default: + exit(1); + } + } + + if (!quiet) { + setbuf(stdout, NULL); + setbuf(stderr, NULL); + } + std::string path("/tmp/gvisor_events.sock"); + if (optind < argc) { + path = argv[optind]; + } + if (path.empty()) { + err(1, "empty file name"); + } + printf("Socket address %s\n", path.c_str()); + unlink(path.c_str()); + + int sock = socket(AF_UNIX, SOCK_SEQPACKET, 0); + if (sock < 0) { + err(1, "socket"); + } + + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, path.c_str(), path.size() + 1); + if (bind(sock, reinterpret_cast(&addr), sizeof(addr))) { + err(1, "bind"); + } + if (listen(sock, 5) < 0) { + err(1, "listen"); + } + + int epoll_fd = epoll_create(1); + if (epoll_fd < 0) { + err(1, "epoll_create"); + } + startPollThread(epoll_fd); + + for (;;) { + int client = accept(sock, nullptr, nullptr); + if (client < 0) { + if (errno == EINTR) { + continue; + } + err(1, "accept"); + } + printf("Connection accepted\n"); + + struct epoll_event evt; + evt.data.fd = client; + evt.events = EPOLLIN; + if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, client, &evt) < 0) { + err(1, "epoll_ctl(ADD)"); + } + } + + close(sock); + unlink(path.c_str()); + + return 0; +} diff --git a/pkg/sentry/seccheck/checkers/remote/BUILD b/pkg/sentry/seccheck/checkers/remote/BUILD new file mode 100644 index 000000000..c3b0e8034 --- /dev/null +++ b/pkg/sentry/seccheck/checkers/remote/BUILD @@ -0,0 +1,43 @@ +load("//tools:defs.bzl", "go_library", "go_test") + +package(licenses = ["notice"]) + +go_library( + name = "remote", + srcs = ["remote.go"], + marshal = True, + visibility = ["//:sandbox"], + deps = [ + "//pkg/cleanup", + "//pkg/context", + "//pkg/fd", + "//pkg/log", + "//pkg/sentry/seccheck", + "//pkg/sentry/seccheck/points:points_go_proto", + "@org_golang_google_protobuf//proto:go_default_library", + "@org_golang_google_protobuf//types/known/anypb:go_default_library", + "@org_golang_x_sys//unix:go_default_library", + ], +) + +go_test( + name = "remote_test", + size = "small", + srcs = ["remote_test.go"], + data = [ + "//examples/seccheck:server_cc", + ], + library = ":remote", + deps = [ + "//pkg/cleanup", + "//pkg/fd", + "//pkg/sentry/seccheck", + "//pkg/sentry/seccheck/points:points_go_proto", + "//pkg/sync", + "//pkg/test/testutil", + "@com_github_cenkalti_backoff//:go_default_library", + "@org_golang_google_protobuf//proto:go_default_library", + "@org_golang_google_protobuf//types/known/anypb:go_default_library", + "@org_golang_x_sys//unix:go_default_library", + ], +) diff --git a/pkg/sentry/seccheck/checkers/remote/remote.go b/pkg/sentry/seccheck/checkers/remote/remote.go new file mode 100644 index 000000000..a01504c09 --- /dev/null +++ b/pkg/sentry/seccheck/checkers/remote/remote.go @@ -0,0 +1,152 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package remote defines a seccheck.Checker that serializes points to a remote +// process. Points are serialized using the protobuf format, asynchronously. +package remote + +import ( + "fmt" + "os" + + "gvisor.dev/gvisor/pkg/log" + + "golang.org/x/sys/unix" + "google.golang.org/protobuf/proto" + "gvisor.dev/gvisor/pkg/cleanup" + "gvisor.dev/gvisor/pkg/context" + "gvisor.dev/gvisor/pkg/fd" + "gvisor.dev/gvisor/pkg/sentry/seccheck" + + "google.golang.org/protobuf/types/known/anypb" + pb "gvisor.dev/gvisor/pkg/sentry/seccheck/points/points_go_proto" +) + +// Remote sends a serialized point to a remote process asynchronously over a +// SOCK_SEQPACKET Unix-domain socket. Each message corresponds to a single +// serialized point proto, preceded by a standard header. If the point cannot +// be sent, e.g. buffer full, the point is dropped on the floor to avoid +// delaying/hanging indefinitely the application. +type Remote struct { + seccheck.CheckerDefaults + + endpoint *fd.FD +} + +var _ seccheck.Checker = (*Remote)(nil) + +func setup(path string) (*os.File, error) { + log.Debugf("Remote sink connecting to %q", path) + socket, err := unix.Socket(unix.AF_UNIX, unix.SOCK_SEQPACKET, 0) + if err != nil { + return nil, fmt.Errorf("socket(AF_UNIX, SOCK_SEQPACKET, 0): %w", err) + } + f := os.NewFile(uintptr(socket), path) + cu := cleanup.Make(func() { + _ = f.Close() + }) + defer cu.Clean() + + addr := unix.SockaddrUnix{Name: path} + if err := unix.Connect(int(f.Fd()), &addr); err != nil { + return nil, fmt.Errorf("connect(%q): %w", path, err) + } + cu.Release() + return f, nil +} + +// New creates a new Remote checker. +func New(_ map[string]interface{}, endpoint *fd.FD) (seccheck.Checker, error) { + if endpoint == nil { + return nil, fmt.Errorf("remote sink requires an endpoint") + } + // TODO(gvisor.dev/issue/4805): perform version handshake with remote: + // 1. sentry and remote exchange versions + // 2. sentry continues if remote >= min(sentry) + // 3. remote continues if sentry >= min(remote). + // min() being the minimal supported version. Let's say current sentry + // supports batching but remote doesn't, sentry can chose to not batch or + // refuse the connection. + return &Remote{endpoint: endpoint}, nil +} + +// Header is used to describe the message being sent to the remote process. +// +// +marshal +type Header struct { + // HeaderSize is the size of the header in bytes. The payload comes + // immediatelly after the header. The length is needed to allow the header to + // expand in the future without breaking remotes that do not yet understand + // the new fields. + HeaderSize uint16 + _ uint16 + // DroppedCount is the number of points that failed to be written and had to + // be dropped. It wraps around after max(uint32). + DroppedCount uint32 +} + +// headerStructSize size of header struct in bytes. +const headerStructSize = 8 + +// TODO(gvisor.dev/issue/4805) Any requires writing the full type URL to the +// message. We're not memory bandwidth bound, but having an enum event type in +// the header to identify the proto type would reduce message size and speed +// up event dispatch in the consumer. +func (r *Remote) writeAny(any *anypb.Any) error { + out, err := proto.Marshal(any) + if err != nil { + return err + } + hdr := Header{ + HeaderSize: uint16(headerStructSize), + } + var hdrOut [headerStructSize]byte + hdr.MarshalUnsafe(hdrOut[:]) + + // TODO(gvisor.dev/issue/4805): Change to non-blocking write. Count as dropped + // if write fails. + _, err = unix.Writev(r.endpoint.FD(), [][]byte{hdrOut[:], out}) + return err +} + +func (r *Remote) write(msg proto.Message) { + any, err := anypb.New(msg) + if err != nil { + log.Debugf("anypd.New(%+v): %v", msg, err) + return + } + if err := r.writeAny(any); err != nil { + log.Debugf("writeAny(%+v): %v", any, err) + return + } + return +} + +// Clone implements seccheck.Checker. +func (r *Remote) Clone(_ context.Context, _ seccheck.FieldSet, info *pb.CloneInfo) error { + r.write(info) + return nil +} + +// Execve implements seccheck.Checker. +func (r *Remote) Execve(_ context.Context, _ seccheck.FieldSet, info *pb.ExecveInfo) error { + r.write(info) + return nil +} + +// ExitNotifyParent implements seccheck.Checker. +func (r *Remote) ExitNotifyParent(_ context.Context, _ seccheck.FieldSet, info *pb.ExitNotifyParentInfo) error { + r.write(info) + return nil +} diff --git a/pkg/sentry/seccheck/checkers/remote/remote_test.go b/pkg/sentry/seccheck/checkers/remote/remote_test.go new file mode 100644 index 000000000..bdcef353b --- /dev/null +++ b/pkg/sentry/seccheck/checkers/remote/remote_test.go @@ -0,0 +1,329 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/cenkalti/backoff" + "golang.org/x/sys/unix" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + "gvisor.dev/gvisor/pkg/cleanup" + "gvisor.dev/gvisor/pkg/fd" + "gvisor.dev/gvisor/pkg/sentry/seccheck" + pb "gvisor.dev/gvisor/pkg/sentry/seccheck/points/points_go_proto" + "gvisor.dev/gvisor/pkg/sync" + "gvisor.dev/gvisor/pkg/test/testutil" +) + +func waitForFile(path string) error { + return testutil.Poll(func() error { + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + return err + } + return &backoff.PermanentError{Err: err} + } + return nil + }, 5*time.Second) +} + +type exampleServer struct { + path string + cmd *exec.Cmd + out bytes.Buffer +} + +func newExampleServer(quiet bool) (*exampleServer, error) { + exe, err := testutil.FindFile("examples/seccheck/server_cc") + if err != nil { + return nil, fmt.Errorf("error finding server_cc: %v", err) + } + + dir, err := os.MkdirTemp(os.TempDir(), "remote") + if err != nil { + return nil, fmt.Errorf("Setup(%q): %v", dir, err) + } + + server := &exampleServer{path: filepath.Join(dir, "remote.sock")} + server.cmd = exec.Command(exe, server.path) + if quiet { + server.cmd.Args = append(server.cmd.Args, "-q") + } + server.cmd.Stdout = &server.out + server.cmd.Stderr = &server.out + if err := server.cmd.Start(); err != nil { + os.RemoveAll(dir) + return nil, fmt.Errorf("error running %q: %v", exe, err) + } + + if err := waitForFile(server.path); err != nil { + server.stop() + return nil, fmt.Errorf("error waiting for server file %q: %w", server.path, err) + } + return server, nil +} + +func (s *exampleServer) stop() { + _ = s.cmd.Process.Kill() + _ = s.cmd.Wait() + _ = os.Remove(s.path) +} + +type server struct { + path string + fd *fd.FD + stopCh chan struct{} + + mu sync.Mutex + // +checklocks:mu + points []*anypb.Any +} + +func newServer() (*server, error) { + dir, err := ioutil.TempDir(os.TempDir(), "remote") + if err != nil { + return nil, err + } + server, err := newServerPath(filepath.Join(dir, "remote.sock")) + if err != nil { + _ = os.RemoveAll(dir) + return nil, err + } + return server, nil +} + +func newServerPath(path string) (*server, error) { + socket, err := unix.Socket(unix.AF_UNIX, unix.SOCK_SEQPACKET, 0) + if err != nil { + return nil, fmt.Errorf("socket(AF_UNIX, SOCK_SEQPACKET, 0): %w", err) + } + cu := cleanup.Make(func() { + _ = unix.Close(socket) + }) + defer cu.Clean() + + sa := &unix.SockaddrUnix{Name: path} + if err := unix.Bind(socket, sa); err != nil { + return nil, fmt.Errorf("bind(%q): %w", path, err) + } + if err := unix.Listen(socket, 5); err != nil { + return nil, fmt.Errorf("listen(): %w", err) + } + + server := &server{ + path: path, + fd: fd.New(socket), + stopCh: make(chan struct{}), + } + go server.run() + cu.Release() + return server, nil +} + +func (s *server) run() { + defer func() { + s.stopCh <- struct{}{} + }() + for { + client, _, err := unix.Accept(s.fd.FD()) + if err != nil { + panic(err) + } + go s.handleClient(client) + } +} + +func (s *server) handleClient(client int) { + defer unix.Close(client) + + var buf = make([]byte, 1024*1024) + for { + read, err := unix.Read(client, buf) + if err != nil { + return + } + if read == 0 { + return + } + if read <= headerStructSize { + panic("invalid message") + } + hdr := Header{} + hdr.UnmarshalUnsafe(buf[0:headerStructSize]) + msg := &anypb.Any{} + if err := proto.Unmarshal(buf[hdr.HeaderSize:read], msg); err != nil { + panic("invalid proto") + } + s.mu.Lock() + s.points = append(s.points, msg) + s.mu.Unlock() + } +} + +func (s *server) count() int { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.points) +} + +func (s *server) getPoints() []*anypb.Any { + s.mu.Lock() + defer s.mu.Unlock() + cpy := make([]*anypb.Any, len(s.points)) + copy(cpy, s.points) + return cpy +} + +func (s *server) wait() { + <-s.stopCh +} + +func (s *server) close() { + _ = s.fd.Close() + _ = os.Remove(s.path) +} + +func TestBasic(t *testing.T) { + server, err := newServer() + if err != nil { + t.Fatalf("newServer(): %v", err) + } + defer server.close() + + endpoint, err := setup(server.path) + if err != nil { + t.Fatalf("setup(): %v", err) + } + endpointFD, err := fd.NewFromFile(endpoint) + if err != nil { + _ = endpoint.Close() + t.Fatalf("NewFromFile(): %v", err) + } + _ = endpoint.Close() + + r, err := New(nil, endpointFD) + if err != nil { + t.Fatalf("New(): %v", err) + } + + info := &pb.ExitNotifyParentInfo{ExitStatus: 123} + if err := r.ExitNotifyParent(nil, seccheck.FieldSet{}, info); err != nil { + t.Fatalf("ExitNotifyParent: %v", err) + } + + testutil.Poll(func() error { + if server.count() == 0 { + return fmt.Errorf("waiting for points to arrive") + } + return nil + }, 5*time.Second) + if want, got := 1, server.count(); want != got { + t.Errorf("wrong number of points, want: %d, got: %d", want, got) + } + any := server.getPoints()[0] + + got := &pb.ExitNotifyParentInfo{} + if err := any.UnmarshalTo(got); err != nil { + t.Errorf("any.UnmarshallTo(ExitNotifyParentInfo): %v", err) + } + if !proto.Equal(info, got) { + t.Errorf("Received point is different, want: %+v, got: %+v", info, got) + } +} + +// Test that the example C++ server works. It's easier to test from here and +// also changes that can break it will likely originate here. +func TestExample(t *testing.T) { + server, err := newExampleServer(false) + if err != nil { + t.Fatalf("newExampleServer(): %v", err) + } + defer server.stop() + + endpoint, err := setup(server.path) + if err != nil { + t.Fatalf("setup(): %v", err) + } + endpointFD, err := fd.NewFromFile(endpoint) + if err != nil { + _ = endpoint.Close() + t.Fatalf("NewFromFile(): %v", err) + } + _ = endpoint.Close() + + r, err := New(nil, endpointFD) + if err != nil { + t.Fatalf("New(): %v", err) + } + + info := pb.ExitNotifyParentInfo{ExitStatus: 123} + if err := r.ExitNotifyParent(nil, seccheck.FieldSet{}, &info); err != nil { + t.Fatalf("ExitNotifyParent: %v", err) + } + check := func() error { + if got := server.out.String(); !strings.Contains(got, "gvisor.sentry.ExitNotifyParentInfo => exit_status: 123") { + return fmt.Errorf("ExitNotifyParentInfo point didn't get to the server, out: %q", got) + } + return nil + } + if err := testutil.Poll(check, time.Second); err != nil { + t.Errorf(err.Error()) + } +} + +func BenchmarkSmall(t *testing.B) { + // Run server in a separate process just to isolate it as much as possible. + server, err := newExampleServer(false) + if err != nil { + t.Fatalf("newExampleServer(): %v", err) + } + defer server.stop() + + endpoint, err := setup(server.path) + if err != nil { + t.Fatalf("setup(): %v", err) + } + endpointFD, err := fd.NewFromFile(endpoint) + if err != nil { + _ = endpoint.Close() + t.Fatalf("NewFromFile(): %v", err) + } + _ = endpoint.Close() + + r, err := New(nil, endpointFD) + if err != nil { + t.Fatalf("New(): %v", err) + } + + t.ResetTimer() + t.RunParallel(func(sub *testing.PB) { + for sub.Next() { + info := pb.ExitNotifyParentInfo{ExitStatus: 123} + if err := r.ExitNotifyParent(nil, seccheck.FieldSet{}, &info); err != nil { + t.Fatalf("ExitNotifyParent: %v", err) + } + } + }) +}