From c68ab523c6c357239a48734aba8498a762e755ef Mon Sep 17 00:00:00 2001 From: Zero King Date: Sun, 9 Jul 2017 15:02:08 +0000 Subject: [PATCH] Initial project import --- ci/buildWorker.go | 86 ++++++++++++++++++++++++++++++++++++++ ci/logger/log.go | 94 ++++++++++++++++++++++++++++++++++++++++++ ci/logger/remoteLog.go | 84 +++++++++++++++++++++++++++++++++++++ ci/mpbb.go | 33 +++++++++++++++ ci/portlist.go | 32 ++++++++++++++ ci/portutil.go | 37 +++++++++++++++++ ci/runner/main.go | 23 +++++++++++ ci/session.go | 78 +++++++++++++++++++++++++++++++++++ ci/worker.go | 14 +++++++ pr/dbutil.go | 76 ++++++++++++++++++++++++++++++++++ pr/prbot/main.go | 20 +++++++++ pr/webhook/server.go | 63 ++++++++++++++++++++++++++++ 12 files changed, 640 insertions(+) create mode 100644 ci/buildWorker.go create mode 100644 ci/logger/log.go create mode 100644 ci/logger/remoteLog.go create mode 100644 ci/mpbb.go create mode 100644 ci/portlist.go create mode 100644 ci/portutil.go create mode 100644 ci/runner/main.go create mode 100644 ci/session.go create mode 100644 ci/worker.go create mode 100644 pr/dbutil.go create mode 100644 pr/prbot/main.go create mode 100644 pr/webhook/server.go diff --git a/ci/buildWorker.go b/ci/buildWorker.go new file mode 100644 index 0000000..9ce145e --- /dev/null +++ b/ci/buildWorker.go @@ -0,0 +1,86 @@ +package ci + +import ( + "os/exec" + "path" + + "github.com/macports/mpbot-github/ci/logger" +) + +// buildWorker receive ports from portChan +type buildWorker struct { + worker + portChan chan string +} + +func newBuildWorker(session *Session) *buildWorker { + return &buildWorker{ + worker: worker{session: session, quitChan: make(chan byte)}, + portChan: make(chan string), + } +} + +func (worker *buildWorker) start() { + returnCode := byte(0) + for { + select { + case port := <-worker.portChan: + if port == "" { + worker.quitChan <- returnCode + return + } + subports, err := ListSubports(port) + if err != nil { + returnCode = 1 + logger.GlobalLogger.LogTextChan <- &logger.LogText{"port-" + port + "-subports-fail", []byte(err.Error())} + continue + } + for _, subport := range subports { + DeactivateAllPorts() + portTmpDir := path.Join(worker.session.tmpDir, subport) + logFilename := path.Join(worker.session.tmpDir, "port-"+subport+"-dep-install.log") + err := mpbbToLog("install-dependencies", subport, portTmpDir, logFilename) + statusString := "success" + if err != nil { + if eerr, ok := err.(*exec.ExitError); ok { + if !eerr.Success() { + returnCode = 1 + statusString = "fail" + } + } + } + logger.GlobalLogger.LogFileChan <- &logger.LogFile{ + "port-" + subport + "-dep-summary-" + statusString, + path.Join(portTmpDir, "logs/dependencies-progress.txt"), + } + if err != nil { + logger.GlobalLogger.LogBigFileChan <- &logger.LogFile{ + "port-" + port + "-dep-install-output-" + statusString, + logFilename, + } + continue + } + + logFilename = path.Join(worker.session.tmpDir, "port-"+subport+"-install.log") + mpbbToLog("install-port", subport, portTmpDir, logFilename) + statusString = "success" + if err != nil { + if eerr, ok := err.(*exec.ExitError); ok { + if !eerr.Success() { + returnCode = 1 + statusString = "fail" + } + } + } + logger.GlobalLogger.LogFileChan <- &logger.LogFile{ + "port-" + subport + "-install-summary-" + statusString, + path.Join(portTmpDir, "logs/ports-progress.txt"), + } + logger.GlobalLogger.LogBigFileChan <- &logger.LogFile{ + "port-" + port + "-install-output-" + statusString, + logFilename, + } + } + } + } +} diff --git a/ci/logger/log.go b/ci/logger/log.go new file mode 100644 index 0000000..9656543 --- /dev/null +++ b/ci/logger/log.go @@ -0,0 +1,94 @@ +package logger + +import ( + "io" + "mime/multipart" + "os" + "time" +) + +// This is the actual logger used by the CI bot +var GlobalLogger *Logger = newLogger(os.Stdout) + +func init() { + go GlobalLogger.Run() +} + +// Logger only exits when nil is send to its LogTextChan. +type Logger struct { + LogTextChan chan *LogText + LogFileChan chan *LogFile + LogBigFileChan chan *LogFile + mimeWriter *multipart.Writer + quitChan chan byte + remoteLogger *remoteLogger +} + +type LogFile struct { + FieldName, Filename string +} + +type LogText struct { + FieldName string + Text []byte +} + +func newLogger(w io.Writer) *Logger { + logger := &Logger{ + mimeWriter: multipart.NewWriter(w), + quitChan: make(chan byte), + LogTextChan: make(chan *LogText, 4), + LogFileChan: make(chan *LogFile, 4), + LogBigFileChan: make(chan *LogFile, 4), + } + logger.remoteLogger = newRemoteLogger(logger) + return logger +} + +func (l *Logger) Run() { + go l.remoteLogger.run() + for { + select { + case logFile := <-l.LogFileChan: + if logFile == nil { + continue + } + writer, err := l.mimeWriter.CreateFormField(logFile.FieldName) + if err != nil { + continue + } + file, err := os.Open(logFile.Filename) + if err != nil { + continue + } + + io.Copy(writer, file) + + file.Close() + os.Remove(logFile.Filename) + case logText := <-l.LogTextChan: + if logText == nil { + l.remoteLogger.logBigFileChan <- nil + continue + } + if logText.FieldName == "" { + l.mimeWriter.Close() + l.quitChan <- 0 + return + } + writer, err := l.mimeWriter.CreateFormField(logText.FieldName) + if err != nil { + continue + } + _, err = writer.Write(logText.Text) + case logBigFile := <-l.LogBigFileChan: + l.remoteLogger.logBigFileChan <- logBigFile + case <-time.After(time.Minute * 5): + l.LogTextChan <- &LogText{"keep-alive", []byte{}} + } + } +} + +func (l *Logger) Wait() byte { + return <-l.quitChan +} diff --git a/ci/logger/remoteLog.go b/ci/logger/remoteLog.go new file mode 100644 index 0000000..8ac77c7 --- /dev/null +++ b/ci/logger/remoteLog.go @@ -0,0 +1,84 @@ +package logger + +import ( + "bytes" + "io" + "mime/multipart" + "net/http" + "net/url" + "os" +) + +var pasteURL = &url.URL{ + Scheme: "https", + Host: "paste.macports.org", + Path: "/", +} + +type remoteLogger struct { + logBigFileChan chan *LogFile + parent *Logger + quitChan chan byte + httpClient *http.Client +} + +func newRemoteLogger(parent *Logger) *remoteLogger { + return &remoteLogger{ + logBigFileChan: make(chan *LogFile, 4), + parent: parent, + quitChan: make(chan byte), + httpClient: &http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + } +} + +func (r *remoteLogger) run() { + for { + select { + case logBigFile := <-r.logBigFileChan: + if logBigFile == nil { + r.parent.LogTextChan <- &LogText{FieldName: "", Text: nil} + return + } + fileInfo, err := os.Stat(logBigFile.Filename) + if err != nil { + // TODO: print err in log + continue + } + file, err := os.Open(logBigFile.Filename) + if err != nil { + continue + } + buf := new(bytes.Buffer) + mimeWriter := multipart.NewWriter(buf) + writer, err := mimeWriter.CreateFormField("paste") + if err != nil { + continue + } + // Max 8 MiB + if fileSize := fileInfo.Size(); fileSize > 8*1024*1024 { + file.Seek(fileSize-8*1024*1024, 0) + } + io.Copy(writer, file) + file.Close() + mimeWriter.Close() + resp, err := r.httpClient.Post(pasteURL.String(), mimeWriter.FormDataContentType(), buf) + if err != nil { + continue + } + // TODO: retry 3 times + loc := resp.Header.Get("Location") + if loc == "" { + continue + } + u, err := pasteURL.Parse(loc) + if err != nil { + continue + } + GlobalLogger.LogTextChan <- &LogText{logBigFile.FieldName + "-pastebin", []byte(u.String())} + } + } +} diff --git a/ci/mpbb.go b/ci/mpbb.go new file mode 100644 index 0000000..58758f2 --- /dev/null +++ b/ci/mpbb.go @@ -0,0 +1,33 @@ +package ci + +import ( + "bufio" + "os" + "os/exec" +) + +// mpbbToLog executes `mpbb` and saves output to a file at logFilePath. +func mpbbToLog(command, port, workDir, logFilePath string) error { + var mpbbCmd *exec.Cmd + if workDir != "" { + mpbbCmd = exec.Command("mpbb", "--work-dir", workDir, command, port) + } else { + mpbbCmd = exec.Command("mpbb", command, port) + } + + logFile, err := os.Create(logFilePath) + if err != nil { + return err + } + logWriter := bufio.NewWriter(logFile) + // other logs in workDir/logs + mpbbCmd.Stdout = logWriter + mpbbCmd.Stderr = logWriter + if err = mpbbCmd.Start(); err != nil { + return err + } + err = mpbbCmd.Wait() + logWriter.Flush() + logFile.Close() + return err +} diff --git a/ci/portlist.go b/ci/portlist.go new file mode 100644 index 0000000..2421fbb --- /dev/null +++ b/ci/portlist.go @@ -0,0 +1,32 @@ +package ci + +import ( + "bufio" + "os/exec" + "regexp" +) + +// List second part of path of Portfiles that was changed in the PR. +func GetChangedPortList() ([]string, error) { + gitCmd := exec.Command("git", "diff", "--name-status", "macports/master...HEAD", "--") + stdout, err := gitCmd.StdoutPipe() + if err != nil { + return nil, err + } + if err = gitCmd.Start(); err != nil { + return nil, err + } + ports := make([]string, 0, 1) + gitRegexp := regexp.MustCompile(`[AM]\t[^\._/][^/]*/([^/]+)/Portfile`) + stdoutScanner := bufio.NewScanner(stdout) + for stdoutScanner.Scan() { + line := stdoutScanner.Text() + if match := gitRegexp.FindStringSubmatch(line); match != nil { + ports = append(ports, match[1]) + } + } + if err = gitCmd.Wait(); err != nil { + return nil, err + } + return ports, nil +} diff --git a/ci/portutil.go b/ci/portutil.go new file mode 100644 index 0000000..b1581a4 --- /dev/null +++ b/ci/portutil.go @@ -0,0 +1,37 @@ +package ci + +import ( + "bufio" + "os/exec" +) + +// Only deactivate ports to save time when a dependency +// is needed across builds. It should be able to avoid +// conflicts. +func DeactivateAllPorts() { + deactivateCmd := exec.Command("port", "deactivate", "active") + deactivateCmd.Start() + deactivateCmd.Wait() +} + +// List all subports of a given port. +func ListSubports(port string) ([]string, error) { + listCmd := exec.Command("port", "-q", "info", "--index", "--line", "--name", port, "subportof:"+port) + stdout, err := listCmd.StdoutPipe() + if err != nil { + return nil, err + } + if err = listCmd.Start(); err != nil { + return nil, err + } + subports := make([]string, 0, 1) + stdoutScanner := bufio.NewScanner(stdout) + for stdoutScanner.Scan() { + line := stdoutScanner.Text() + subports = append(subports, line) + } + if err = listCmd.Wait(); err != nil { + return nil, err + } + return subports, nil +} diff --git a/ci/runner/main.go b/ci/runner/main.go new file mode 100644 index 0000000..2483ee0 --- /dev/null +++ b/ci/runner/main.go @@ -0,0 +1,23 @@ +package main + +import ( + "log" + + "github.com/macports/mpbot-github/ci" + "github.com/macports/mpbot-github/ci/logger" +) + +// Entry point of the CI bot +func main() { + session, err := ci.NewSession() + if err != nil { + log.Fatal(err) + } + err = session.Run() + // Signal the logger to exit + logger.GlobalLogger.LogTextChan <- nil + logger.GlobalLogger.Wait() + if err != nil { + log.Fatal(err) + } +} diff --git a/ci/session.go b/ci/session.go new file mode 100644 index 0000000..643701d --- /dev/null +++ b/ci/session.go @@ -0,0 +1,78 @@ +package ci + +import ( + "errors" + "io/ioutil" + "os/exec" + "strings" + + "github.com/macports/mpbot-github/ci/logger" +) + +// Session is the public interface of the ci package. +// workers are not exported. +type Session struct { + tmpDir string + ports []string + buildCounter uint32 + // TODO: split to lintResults and buildResults or use unified struct type + // TODO: Return error in Run() if build failed + // Collects lint and build failure + results chan string +} + +func NewSession() (*Session, error) { + tmpDir, err := ioutil.TempDir("", "ci-build-") + if err != nil { + return nil, err + } + ports, err := GetChangedPortList() + if err != nil { + // No cleanup in CI + //os.RemoveAll(tmpDir) + return nil, err + } + return &Session{ + tmpDir: tmpDir, + ports: ports, + results: make(chan string), + }, nil +} + +// Run() blocks until all ports are tested and all logs are printed +// or queued in the GlobalLogger. +func (session *Session) Run() error { + logger.GlobalLogger.LogTextChan <- &logger.LogText{"port-list", []byte(strings.Join(session.ports, "\n"))} + if len(session.ports) == 0 { + return nil + } + var err error + bWorker := newBuildWorker(session) + go bWorker.start() + go func() { + for _, port := range session.ports { + bWorker.portChan <- port + } + bWorker.portChan <- "" + }() + + { + lintArgs := make([]string, len(session.ports)+2) + lintArgs[0] = "-p" + lintArgs[1] = "lint" + copy(lintArgs[2:], session.ports) + lintCmd := exec.Command("port", lintArgs...) + out, err := lintCmd.CombinedOutput() + statusString := "success" + if err != nil { + statusString = "fail" + err = errors.New("lint failed") + } + logger.GlobalLogger.LogTextChan <- &logger.LogText{"port-lint-output-" + statusString, out} + } + + if bWorker.wait() != 0 { + err = errors.New("build failed") + } + return err +} diff --git a/ci/worker.go b/ci/worker.go new file mode 100644 index 0000000..142aa5a --- /dev/null +++ b/ci/worker.go @@ -0,0 +1,14 @@ +package ci + +// worker describes a generic worker holding a pointer +// to the Session it belongs to and a wait() method +// that pauses execution until the worker quits. +type worker struct { + session *Session + // signal to unblock wait() + quitChan chan byte +} + +func (worker *worker) wait() byte { + return <-worker.quitChan +} diff --git a/pr/dbutil.go b/pr/dbutil.go new file mode 100644 index 0000000..b8288b5 --- /dev/null +++ b/pr/dbutil.go @@ -0,0 +1,76 @@ +package pr + +import ( + "database/sql" + "log" + + _ "github.com/lib/pq" +) + +type Maintainer struct { + primary string + others []string + openMaintainer bool + noMaintainer bool +} + +var tracDB *sql.DB +var wwwDB *sql.DB + +// Create connections to DBs +func init() { + var err error + tracDB, err = sql.Open("postgres", "host=/tmp dbname=l2dy") + if err != nil { + log.Fatal(err) + } + wwwDB, err = sql.Open("postgres", "host=/tmp dbname=l2dy") + if err != nil { + log.Fatal(err) + } +} + +func GetGitHubHandle(email string) (string, error) { + sid := "" + err := tracDB.QueryRow("SELECT sid "+ + "FROM trac_macports.session_attribute "+ + "WHERE value = $1 "+ + "AND name = 'email' "+ + "AND authenticated = 1 "+ + "LIMIT 1", email). + Scan(&sid) + if err != nil { + return "", err + } + return sid, nil +} + +// GetMaintainer returns the maintainers of a port, +// the primary maintainer is always the first in the slice. +// TODO: parse multi identity per maintainer and email +func GetMaintainer(port string) (*Maintainer, error) { + rows, err := wwwDB.Query("SELECT maintainer, is_primary "+ + "FROM public.maintainers "+ + "WHERE portfile = $1", port) + if err != nil { + return nil, err + } + defer rows.Close() + + maintainer := new(Maintainer) + maintainerCursor := "" + isPrimary := false + + for rows.Next() { + if err := rows.Scan(&maintainerCursor, &isPrimary); err != nil { + return nil, err + } + if isPrimary { + maintainer.primary = maintainerCursor + } else { + maintainer.others = append(maintainer.others, maintainerCursor) + } + } + + return maintainer, nil +} diff --git a/pr/prbot/main.go b/pr/prbot/main.go new file mode 100644 index 0000000..5ca0dc4 --- /dev/null +++ b/pr/prbot/main.go @@ -0,0 +1,20 @@ +package main + +import ( + "flag" + "github.com/macports/mpbot-github/pr/webhook" + "log" + "os" +) + +// Entry point of the PR bot +func main() { + webhookAddr := flag.String("l", "localhost:8081", "listen address for webhook events") + flag.Parse() + hubSecret := []byte(os.Getenv("HUB_WEBHOOK_SECRET")) + if len(hubSecret) == 0 { + log.Fatal("HUB_WEBHOOK_SECRET not found") + } + + webhook.NewReceiver(*webhookAddr, hubSecret).Start() +} diff --git a/pr/webhook/server.go b/pr/webhook/server.go new file mode 100644 index 0000000..9c5809a --- /dev/null +++ b/pr/webhook/server.go @@ -0,0 +1,63 @@ +package webhook + +import ( + "crypto/hmac" + "crypto/sha1" + "encoding/hex" + "io/ioutil" + "net/http" + "strings" +) + +type Receiver struct { + listenAddr string + hubSecret []byte +} + +func NewReceiver(listenAddr string, hubSecret []byte) *Receiver { + return &Receiver{ + listenAddr: listenAddr, + hubSecret: hubSecret, + } +} + +func (receiver *Receiver) Start() { + mux := http.NewServeMux() + + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + sigStr := r.Header.Get("X-Hub-Signature") + if len(sigStr) != 45 || !strings.HasPrefix(sigStr, "sha1=") { + w.WriteHeader(http.StatusBadRequest) + return + } + + sig, err := hex.DecodeString(sigStr[5:]) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + + body, err := ioutil.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + + if !receiver.checkMAC(body, sig) { + w.WriteHeader(http.StatusBadRequest) + return + } + + w.WriteHeader(http.StatusNoContent) + }) + + http.ListenAndServe(receiver.listenAddr, mux) +} + +// checkMAC reports whether messageMAC is a valid HMAC tag for message. +func (receiver *Receiver) checkMAC(message, messageMAC []byte) bool { + mac := hmac.New(sha1.New, receiver.hubSecret) + mac.Write(message) + expectedMAC := mac.Sum(nil) + return hmac.Equal(messageMAC, expectedMAC) +}