Initial project import

ci/buildWorker.go

@@ -0,0 +1,86 @@
+package ci
+
+import (
+	"os/exec"
+	"path"
+
+	"github.com/macports/mpbot-github/ci/logger"
+)
+
+// buildWorker receive ports from portChan
+type buildWorker struct {
+	worker
+	portChan chan string
+}
+
+func newBuildWorker(session *Session) *buildWorker {
+	return &buildWorker{
+		worker:   worker{session: session, quitChan: make(chan byte)},
+		portChan: make(chan string),
+	}
+}
+
+func (worker *buildWorker) start() {
+	returnCode := byte(0)
+	for {
+		select {
+		case port := <-worker.portChan:
+			if port == "" {
+				worker.quitChan <- returnCode
+				return
+			}
+			subports, err := ListSubports(port)
+			if err != nil {
+				returnCode = 1
+				logger.GlobalLogger.LogTextChan <- &logger.LogText{"port-" + port + "-subports-fail", []byte(err.Error())}
+				continue
+			}
+			for _, subport := range subports {
+				DeactivateAllPorts()
+				portTmpDir := path.Join(worker.session.tmpDir, subport)
+				logFilename := path.Join(worker.session.tmpDir, "port-"+subport+"-dep-install.log")
+				err := mpbbToLog("install-dependencies", subport, portTmpDir, logFilename)
+				statusString := "success"
+				if err != nil {
+					if eerr, ok := err.(*exec.ExitError); ok {
+						if !eerr.Success() {
+							returnCode = 1
+							statusString = "fail"
+						}
+					}
+				}
+				logger.GlobalLogger.LogFileChan <- &logger.LogFile{
+					"port-" + subport + "-dep-summary-" + statusString,
+					path.Join(portTmpDir, "logs/dependencies-progress.txt"),
+				}
+				if err != nil {
+					logger.GlobalLogger.LogBigFileChan <- &logger.LogFile{
+						"port-" + port + "-dep-install-output-" + statusString,
+						logFilename,
+					}
+					continue
+				}
+
+				logFilename = path.Join(worker.session.tmpDir, "port-"+subport+"-install.log")
+				mpbbToLog("install-port", subport, portTmpDir, logFilename)
+				statusString = "success"
+				if err != nil {
+					if eerr, ok := err.(*exec.ExitError); ok {
+						if !eerr.Success() {
+							returnCode = 1
+							statusString = "fail"
+						}
+					}
+				}
+				logger.GlobalLogger.LogFileChan <- &logger.LogFile{
+					"port-" + subport + "-install-summary-" + statusString,
+					path.Join(portTmpDir, "logs/ports-progress.txt"),
+				}
+				logger.GlobalLogger.LogBigFileChan <- &logger.LogFile{
+					"port-" + port + "-install-output-" + statusString,
+					logFilename,
+				}
+			}
+		}
+	}
+}

ci/logger/log.go

@@ -0,0 +1,94 @@
+package logger
+
+import (
+	"io"
+	"mime/multipart"
+	"os"
+	"time"
+)
+
+// This is the actual logger used by the CI bot
+var GlobalLogger *Logger = newLogger(os.Stdout)
+
+func init() {
+	go GlobalLogger.Run()
+}
+
+// Logger only exits when nil is send to its LogTextChan.
+type Logger struct {
+	LogTextChan    chan *LogText
+	LogFileChan    chan *LogFile
+	LogBigFileChan chan *LogFile
+	mimeWriter     *multipart.Writer
+	quitChan       chan byte
+	remoteLogger   *remoteLogger
+}
+
+type LogFile struct {
+	FieldName, Filename string
+}
+
+type LogText struct {
+	FieldName string
+	Text      []byte
+}
+
+func newLogger(w io.Writer) *Logger {
+	logger := &Logger{
+		mimeWriter:     multipart.NewWriter(w),
+		quitChan:       make(chan byte),
+		LogTextChan:    make(chan *LogText, 4),
+		LogFileChan:    make(chan *LogFile, 4),
+		LogBigFileChan: make(chan *LogFile, 4),
+	}
+	logger.remoteLogger = newRemoteLogger(logger)
+	return logger
+}
+
+func (l *Logger) Run() {
+	go l.remoteLogger.run()
+	for {
+		select {
+		case logFile := <-l.LogFileChan:
+			if logFile == nil {
+				continue
+			}
+			writer, err := l.mimeWriter.CreateFormField(logFile.FieldName)
+			if err != nil {
+				continue
+			}
+			file, err := os.Open(logFile.Filename)
+			if err != nil {
+				continue
+			}
+
+			io.Copy(writer, file)
+
+			file.Close()
+			os.Remove(logFile.Filename)
+		case logText := <-l.LogTextChan:
+			if logText == nil {
+				l.remoteLogger.logBigFileChan <- nil
+				continue
+			}
+			if logText.FieldName == "" {
+				l.mimeWriter.Close()
+				l.quitChan <- 0
+				return
+			}
+			writer, err := l.mimeWriter.CreateFormField(logText.FieldName)
+			if err != nil {
+				continue
+			}
+			_, err = writer.Write(logText.Text)
+		case logBigFile := <-l.LogBigFileChan:
+			l.remoteLogger.logBigFileChan <- logBigFile
+		case <-time.After(time.Minute * 5):
+			l.LogTextChan <- &LogText{"keep-alive", []byte{}}
+		}
+	}
+}
+
+func (l *Logger) Wait() byte {
+	return <-l.quitChan
+}

ci/logger/remoteLog.go

@@ -0,0 +1,84 @@
+package logger
+
+import (
+	"bytes"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"net/url"
+	"os"
+)
+
+var pasteURL = &url.URL{
+	Scheme: "https",
+	Host:   "paste.macports.org",
+	Path:   "/",
+}
+
+type remoteLogger struct {
+	logBigFileChan chan *LogFile
+	parent         *Logger
+	quitChan       chan byte
+	httpClient     *http.Client
+}
+
+func newRemoteLogger(parent *Logger) *remoteLogger {
+	return &remoteLogger{
+		logBigFileChan: make(chan *LogFile, 4),
+		parent:         parent,
+		quitChan:       make(chan byte),
+		httpClient: &http.Client{
+			CheckRedirect: func(req *http.Request, via []*http.Request) error {
+				return http.ErrUseLastResponse
+			},
+		},
+	}
+}
+
+func (r *remoteLogger) run() {
+	for {
+		select {
+		case logBigFile := <-r.logBigFileChan:
+			if logBigFile == nil {
+				r.parent.LogTextChan <- &LogText{FieldName: "", Text: nil}
+				return
+			}
+			fileInfo, err := os.Stat(logBigFile.Filename)
+			if err != nil {
+				// TODO: print err in log
+				continue
+			}
+			file, err := os.Open(logBigFile.Filename)
+			if err != nil {
+				continue
+			}
+			buf := new(bytes.Buffer)
+			mimeWriter := multipart.NewWriter(buf)
+			writer, err := mimeWriter.CreateFormField("paste")
+			if err != nil {
+				continue
+			}
+			// Max 8 MiB
+			if fileSize := fileInfo.Size(); fileSize > 8*1024*1024 {
+				file.Seek(fileSize-8*1024*1024, 0)
+			}
+			io.Copy(writer, file)
+			file.Close()
+			mimeWriter.Close()
+			resp, err := r.httpClient.Post(pasteURL.String(), mimeWriter.FormDataContentType(), buf)
+			if err != nil {
+				continue
+			}
+			// TODO: retry 3 times
+			loc := resp.Header.Get("Location")
+			if loc == "" {
+				continue
+			}
+			u, err := pasteURL.Parse(loc)
+			if err != nil {
+				continue
+			}
+			GlobalLogger.LogTextChan <- &LogText{logBigFile.FieldName + "-pastebin", []byte(u.String())}
+		}
+	}
+}

ci/mpbb.go

@@ -0,0 +1,33 @@
+package ci
+
+import (
+	"bufio"
+	"os"
+	"os/exec"
+)
+
+// mpbbToLog executes `mpbb` and saves output to a file at logFilePath.
+func mpbbToLog(command, port, workDir, logFilePath string) error {
+	var mpbbCmd *exec.Cmd
+	if workDir != "" {
+		mpbbCmd = exec.Command("mpbb", "--work-dir", workDir, command, port)
+	} else {
+		mpbbCmd = exec.Command("mpbb", command, port)
+	}
+
+	logFile, err := os.Create(logFilePath)
+	if err != nil {
+		return err
+	}
+	logWriter := bufio.NewWriter(logFile)
+	// other logs in workDir/logs
+	mpbbCmd.Stdout = logWriter
+	mpbbCmd.Stderr = logWriter
+	if err = mpbbCmd.Start(); err != nil {
+		return err
+	}
+	err = mpbbCmd.Wait()
+	logWriter.Flush()
+	logFile.Close()
+	return err
+}

ci/portlist.go

@@ -0,0 +1,32 @@
+package ci
+
+import (
+	"bufio"
+	"os/exec"
+	"regexp"
+)
+
+// List second part of path of Portfiles that was changed in the PR.
+func GetChangedPortList() ([]string, error) {
+	gitCmd := exec.Command("git", "diff", "--name-status", "macports/master...HEAD", "--")
+	stdout, err := gitCmd.StdoutPipe()
+	if err != nil {
+		return nil, err
+	}
+	if err = gitCmd.Start(); err != nil {
+		return nil, err
+	}
+	ports := make([]string, 0, 1)
+	gitRegexp := regexp.MustCompile(`[AM]\t[^\._/][^/]*/([^/]+)/Portfile`)
+	stdoutScanner := bufio.NewScanner(stdout)
+	for stdoutScanner.Scan() {
+		line := stdoutScanner.Text()
+		if match := gitRegexp.FindStringSubmatch(line); match != nil {
+			ports = append(ports, match[1])
+		}
+	}
+	if err = gitCmd.Wait(); err != nil {
+		return nil, err
+	}
+	return ports, nil
+}

ci/portutil.go

@@ -0,0 +1,37 @@
+package ci
+
+import (
+	"bufio"
+	"os/exec"
+)
+
+// Only deactivate ports to save time when a dependency
+// is needed across builds. It should be able to avoid
+// conflicts.
+func DeactivateAllPorts() {
+	deactivateCmd := exec.Command("port", "deactivate", "active")
+	deactivateCmd.Start()
+	deactivateCmd.Wait()
+}
+
+// List all subports of a given port.
+func ListSubports(port string) ([]string, error) {
+	listCmd := exec.Command("port", "-q", "info", "--index", "--line", "--name", port, "subportof:"+port)
+	stdout, err := listCmd.StdoutPipe()
+	if err != nil {
+		return nil, err
+	}
+	if err = listCmd.Start(); err != nil {
+		return nil, err
+	}
+	subports := make([]string, 0, 1)
+	stdoutScanner := bufio.NewScanner(stdout)
+	for stdoutScanner.Scan() {
+		line := stdoutScanner.Text()
+		subports = append(subports, line)
+	}
+	if err = listCmd.Wait(); err != nil {
+		return nil, err
+	}
+	return subports, nil
+}

ci/runner/main.go

@@ -0,0 +1,23 @@
+package main
+
+import (
+	"log"
+
+	"github.com/macports/mpbot-github/ci"
+	"github.com/macports/mpbot-github/ci/logger"
+)
+
+// Entry point of the CI bot
+func main() {
+	session, err := ci.NewSession()
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = session.Run()
+	// Signal the logger to exit
+	logger.GlobalLogger.LogTextChan <- nil
+	logger.GlobalLogger.Wait()
+	if err != nil {
+		log.Fatal(err)
+	}
+}

ci/session.go

@@ -0,0 +1,78 @@
+package ci
+
+import (
+	"errors"
+	"io/ioutil"
+	"os/exec"
+	"strings"
+
+	"github.com/macports/mpbot-github/ci/logger"
+)
+
+// Session is the public interface of the ci package.
+// workers are not exported.
+type Session struct {
+	tmpDir       string
+	ports        []string
+	buildCounter uint32
+	// TODO: split to lintResults and buildResults or use unified struct type
+	// TODO: Return error in Run() if build failed
+	// Collects lint and build failure
+	results chan string
+}
+
+func NewSession() (*Session, error) {
+	tmpDir, err := ioutil.TempDir("", "ci-build-")
+	if err != nil {
+		return nil, err
+	}
+	ports, err := GetChangedPortList()
+	if err != nil {
+		// No cleanup in CI
+		//os.RemoveAll(tmpDir)
+		return nil, err
+	}
+	return &Session{
+		tmpDir:  tmpDir,
+		ports:   ports,
+		results: make(chan string),
+	}, nil
+}
+
+// Run() blocks until all ports are tested and all logs are printed
+// or queued in the GlobalLogger.
+func (session *Session) Run() error {
+	logger.GlobalLogger.LogTextChan <- &logger.LogText{"port-list", []byte(strings.Join(session.ports, "\n"))}
+	if len(session.ports) == 0 {
+		return nil
+	}
+	var err error
+	bWorker := newBuildWorker(session)
+	go bWorker.start()
+	go func() {
+		for _, port := range session.ports {
+			bWorker.portChan <- port
+		}
+		bWorker.portChan <- ""
+	}()
+
+	{
+		lintArgs := make([]string, len(session.ports)+2)
+		lintArgs[0] = "-p"
+		lintArgs[1] = "lint"
+		copy(lintArgs[2:], session.ports)
+		lintCmd := exec.Command("port", lintArgs...)
+		out, err := lintCmd.CombinedOutput()
+		statusString := "success"
+		if err != nil {
+			statusString = "fail"
+			err = errors.New("lint failed")
+		}
+		logger.GlobalLogger.LogTextChan <- &logger.LogText{"port-lint-output-" + statusString, out}
+	}
+
+	if bWorker.wait() != 0 {
+		err = errors.New("build failed")
+	}
+	return err
+}

ci/worker.go

@@ -0,0 +1,14 @@
+package ci
+
+// worker describes a generic worker holding a pointer
+// to the Session it belongs to and a wait() method
+// that pauses execution until the worker quits.
+type worker struct {
+	session *Session
+	// signal to unblock wait()
+	quitChan chan byte
+}
+
+func (worker *worker) wait() byte {
+	return <-worker.quitChan
+}

pr/dbutil.go

@@ -0,0 +1,76 @@
+package pr
+
+import (
+	"database/sql"
+	"log"
+
+	_ "github.com/lib/pq"
+)
+
+type Maintainer struct {
+	primary        string
+	others         []string
+	openMaintainer bool
+	noMaintainer   bool
+}
+
+var tracDB *sql.DB
+var wwwDB *sql.DB
+
+// Create connections to DBs
+func init() {
+	var err error
+	tracDB, err = sql.Open("postgres", "host=/tmp dbname=l2dy")
+	if err != nil {
+		log.Fatal(err)
+	}
+	wwwDB, err = sql.Open("postgres", "host=/tmp dbname=l2dy")
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func GetGitHubHandle(email string) (string, error) {
+	sid := ""
+	err := tracDB.QueryRow("SELECT sid "+
+		"FROM trac_macports.session_attribute "+
+		"WHERE value = $1 "+
+		"AND name = 'email' "+
+		"AND authenticated = 1 "+
+		"LIMIT 1", email).
+		Scan(&sid)
+	if err != nil {
+		return "", err
+	}
+	return sid, nil
+}
+
+// GetMaintainer returns the maintainers of a port,
+// the primary maintainer is always the first in the slice.
+// TODO: parse multi identity per maintainer and email
+func GetMaintainer(port string) (*Maintainer, error) {
+	rows, err := wwwDB.Query("SELECT maintainer, is_primary "+
+		"FROM public.maintainers "+
+		"WHERE portfile = $1", port)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	maintainer := new(Maintainer)
+	maintainerCursor := ""
+	isPrimary := false
+
+	for rows.Next() {
+		if err := rows.Scan(&maintainerCursor, &isPrimary); err != nil {
+			return nil, err
+		}
+		if isPrimary {
+			maintainer.primary = maintainerCursor
+		} else {
+			maintainer.others = append(maintainer.others, maintainerCursor)
+		}
+	}
+
+	return maintainer, nil
+}

pr/prbot/main.go

@@ -0,0 +1,20 @@
+package main
+
+import (
+	"flag"
+	"github.com/macports/mpbot-github/pr/webhook"
+	"log"
+	"os"
+)
+
+// Entry point of the PR bot
+func main() {
+	webhookAddr := flag.String("l", "localhost:8081", "listen address for webhook events")
+	flag.Parse()
+	hubSecret := []byte(os.Getenv("HUB_WEBHOOK_SECRET"))
+	if len(hubSecret) == 0 {
+		log.Fatal("HUB_WEBHOOK_SECRET not found")
+	}
+
+	webhook.NewReceiver(*webhookAddr, hubSecret).Start()
+}

pr/webhook/server.go

@@ -0,0 +1,63 @@
+package webhook
+
+import (
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/hex"
+	"io/ioutil"
+	"net/http"
+	"strings"
+)
+
+type Receiver struct {
+	listenAddr string
+	hubSecret  []byte
+}
+
+func NewReceiver(listenAddr string, hubSecret []byte) *Receiver {
+	return &Receiver{
+		listenAddr: listenAddr,
+		hubSecret:  hubSecret,
+	}
+}
+
+func (receiver *Receiver) Start() {
+	mux := http.NewServeMux()
+
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		sigStr := r.Header.Get("X-Hub-Signature")
+		if len(sigStr) != 45 || !strings.HasPrefix(sigStr, "sha1=") {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+
+		sig, err := hex.DecodeString(sigStr[5:])
+		if err != nil {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+
+		body, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+
+		if !receiver.checkMAC(body, sig) {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+
+		w.WriteHeader(http.StatusNoContent)
+	})
+
+	http.ListenAndServe(receiver.listenAddr, mux)
+}
+
+// checkMAC reports whether messageMAC is a valid HMAC tag for message.
+func (receiver *Receiver) checkMAC(message, messageMAC []byte) bool {
+	mac := hmac.New(sha1.New, receiver.hubSecret)
+	mac.Write(message)
+	expectedMAC := mac.Sum(nil)
+	return hmac.Equal(messageMAC, expectedMAC)
+}