pr: Travis webhook support

ci/logger/constants/constants.go

@@ -0,0 +1,3 @@
+package constants
+
+const MIMEBoundary = "9bba227c544541bbafbe7a4fc806bab5489eae04f16d9303cac42e9eff2e"

ci/logger/log.go

@@ -5,10 +5,12 @@ import (
 	"mime/multipart"
 	"os"
 	"time"
+
+	"github.com/macports/mpbot-github/ci/logger/constants"
 )
 
 // This is the actual logger used by the CI bot
-var GlobalLogger *Logger = newLogger(os.Stdout)
+var GlobalLogger = newLogger(os.Stdout)
 
 func init() {
 	go GlobalLogger.Run()
@@ -38,6 +40,7 @@ func newLogger(w io.Writer) *Logger {
 		mimeWriter: multipart.NewWriter(w),
 		quitChan:   make(chan byte),
 	}
+	logger.mimeWriter.SetBoundary(constants.MIMEBoundary)
 	logger.remoteLogger = newRemoteLogger(logger)
 	return logger
 }

pr/prbot/main.go

@@ -33,7 +33,11 @@ func main() {
 
 	dbHelper, err := db.NewDBHelper()
 	if err != nil {
-		log.Fatal(err)
+		if prodFlag {
+			log.Fatal(err)
+		} else {
+			log.Println(err)
+		}
 	}
 
 	cronManager := cron.Manager{

pr/webhook/server.go

@@ -2,9 +2,15 @@ package webhook
 
 import (
 	"context"
+	"crypto"
 	"crypto/hmac"
+	"crypto/rsa"
 	"crypto/sha1"
+	"crypto/x509"
+	"encoding/base64"
 	"encoding/hex"
+	"encoding/json"
+	"encoding/pem"
 	"io/ioutil"
 	"log"
 	"net/http"
@@ -17,15 +23,17 @@ import (
 )
 
 type Receiver struct {
-	server       *http.Server
-	hookSecret   []byte
-	production   bool
-	testing      bool
-	githubClient githubapi.Client
-	dbHelper     db.DBHelper
-	wg           sync.WaitGroup
-	members      *map[string]bool
-	membersLock  sync.RWMutex
+	server           *http.Server
+	hookSecret       []byte
+	production       bool
+	testing          bool
+	githubClient     githubapi.Client
+	dbHelper         db.DBHelper
+	wg               sync.WaitGroup
+	members          *map[string]bool
+	membersLock      sync.RWMutex
+	travisPubKey     *rsa.PublicKey
+	travisPubKeyLock sync.RWMutex
 }
 
 func NewReceiver(listenAddr string, hookSecret []byte, botSecret string, production bool, dbHelper db.DBHelper) *Receiver {
@@ -88,7 +96,54 @@ func (receiver *Receiver) Start() {
 		w.WriteHeader(http.StatusNoContent)
 	})
 
+	mux.HandleFunc("/travis", func(w http.ResponseWriter, r *http.Request) {
+		sigStr := r.Header.Get("Signature")
+
+		sig, err := base64.StdEncoding.DecodeString(sigStr)
+		if err != nil {
+			log.Println(err)
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+
+		receiver.wg.Add(1)
+
+		body := []byte(r.FormValue("payload"))
+
+		if len(body) == 0 {
+			w.WriteHeader(http.StatusBadRequest)
+			receiver.wg.Done()
+			return
+		}
+
+		hashed := sha1.Sum(body)
+		receiver.travisPubKeyLock.RLock()
+		err = rsa.VerifyPKCS1v15(receiver.travisPubKey, crypto.SHA1, hashed[:], sig)
+		receiver.travisPubKeyLock.RUnlock()
+		if err != nil {
+			log.Println(err)
+			w.WriteHeader(http.StatusBadRequest)
+			receiver.wg.Done()
+			return
+		}
+
+		var payload TravisWebhookPayload
+
+		err = json.Unmarshal(body, &payload)
+		if err != nil {
+			log.Println(err)
+			w.WriteHeader(http.StatusBadRequest)
+			receiver.wg.Done()
+			return
+		}
+
+		go receiver.handleTravisWebhook(payload)
+
+		w.WriteHeader(http.StatusNoContent)
+	})
+
 	go receiver.updateMembers()
+	receiver.updateTravisPubKey()
 
 	receiver.server.Handler = mux
 	receiver.server.ListenAndServe()
@@ -99,6 +154,27 @@ func (receiver *Receiver) Shutdown() {
 	receiver.wg.Wait()
 }
 
+func (receiver *Receiver) updateTravisPubKey() {
+	const travisPubKeyPEM = "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvtjdLkS+FP+0fPC09j25\ny/PiuYDDivIT86COVedvlElk99BBYTrqNaJybxjXbIZ1Q6xFNhOY+iTcBr4E1zJu\ntizF3Xi0V9tOuP/M8Wn4Y/1lCWbQKlWrNQuqNBmhovF4K3mDCYswVbpgTmp+JQYu\nBm9QMdieZMNry5s6aiMA9aSjDlNyedvSENYo18F+NYg1J0C0JiPYTxheCb4optr1\n5xNzFKhAkuGs4XTOA5C7Q06GCKtDNf44s/CVE30KODUxBi0MCKaxiXw/yy55zxX2\n/YdGphIyQiA5iO1986ZmZCLLW8udz9uhW5jUr3Jlp9LbmphAC61bVSf4ou2YsJaN\n0QIDAQAB\n-----END PUBLIC KEY-----"
+
+	p, _ := pem.Decode([]byte(travisPubKeyPEM))
+	if p == nil || p.Type != "PUBLIC KEY" {
+		log.Println("travis: invalid public key")
+		return
+	}
+
+	travisPubKey, err := x509.ParsePKIXPublicKey(p.Bytes)
+	if err != nil {
+		return
+	}
+
+	if pubKey, ok := travisPubKey.(*rsa.PublicKey); ok {
+		receiver.travisPubKeyLock.Lock()
+		receiver.travisPubKey = pubKey
+		receiver.travisPubKeyLock.Unlock()
+	}
+}
+
 func (receiver *Receiver) updateMembers() {
 	for ; ; time.Sleep(24 * time.Hour) {
 		users, err := receiver.githubClient.ListOrgMembers("macports")

pr/webhook/travis.go

@@ -0,0 +1,181 @@
+package webhook
+
+import (
+	"bufio"
+	"io"
+	"io/ioutil"
+	"log"
+	"mime/multipart"
+	"net/http"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/macports/mpbot-github/ci/logger/constants"
+)
+
+type TravisWebhookPayload struct {
+	ID                int    `json:"id"`
+	Number            string `json:"number"`
+	Type              string `json:"type"`
+	State             string `json:"state"`
+	Status            int    `json:"status"`
+	Result            int    `json:"result"`
+	StatusMessage     string `json:"status_message"`
+	ResultMessage     string `json:"result_message"`
+	Duration          int    `json:"duration"`
+	BuildURL          string `json:"build_url"`
+	Branch            string `json:"branch"`
+	PullRequest       bool   `json:"pull_request"`
+	PullRequestNumber int    `json:"pull_request_number"`
+	PullRequestTitle  string `json:"pull_request_title"`
+	Repository        struct {
+		ID        int    `json:"id"`
+		Name      string `json:"name"`
+		OwnerName string `json:"owner_name"`
+	} `json:"repository"`
+	Matrix []struct {
+		ID       int    `json:"id"`
+		ParentID int    `json:"parent_id"`
+		Number   string `json:"number"`
+		State    string `json:"state"`
+		Config   struct {
+			Os       string `json:"os"`
+			OsxImage string `json:"osx_image"`
+		} `json:"config"`
+		Status       int  `json:"status"`
+		Result       int  `json:"result"`
+		AllowFailure bool `json:"allow_failure"`
+	} `json:"matrix"`
+}
+
+func (receiver *Receiver) handleTravisWebhook(payload TravisWebhookPayload) {
+	defer func() {
+		if r := recover(); r != nil {
+			log.Println(r)
+		}
+
+		if !receiver.testing {
+			receiver.wg.Done()
+		}
+	}()
+
+	if !payload.PullRequest {
+		return
+	}
+
+	if payload.Repository.OwnerName != "macports" && payload.Repository.OwnerName != "macports-staging" {
+		return
+	}
+
+	log.Println("PR #" + strconv.Itoa(payload.PullRequestNumber) + " " + payload.ResultMessage + " on Travis CI")
+
+	comment := "[Travis Build #" + payload.Number + "](" + payload.BuildURL + ") " + payload.ResultMessage + ".\n\n"
+	timeOut := false
+	lintDone := false
+
+	log.Println("Processing " + strconv.Itoa(len(payload.Matrix)) + " job(s)")
+
+	for _, job := range payload.Matrix {
+		req, err := http.NewRequest(
+			"GET",
+			"https://api.travis-ci.org/job/"+strconv.Itoa(job.ID)+"/log",
+			nil,
+		)
+		if err != nil {
+			continue
+		}
+
+		req.Header.Set("Travis-API-Version", "3")
+		req.Header.Set("Accept", "text/plain")
+
+		log.Println("Fetching logs for job #" + strconv.Itoa(job.ID))
+
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			continue
+		}
+
+		bufReader := bufio.NewReader(resp.Body)
+
+		for {
+			line, err := bufReader.ReadString('\n')
+			if err != nil {
+				break
+			}
+			if strings.Contains(line, "$ sudo ./runner") {
+				break
+			}
+		}
+
+		body, err := ioutil.ReadAll(bufReader)
+		resp.Body.Close()
+		if err != nil {
+			continue
+		}
+		bodyStr := string(body)
+		bodyStr = strings.Replace(bodyStr, "\n\n\n\r", "\n", -1)
+		bodyStr = strings.Replace(bodyStr, "\r", "", -1)
+
+		mr := multipart.NewReader(strings.NewReader(bodyStr), constants.MIMEBoundary)
+		for {
+			p, err := mr.NextPart()
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				log.Println(err)
+				return
+			}
+			pName := p.FormName()
+			content, err := ioutil.ReadAll(p)
+			if err == io.ErrUnexpectedEOF {
+				if strings.Contains(
+					string(content),
+					"The job exceeded the maximum time limit for jobs, and has been terminated.",
+				) {
+					timeOut = true
+				}
+				break
+			}
+			if err != nil {
+				log.Println(err)
+				continue
+			}
+			if pName == "keep-alive" {
+				continue
+			}
+			if err != nil {
+				log.Println(err)
+				continue
+			}
+			if strings.HasPrefix(pName, "port-lint-output-") && len(content) > 0 && !lintDone {
+				comment += "<details><summary>Lint results</summary>\n\n```\n" + string(content) + "```\n</details>\n\n<br>\n\n"
+				lintDone = true
+			}
+			if strings.HasSuffix(pName, "-pastebin") {
+				pastebinRegex := regexp.MustCompile(`^port-(.*)(-dep)?-install-output-(success|fail)-pastebin$`)
+				pbInfo := pastebinRegex.FindStringSubmatch(pName)
+				if pbInfo == nil {
+					continue
+				}
+				comment += "Port " + pbInfo[1]
+				if pbInfo[2] == "-dep" {
+					comment += "'s dependencies"
+				}
+				comment += " **" + pbInfo[3] + "** on " + job.Config.OsxImage + ". [Log](" + string(content) + ")\n"
+			}
+		}
+	}
+
+	if timeOut {
+		comment += "The build timed out."
+	}
+
+	receiver.githubClient.CreateComment(
+		payload.Repository.OwnerName,
+		payload.Repository.Name,
+		payload.PullRequestNumber,
+		&comment,
+	)
+}