From f47049884667d9b7cdb59608448385fbcb22cb3d Mon Sep 17 00:00:00 2001 From: Zero King Date: Tue, 13 Mar 2018 16:53:27 +0000 Subject: [PATCH] pr: Travis webhook support --- ci/logger/constants/constants.go | 3 + ci/logger/log.go | 5 +- pr/prbot/main.go | 6 +- pr/webhook/server.go | 94 ++++++++++++++-- pr/webhook/travis.go | 181 +++++++++++++++++++++++++++++++ 5 files changed, 278 insertions(+), 11 deletions(-) create mode 100644 ci/logger/constants/constants.go create mode 100644 pr/webhook/travis.go diff --git a/ci/logger/constants/constants.go b/ci/logger/constants/constants.go new file mode 100644 index 0000000..0818bd2 --- /dev/null +++ b/ci/logger/constants/constants.go @@ -0,0 +1,3 @@ +package constants + +const MIMEBoundary = "9bba227c544541bbafbe7a4fc806bab5489eae04f16d9303cac42e9eff2e" diff --git a/ci/logger/log.go b/ci/logger/log.go index e2c9162..35d2400 100644 --- a/ci/logger/log.go +++ b/ci/logger/log.go @@ -5,10 +5,12 @@ import ( "mime/multipart" "os" "time" + + "github.com/macports/mpbot-github/ci/logger/constants" ) // This is the actual logger used by the CI bot -var GlobalLogger *Logger = newLogger(os.Stdout) +var GlobalLogger = newLogger(os.Stdout) func init() { go GlobalLogger.Run() @@ -38,6 +40,7 @@ func newLogger(w io.Writer) *Logger { mimeWriter: multipart.NewWriter(w), quitChan: make(chan byte), } + logger.mimeWriter.SetBoundary(constants.MIMEBoundary) logger.remoteLogger = newRemoteLogger(logger) return logger } diff --git a/pr/prbot/main.go b/pr/prbot/main.go index 79be55f..b7206d9 100644 --- a/pr/prbot/main.go +++ b/pr/prbot/main.go @@ -33,7 +33,11 @@ func main() { dbHelper, err := db.NewDBHelper() if err != nil { - log.Fatal(err) + if prodFlag { + log.Fatal(err) + } else { + log.Println(err) + } } cronManager := cron.Manager{ diff --git a/pr/webhook/server.go b/pr/webhook/server.go index cfe73e3..39fa666 100644 --- a/pr/webhook/server.go +++ b/pr/webhook/server.go @@ -2,9 +2,15 @@ package webhook import ( "context" + "crypto" "crypto/hmac" + "crypto/rsa" "crypto/sha1" + "crypto/x509" + "encoding/base64" "encoding/hex" + "encoding/json" + "encoding/pem" "io/ioutil" "log" "net/http" @@ -17,15 +23,17 @@ import ( ) type Receiver struct { - server *http.Server - hookSecret []byte - production bool - testing bool - githubClient githubapi.Client - dbHelper db.DBHelper - wg sync.WaitGroup - members *map[string]bool - membersLock sync.RWMutex + server *http.Server + hookSecret []byte + production bool + testing bool + githubClient githubapi.Client + dbHelper db.DBHelper + wg sync.WaitGroup + members *map[string]bool + membersLock sync.RWMutex + travisPubKey *rsa.PublicKey + travisPubKeyLock sync.RWMutex } func NewReceiver(listenAddr string, hookSecret []byte, botSecret string, production bool, dbHelper db.DBHelper) *Receiver { @@ -88,7 +96,54 @@ func (receiver *Receiver) Start() { w.WriteHeader(http.StatusNoContent) }) + mux.HandleFunc("/travis", func(w http.ResponseWriter, r *http.Request) { + sigStr := r.Header.Get("Signature") + + sig, err := base64.StdEncoding.DecodeString(sigStr) + if err != nil { + log.Println(err) + w.WriteHeader(http.StatusBadRequest) + return + } + + receiver.wg.Add(1) + + body := []byte(r.FormValue("payload")) + + if len(body) == 0 { + w.WriteHeader(http.StatusBadRequest) + receiver.wg.Done() + return + } + + hashed := sha1.Sum(body) + receiver.travisPubKeyLock.RLock() + err = rsa.VerifyPKCS1v15(receiver.travisPubKey, crypto.SHA1, hashed[:], sig) + receiver.travisPubKeyLock.RUnlock() + if err != nil { + log.Println(err) + w.WriteHeader(http.StatusBadRequest) + receiver.wg.Done() + return + } + + var payload TravisWebhookPayload + + err = json.Unmarshal(body, &payload) + if err != nil { + log.Println(err) + w.WriteHeader(http.StatusBadRequest) + receiver.wg.Done() + return + } + + go receiver.handleTravisWebhook(payload) + + w.WriteHeader(http.StatusNoContent) + }) + go receiver.updateMembers() + receiver.updateTravisPubKey() receiver.server.Handler = mux receiver.server.ListenAndServe() @@ -99,6 +154,27 @@ func (receiver *Receiver) Shutdown() { receiver.wg.Wait() } +func (receiver *Receiver) updateTravisPubKey() { + const travisPubKeyPEM = "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvtjdLkS+FP+0fPC09j25\ny/PiuYDDivIT86COVedvlElk99BBYTrqNaJybxjXbIZ1Q6xFNhOY+iTcBr4E1zJu\ntizF3Xi0V9tOuP/M8Wn4Y/1lCWbQKlWrNQuqNBmhovF4K3mDCYswVbpgTmp+JQYu\nBm9QMdieZMNry5s6aiMA9aSjDlNyedvSENYo18F+NYg1J0C0JiPYTxheCb4optr1\n5xNzFKhAkuGs4XTOA5C7Q06GCKtDNf44s/CVE30KODUxBi0MCKaxiXw/yy55zxX2\n/YdGphIyQiA5iO1986ZmZCLLW8udz9uhW5jUr3Jlp9LbmphAC61bVSf4ou2YsJaN\n0QIDAQAB\n-----END PUBLIC KEY-----" + + p, _ := pem.Decode([]byte(travisPubKeyPEM)) + if p == nil || p.Type != "PUBLIC KEY" { + log.Println("travis: invalid public key") + return + } + + travisPubKey, err := x509.ParsePKIXPublicKey(p.Bytes) + if err != nil { + return + } + + if pubKey, ok := travisPubKey.(*rsa.PublicKey); ok { + receiver.travisPubKeyLock.Lock() + receiver.travisPubKey = pubKey + receiver.travisPubKeyLock.Unlock() + } +} + func (receiver *Receiver) updateMembers() { for ; ; time.Sleep(24 * time.Hour) { users, err := receiver.githubClient.ListOrgMembers("macports") diff --git a/pr/webhook/travis.go b/pr/webhook/travis.go new file mode 100644 index 0000000..377f0b3 --- /dev/null +++ b/pr/webhook/travis.go @@ -0,0 +1,181 @@ +package webhook + +import ( + "bufio" + "io" + "io/ioutil" + "log" + "mime/multipart" + "net/http" + "regexp" + "strconv" + "strings" + + "github.com/macports/mpbot-github/ci/logger/constants" +) + +type TravisWebhookPayload struct { + ID int `json:"id"` + Number string `json:"number"` + Type string `json:"type"` + State string `json:"state"` + Status int `json:"status"` + Result int `json:"result"` + StatusMessage string `json:"status_message"` + ResultMessage string `json:"result_message"` + Duration int `json:"duration"` + BuildURL string `json:"build_url"` + Branch string `json:"branch"` + PullRequest bool `json:"pull_request"` + PullRequestNumber int `json:"pull_request_number"` + PullRequestTitle string `json:"pull_request_title"` + Repository struct { + ID int `json:"id"` + Name string `json:"name"` + OwnerName string `json:"owner_name"` + } `json:"repository"` + Matrix []struct { + ID int `json:"id"` + ParentID int `json:"parent_id"` + Number string `json:"number"` + State string `json:"state"` + Config struct { + Os string `json:"os"` + OsxImage string `json:"osx_image"` + } `json:"config"` + Status int `json:"status"` + Result int `json:"result"` + AllowFailure bool `json:"allow_failure"` + } `json:"matrix"` +} + +func (receiver *Receiver) handleTravisWebhook(payload TravisWebhookPayload) { + defer func() { + if r := recover(); r != nil { + log.Println(r) + } + + if !receiver.testing { + receiver.wg.Done() + } + }() + + if !payload.PullRequest { + return + } + + if payload.Repository.OwnerName != "macports" && payload.Repository.OwnerName != "macports-staging" { + return + } + + log.Println("PR #" + strconv.Itoa(payload.PullRequestNumber) + " " + payload.ResultMessage + " on Travis CI") + + comment := "[Travis Build #" + payload.Number + "](" + payload.BuildURL + ") " + payload.ResultMessage + ".\n\n" + timeOut := false + lintDone := false + + log.Println("Processing " + strconv.Itoa(len(payload.Matrix)) + " job(s)") + + for _, job := range payload.Matrix { + req, err := http.NewRequest( + "GET", + "https://api.travis-ci.org/job/"+strconv.Itoa(job.ID)+"/log", + nil, + ) + if err != nil { + continue + } + + req.Header.Set("Travis-API-Version", "3") + req.Header.Set("Accept", "text/plain") + + log.Println("Fetching logs for job #" + strconv.Itoa(job.ID)) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + continue + } + + bufReader := bufio.NewReader(resp.Body) + + for { + line, err := bufReader.ReadString('\n') + if err != nil { + break + } + if strings.Contains(line, "$ sudo ./runner") { + break + } + } + + body, err := ioutil.ReadAll(bufReader) + resp.Body.Close() + if err != nil { + continue + } + bodyStr := string(body) + bodyStr = strings.Replace(bodyStr, "\n\n\n\r", "\n", -1) + bodyStr = strings.Replace(bodyStr, "\r", "", -1) + + mr := multipart.NewReader(strings.NewReader(bodyStr), constants.MIMEBoundary) + for { + p, err := mr.NextPart() + if err == io.EOF { + break + } + if err != nil { + log.Println(err) + return + } + pName := p.FormName() + content, err := ioutil.ReadAll(p) + if err == io.ErrUnexpectedEOF { + if strings.Contains( + string(content), + "The job exceeded the maximum time limit for jobs, and has been terminated.", + ) { + timeOut = true + } + break + } + if err != nil { + log.Println(err) + continue + } + if pName == "keep-alive" { + continue + } + if err != nil { + log.Println(err) + continue + } + if strings.HasPrefix(pName, "port-lint-output-") && len(content) > 0 && !lintDone { + comment += "
Lint results\n\n```\n" + string(content) + "```\n
\n\n
\n\n" + lintDone = true + } + if strings.HasSuffix(pName, "-pastebin") { + pastebinRegex := regexp.MustCompile(`^port-(.*)(-dep)?-install-output-(success|fail)-pastebin$`) + pbInfo := pastebinRegex.FindStringSubmatch(pName) + if pbInfo == nil { + continue + } + comment += "Port " + pbInfo[1] + if pbInfo[2] == "-dep" { + comment += "'s dependencies" + } + comment += " **" + pbInfo[3] + "** on " + job.Config.OsxImage + ". [Log](" + string(content) + ")\n" + } + } + } + + if timeOut { + comment += "The build timed out." + } + + receiver.githubClient.CreateComment( + payload.Repository.OwnerName, + payload.Repository.Name, + payload.PullRequestNumber, + &comment, + ) +}