A small and light tool to help with FreeBSD Ports CI (Continuous Integration)
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

main.go 14KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626
  1. package main
  2. import (
  3. "bufio"
  4. "bytes"
  5. "crypto/hmac"
  6. "crypto/sha1"
  7. "crypto/tls"
  8. "encoding/json"
  9. "flag"
  10. "fmt"
  11. "html/template"
  12. "io/ioutil"
  13. "log"
  14. "net"
  15. "net/http"
  16. "net/smtp"
  17. "os"
  18. "os/exec"
  19. "path"
  20. "path/filepath"
  21. "regexp"
  22. "strings"
  23. "sync"
  24. texttemplate "text/template"
  25. "time"
  26. "github.com/NYTimes/gziphandler"
  27. "gopkg.in/yaml.v2"
  28. )
  29. type controller struct {
  30. wg *sync.WaitGroup
  31. cfg *config
  32. queues map[string]*queue
  33. }
  34. type config struct {
  35. Workdir string
  36. Logdir string
  37. Staticdir string
  38. Tmpldir string
  39. Server struct {
  40. Host string
  41. BaseURL string
  42. TLScert string
  43. TLSkey string
  44. }
  45. Webhook struct {
  46. Secret string
  47. }
  48. Notification struct {
  49. StatusAPI struct {
  50. URL string
  51. Token string
  52. }
  53. Email struct {
  54. SmtpHost string
  55. SmtpUser string
  56. SmtpPass string
  57. From string
  58. }
  59. }
  60. Queues []queue
  61. DefaultQueues []string `yaml:"default_queues"`
  62. }
  63. type queue struct {
  64. Name string
  65. Recipe string
  66. Environment map[string]string
  67. Workdir string
  68. PathMatch string
  69. queue chan *job
  70. }
  71. type job struct {
  72. ID string
  73. Port string
  74. Startdate time.Time
  75. Enddate time.Time
  76. Build map[string]*build
  77. PushEvent gitPushEventData
  78. BaseURL string
  79. }
  80. type build struct {
  81. ID string
  82. Queue string
  83. Status string
  84. Logfile string
  85. Startdate time.Time
  86. Enddate time.Time
  87. }
  88. type gitPushEventData struct {
  89. Secret string `json:"secret"`
  90. CommitID string `json:"after"`
  91. Repository struct {
  92. Name string `json:"name"`
  93. FullName string `json:"full_name"`
  94. HTMLURL string `json:"html_url"`
  95. CloneURL string `json:"clone_url"`
  96. } `json:"repository"`
  97. Commits []struct {
  98. Message string `json:"message"`
  99. URL string `json:"url"`
  100. Author struct {
  101. Name string `json:"name"`
  102. EMail string `json:"email"`
  103. Username string `json:"username"`
  104. } `json:"author"`
  105. Added []string `json:"added"`
  106. Removed []string `json:"removed"`
  107. Modified []string `json:"modified"`
  108. } `json:"commits"`
  109. }
  110. func calcSignature(payload *[]byte, secret string) string {
  111. mac := hmac.New(sha1.New, []byte(secret))
  112. mac.Write(*payload)
  113. return fmt.Sprintf("sha1=%x", mac.Sum(nil))
  114. }
  115. func getAffectedPort(data gitPushEventData) string {
  116. lines := strings.Split(data.Commits[0].Message, "\n")
  117. if len(lines) < 1 || strings.IndexByte(lines[0], ':') < 1 {
  118. return ""
  119. }
  120. re := regexp.MustCompile(`^([a-z0-9-]+)/([a-zA-Z0-9-_.]+)$`)
  121. port := strings.TrimSpace(lines[0][:strings.IndexByte(lines[0], ':')])
  122. if re.MatchString(port) {
  123. return port
  124. }
  125. return ""
  126. }
  127. func (c *controller) matchQueues(data gitPushEventData) []queue {
  128. queues := make([]queue, 0)
  129. NEXTQUEUE:
  130. for i := range c.cfg.Queues {
  131. re := regexp.MustCompile(c.cfg.Queues[i].PathMatch)
  132. for commit := range data.Commits {
  133. // Queue name match against PathMatch config
  134. for _, file := range data.Commits[commit].Added {
  135. if re.MatchString(file) {
  136. queues = append(queues, c.cfg.Queues[i])
  137. continue NEXTQUEUE
  138. }
  139. }
  140. for _, file := range data.Commits[commit].Modified {
  141. if re.MatchString(file) {
  142. queues = append(queues, c.cfg.Queues[i])
  143. continue NEXTQUEUE
  144. }
  145. }
  146. // Queue name from commit message tags (CI: yes/no/true/false)
  147. lines := strings.Split(data.Commits[commit].Message, "\n")
  148. for _, line := range lines {
  149. line = strings.ToLower(line)
  150. if strings.HasPrefix(line, "ci:") {
  151. if strings.Contains(line, "no") || strings.Contains(line, "false") {
  152. continue NEXTQUEUE
  153. }
  154. if strings.Contains(line, "yes") || strings.Contains(line, "true") {
  155. queues = append(queues, c.cfg.Queues[i])
  156. continue NEXTQUEUE
  157. }
  158. }
  159. }
  160. }
  161. // Queue name from DefaultQueues config
  162. for _, q := range c.cfg.DefaultQueues {
  163. if q == c.cfg.Queues[i].Name {
  164. queues = append(queues, c.cfg.Queues[i])
  165. }
  166. }
  167. }
  168. return queues
  169. }
  170. func (j *job) StatusOverall() string {
  171. // status: pending | failure | success
  172. for _, b := range j.Build {
  173. if b.Status == "pending" {
  174. return b.Status
  175. }
  176. }
  177. for _, b := range j.Build {
  178. if b.Status == "failure" {
  179. return b.Status
  180. }
  181. }
  182. return "success"
  183. }
  184. func (j *job) StartDate() string {
  185. return j.Startdate.Format(time.RFC850)
  186. }
  187. func (j *job) EndDate() string {
  188. return j.Enddate.Format(time.RFC850)
  189. }
  190. func (j *job) TimeNow() string {
  191. return time.Now().Format(time.RFC850)
  192. }
  193. func (b *build) Runtime() string {
  194. diff := b.Enddate.Sub(b.Startdate).Round(time.Second)
  195. return fmt.Sprintf("%s", diff.String())
  196. }
  197. func (b *build) LogfileContent() string {
  198. raw, err := ioutil.ReadFile(b.Logfile)
  199. if err != nil {
  200. return ""
  201. }
  202. return string(raw)
  203. }
  204. func (c *controller) renderBuildTemplate(j *job) {
  205. tmpl, err := template.ParseFiles(path.Join(c.cfg.Tmpldir, "index.html"))
  206. if err != nil {
  207. log.Printf("Failed parsing template: %v", err)
  208. return
  209. }
  210. outfile, _ := os.Create(path.Join(c.cfg.Logdir, j.ID, "index.html"))
  211. defer outfile.Close()
  212. writer := bufio.NewWriter(outfile)
  213. err = tmpl.Execute(writer, &j)
  214. if err != nil {
  215. log.Printf("Failed executing template: %v", err)
  216. return
  217. }
  218. writer.Flush()
  219. outfile.Sync()
  220. }
  221. func (c *controller) renderEmailTemplate(j *job) string {
  222. tmpl, err := texttemplate.ParseFiles(path.Join(c.cfg.Tmpldir, "email.txt"))
  223. if err != nil {
  224. log.Printf("Failed parsing template: %v", err)
  225. return ""
  226. }
  227. var out bytes.Buffer
  228. err = tmpl.Execute(&out, &j)
  229. if err != nil {
  230. log.Printf("Failed executing template: %v", err)
  231. return ""
  232. }
  233. return out.String()
  234. }
  235. func (c *controller) evalEnvVariable(j *job, key string, val string) (string, string) {
  236. tmpl, err := texttemplate.New(key).Parse(val)
  237. if err != nil {
  238. log.Printf("Failed parsing env var %s=%s: %v", key, val, err)
  239. return key, ""
  240. }
  241. var out bytes.Buffer
  242. err = tmpl.Execute(&out, &j)
  243. if err != nil {
  244. log.Printf("Failed executing env var %s: %v", key, err)
  245. return key, ""
  246. }
  247. return key, out.String()
  248. }
  249. func (c *controller) sendStatusUpdate(j *job, b *build) error {
  250. target := ""
  251. if b.Status != "pending" {
  252. target = j.BaseURL
  253. }
  254. if c.cfg.Notification.StatusAPI.URL != "" {
  255. url := fmt.Sprintf("%s/repos/%s/statuses/%s?access_token=%s",
  256. c.cfg.Notification.StatusAPI.URL, j.PushEvent.Repository.FullName,
  257. j.PushEvent.CommitID, c.cfg.Notification.StatusAPI.Token)
  258. jsonValue, _ := json.Marshal(map[string]string{
  259. "state": b.Status,
  260. "target_url": target,
  261. "context": b.Queue,
  262. })
  263. _, err := http.Post(url, "application/json", bytes.NewBuffer(jsonValue))
  264. if err != nil {
  265. log.Printf("StatusAPI request to %s failed: %s\n", url, err)
  266. }
  267. }
  268. if c.cfg.Notification.Email.SmtpHost != "" && j.StatusOverall() != "pending" {
  269. data := c.renderEmailTemplate(j)
  270. if data != "" {
  271. var auth smtp.Auth
  272. host, _, _ := net.SplitHostPort(c.cfg.Notification.Email.SmtpHost)
  273. if c.cfg.Notification.Email.SmtpUser != "" && c.cfg.Notification.Email.SmtpPass != "" {
  274. auth = smtp.PlainAuth("", c.cfg.Notification.Email.SmtpUser, c.cfg.Notification.Email.SmtpPass, host)
  275. }
  276. err := smtp.SendMail(
  277. c.cfg.Notification.Email.SmtpHost,
  278. auth,
  279. c.cfg.Notification.Email.From,
  280. []string{j.PushEvent.Commits[0].Author.EMail},
  281. []byte(data),
  282. )
  283. if err != nil {
  284. log.Printf("EMail delivery failed: %v\n", err)
  285. }
  286. }
  287. }
  288. return nil
  289. }
  290. func (c *controller) startWorker(q *queue) {
  291. defer c.wg.Done()
  292. for {
  293. var j *job
  294. select {
  295. case j = <-q.queue:
  296. b := j.Build[q.Name]
  297. b.Startdate = time.Now()
  298. log.Printf("ID %s started on %s\n", j.ID, q.Name)
  299. os.MkdirAll(path.Join(c.cfg.Logdir, j.ID), os.ModePerm)
  300. c.sendStatusUpdate(j, b)
  301. c.renderBuildTemplate(j)
  302. env := os.Environ()
  303. for k, v := range q.Environment {
  304. key, val := c.evalEnvVariable(j, k, v)
  305. env = append(env, fmt.Sprintf("%s=%s", key, val))
  306. }
  307. os.MkdirAll(q.Workdir, os.ModePerm)
  308. cmd := exec.Cmd{
  309. Dir: q.Workdir,
  310. Env: env,
  311. Path: "/usr/bin/make",
  312. Args: []string{
  313. "make",
  314. "-C", q.Workdir,
  315. "-f", fmt.Sprintf("%s/%s.mk", c.cfg.Workdir, q.Recipe),
  316. "all",
  317. },
  318. }
  319. output, err := cmd.CombinedOutput()
  320. if err != nil {
  321. b.Status = "failure"
  322. } else {
  323. b.Status = "success"
  324. }
  325. b.Enddate = time.Now()
  326. j.Enddate = time.Now()
  327. b.Logfile = path.Join(c.cfg.Logdir, j.ID, b.ID+".log")
  328. ioutil.WriteFile(b.Logfile, output, 0644)
  329. log.Printf("ID %s on %s finished %s\n", j.ID, q.Name, b.Status)
  330. c.sendStatusUpdate(j, b)
  331. c.renderBuildTemplate(j)
  332. case <-time.After(time.Second * 1):
  333. }
  334. }
  335. }
  336. func (c *controller) handleWebhook(w http.ResponseWriter, r *http.Request) {
  337. payload, err := ioutil.ReadAll(r.Body)
  338. if err != nil {
  339. http.Error(w, "Internal Error", http.StatusInternalServerError)
  340. return
  341. }
  342. if r.Header.Get("X-GitHub-Event") != "" {
  343. if r.Header.Get("X-GitHub-Event") != "push" {
  344. http.Error(w, "Invalid webhook", http.StatusBadRequest)
  345. return
  346. }
  347. }
  348. data := gitPushEventData{}
  349. if err = json.Unmarshal(payload, &data); err != nil {
  350. http.Error(w, "Failed to parse webhook data", http.StatusBadRequest)
  351. return
  352. }
  353. if c.cfg.Webhook.Secret != "" {
  354. if r.Header.Get("X-Hub-Signature") != "" {
  355. if calcSignature(&payload, c.cfg.Webhook.Secret) != r.Header.Get("X-Hub-Signature") {
  356. http.Error(w, "Invalid secret", http.StatusBadRequest)
  357. return
  358. }
  359. } else {
  360. if data.Secret != c.cfg.Webhook.Secret {
  361. http.Error(w, "Invalid secret", http.StatusBadRequest)
  362. return
  363. }
  364. }
  365. }
  366. port := getAffectedPort(data)
  367. if port == "" {
  368. fmt.Fprint(w, "No category/port detected")
  369. return
  370. }
  371. job := job{
  372. ID: time.Now().Format("20060102-15:04:05.000"),
  373. Port: port,
  374. Startdate: time.Now(),
  375. Build: make(map[string]*build),
  376. PushEvent: data,
  377. BaseURL: "",
  378. }
  379. job.BaseURL = fmt.Sprintf("%s/%s/%s/", c.cfg.Server.BaseURL, "builds", job.ID)
  380. cnt := 0
  381. for _, q := range c.matchQueues(data) {
  382. b := build{
  383. ID: fmt.Sprintf("%03d", cnt+1),
  384. Queue: q.Name,
  385. Status: "pending",
  386. }
  387. job.Build[q.Name] = &b
  388. select {
  389. case q.queue <- &job:
  390. cnt++
  391. log.Printf("ID %s queued on %s (pos %d)\n", job.ID, q.Name, len(q.queue))
  392. default:
  393. log.Printf("ID %s Queue limit reached on queue %s\n", job.ID, q.Name)
  394. }
  395. }
  396. fmt.Fprintf(w, "ID %s has %d Jobs queued", job.ID, cnt)
  397. }
  398. func (c *controller) startHTTPD() {
  399. defer c.wg.Done()
  400. mux := http.NewServeMux()
  401. staticHandlerGz := gziphandler.GzipHandler(http.StripPrefix("/static/", http.FileServer(http.Dir(c.cfg.Staticdir))))
  402. mux.Handle("/static/", staticHandlerGz)
  403. buildHandlerGz := gziphandler.GzipHandler(http.StripPrefix("/builds/", http.FileServer(http.Dir(c.cfg.Logdir))))
  404. mux.Handle("/builds/", buildHandlerGz)
  405. mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
  406. if r.Method == "GET" {
  407. fmt.Fprint(w, "nothing to see here")
  408. } else {
  409. c.handleWebhook(w, r)
  410. }
  411. })
  412. var err error
  413. if c.cfg.Server.TLScert != "" && c.cfg.Server.TLSkey != "" {
  414. cfg := &tls.Config{
  415. MinVersion: tls.VersionTLS12,
  416. CurvePreferences: []tls.CurveID{tls.CurveP521, tls.CurveP384, tls.CurveP256},
  417. PreferServerCipherSuites: true,
  418. CipherSuites: []uint16{
  419. tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
  420. tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
  421. tls.TLS_RSA_WITH_AES_256_GCM_SHA384,
  422. tls.TLS_RSA_WITH_AES_256_CBC_SHA,
  423. },
  424. }
  425. srv := &http.Server{
  426. Addr: c.cfg.Server.Host,
  427. Handler: mux,
  428. TLSConfig: cfg,
  429. TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler), 0),
  430. }
  431. log.Printf("Listening on %s (https)\n", c.cfg.Server.Host)
  432. err = srv.ListenAndServeTLS(c.cfg.Server.TLScert, c.cfg.Server.TLSkey)
  433. } else {
  434. srv := &http.Server{
  435. Addr: c.cfg.Server.Host,
  436. Handler: mux,
  437. }
  438. log.Printf("Listening on %s (http)\n", c.cfg.Server.Host)
  439. err = srv.ListenAndServe()
  440. }
  441. if err != nil {
  442. log.Printf("Listen failed: %s\n", err)
  443. }
  444. }
  445. func parseConfig(file string) config {
  446. f, err := os.Open(file)
  447. if err != nil {
  448. log.Fatalf("Error: %v", err)
  449. }
  450. defer f.Close()
  451. dec := yaml.NewDecoder(f)
  452. cfg := config{}
  453. err = dec.Decode(&cfg)
  454. if err != nil {
  455. log.Fatalf("Error: %v", err)
  456. }
  457. cfg.Workdir, _ = filepath.Abs(cfg.Workdir)
  458. cfg.Logdir, _ = filepath.Abs(cfg.Logdir)
  459. cfg.Server.BaseURL = strings.TrimSuffix(cfg.Server.BaseURL, "/")
  460. cfg.Notification.StatusAPI.URL = strings.TrimSuffix(cfg.Notification.StatusAPI.URL, "/")
  461. for i := range cfg.Queues {
  462. if cfg.Queues[i].PathMatch == "" {
  463. cfg.Queues[i].PathMatch = "^$"
  464. }
  465. _, err := regexp.Compile(cfg.Queues[i].PathMatch)
  466. if err != nil {
  467. log.Fatalf("Error: %v", err)
  468. }
  469. if cfg.Queues[i].Environment == nil {
  470. cfg.Queues[i].Environment = map[string]string{}
  471. }
  472. _, ok := cfg.Queues[i].Environment["JOB_ID"]
  473. if !ok {
  474. cfg.Queues[i].Environment["JOB_ID"] = "{{.ID}}"
  475. }
  476. _, ok = cfg.Queues[i].Environment["JOB_PORT"]
  477. if !ok {
  478. cfg.Queues[i].Environment["JOB_PORT"] = "{{.Port}}"
  479. }
  480. _, ok = cfg.Queues[i].Environment["COMMIT_ID"]
  481. if !ok {
  482. cfg.Queues[i].Environment["COMMIT_ID"] = "{{.PushEvent.CommitID}}"
  483. }
  484. _, ok = cfg.Queues[i].Environment["REPO_URL"]
  485. if !ok {
  486. cfg.Queues[i].Environment["REPO_URL"] = "{{.PushEvent.Repository.CloneURL}}"
  487. }
  488. _, ok = cfg.Queues[i].Environment["AUTHOR"]
  489. if !ok {
  490. cfg.Queues[i].Environment["AUTHOR"] = "{{(index .PushEvent.Commits 0).Author.Username}}"
  491. }
  492. _, ok = cfg.Queues[i].Environment["AUTHOR_EMAIL"]
  493. if !ok {
  494. cfg.Queues[i].Environment["AUTHOR_EMAIL"] = "{{(index .PushEvent.Commits 0).Author.EMail}}"
  495. }
  496. }
  497. return cfg
  498. }
  499. func main() {
  500. var cfgfile string
  501. flag.StringVar(&cfgfile, "config", "caronade.yaml", "Path to config file")
  502. flag.Parse()
  503. cfg := parseConfig(cfgfile)
  504. wg := sync.WaitGroup{}
  505. ctrl := controller{
  506. wg: &wg,
  507. cfg: &cfg,
  508. queues: make(map[string]*queue),
  509. }
  510. reg := regexp.MustCompile("[^a-zA-Z0-9]+")
  511. for i := range cfg.Queues {
  512. log.Printf("Adding queue %s\n", cfg.Queues[i].Name)
  513. cfg.Queues[i].Workdir = path.Join(cfg.Workdir, reg.ReplaceAllString(cfg.Queues[i].Name, ""))
  514. cfg.Queues[i].queue = make(chan *job, 10)
  515. ctrl.queues[cfg.Queues[i].Name] = &cfg.Queues[i]
  516. wg.Add(1)
  517. go ctrl.startWorker(&cfg.Queues[i])
  518. }
  519. wg.Add(1)
  520. go ctrl.startHTTPD()
  521. wg.Wait()
  522. }