A small and light tool to help with FreeBSD Ports CI (Continuous Integration)
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

main.go 10KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456
  1. package main
  2. import (
  3. "bufio"
  4. "bytes"
  5. "crypto/hmac"
  6. "crypto/sha1"
  7. "crypto/tls"
  8. "encoding/json"
  9. "flag"
  10. "fmt"
  11. "html/template"
  12. "io/ioutil"
  13. "log"
  14. "net/http"
  15. "os"
  16. "os/exec"
  17. "path"
  18. "path/filepath"
  19. "regexp"
  20. "strings"
  21. "sync"
  22. "time"
  23. "github.com/NYTimes/gziphandler"
  24. "gopkg.in/yaml.v2"
  25. )
  26. type controller struct {
  27. wg *sync.WaitGroup
  28. cfg *config
  29. queues map[string]*queue
  30. }
  31. type config struct {
  32. Workdir string
  33. Logdir string
  34. Staticdir string
  35. Tmpldir string
  36. Server struct {
  37. Host string
  38. BaseURL string
  39. TLScert string
  40. TLSkey string
  41. }
  42. Webhook struct {
  43. Secret string
  44. }
  45. Repository struct {
  46. APIURL string
  47. APIToken string
  48. }
  49. Queues []queue
  50. DefaultQueues []string `yaml:"default_queues"`
  51. }
  52. type queue struct {
  53. Name string
  54. Recipe string
  55. Environment map[string]string
  56. Workdir string
  57. queue chan *job
  58. }
  59. type job struct {
  60. ID string
  61. Port string
  62. CommitID string
  63. CommitMsg string
  64. CommitURL string
  65. RepoName string
  66. RepoFullName string
  67. RepoHTMLURL string
  68. RepoCloneURL string
  69. Startdate time.Time
  70. Enddate time.Time
  71. Build map[string]*build
  72. }
  73. type build struct {
  74. ID string
  75. Queue string
  76. Status string
  77. Logfile string
  78. Startdate time.Time
  79. Enddate time.Time
  80. }
  81. type gitPushEventData struct {
  82. Secret string `json:"secret"`
  83. CommitID string `json:"after"`
  84. Repository struct {
  85. Name string `json:"name"`
  86. FullName string `json:"full_name"`
  87. HTMLURL string `json:"html_url"`
  88. CloneURL string `json:"clone_url"`
  89. } `json:"repository"`
  90. Commits []struct {
  91. Message string `json:"message"`
  92. URL string `json:"url"`
  93. } `json:"commits"`
  94. }
  95. func calcSignature(payload *[]byte, secret string) string {
  96. mac := hmac.New(sha1.New, []byte(secret))
  97. mac.Write(*payload)
  98. return fmt.Sprintf("sha1=%x", mac.Sum(nil))
  99. }
  100. func getPortFromMessage(msg string) string {
  101. lines := strings.Split(msg, "\n")
  102. if len(lines) < 1 || strings.IndexByte(lines[0], ':') < 1 {
  103. return ""
  104. }
  105. re := regexp.MustCompile(`^([a-z0-9-]+)/([a-zA-Z0-9-_.]+)$`)
  106. port := strings.TrimSpace(lines[0][:strings.IndexByte(lines[0], ':')])
  107. if re.MatchString(port) {
  108. return port
  109. }
  110. return ""
  111. }
  112. func (c *controller) getQueueInfoFromMessage(msg string) []queue {
  113. queues := make([]queue, 0)
  114. lines := strings.Split(msg, "\n")
  115. for _, line := range lines {
  116. line = strings.ToLower(line)
  117. if strings.HasPrefix(line, "ci:") {
  118. if strings.Contains(line, "no") || strings.Contains(line, "false") {
  119. return queues
  120. }
  121. if strings.Contains(line, "yes") || strings.Contains(line, "true") {
  122. for i := range c.cfg.Queues {
  123. queues = append(queues, c.cfg.Queues[i])
  124. }
  125. return queues
  126. }
  127. }
  128. }
  129. for _, name := range c.cfg.DefaultQueues {
  130. q, ok := c.queues[name]
  131. if !ok {
  132. continue
  133. }
  134. queues = append(queues, *q)
  135. }
  136. return queues
  137. }
  138. func (j *job) StartDate() string {
  139. return j.Startdate.Format(time.RFC850)
  140. }
  141. func (b *build) LogfileContent() string {
  142. raw, err := ioutil.ReadFile(b.Logfile)
  143. if err != nil {
  144. return ""
  145. }
  146. return string(raw)
  147. }
  148. func (c *controller) renderBuildTemplate(j *job) {
  149. tmpl, err := template.ParseFiles(path.Join(c.cfg.Tmpldir, "index.html"))
  150. if err != nil {
  151. log.Printf("Failed parsing template: %v", err)
  152. return
  153. }
  154. outfile, _ := os.Create(path.Join(c.cfg.Logdir, j.ID, "index.html"))
  155. defer outfile.Close()
  156. writer := bufio.NewWriter(outfile)
  157. err = tmpl.Execute(writer, &j)
  158. if err != nil {
  159. log.Printf("Failed executing template: %v", err)
  160. return
  161. }
  162. writer.Flush()
  163. outfile.Sync()
  164. }
  165. func (c *controller) sendStatusUpdate(j *job, b *build) error {
  166. target := ""
  167. if b.Status != "pending" {
  168. target = fmt.Sprintf("%s/builds/%s/", c.cfg.Server.BaseURL, j.ID)
  169. }
  170. url := fmt.Sprintf("%s/repos/%s/statuses/%s?access_token=%s",
  171. c.cfg.Repository.APIURL, j.RepoFullName, j.CommitID, c.cfg.Repository.APIToken)
  172. jsonValue, _ := json.Marshal(map[string]string{
  173. "state": b.Status,
  174. "target_url": target,
  175. "context": b.Queue,
  176. })
  177. _, err := http.Post(url, "application/json", bytes.NewBuffer(jsonValue))
  178. return err
  179. }
  180. func (c *controller) startWorker(q *queue) {
  181. defer c.wg.Done()
  182. for {
  183. var j *job
  184. select {
  185. case j = <-q.queue:
  186. b := j.Build[q.Name]
  187. b.Startdate = time.Now()
  188. log.Printf("ID %s started on %s\n", j.ID, q.Name)
  189. c.sendStatusUpdate(j, b)
  190. env := append(os.Environ(),
  191. fmt.Sprintf("JOB_ID=%s", j.ID),
  192. fmt.Sprintf("COMMIT_ID=%s", j.CommitID),
  193. fmt.Sprintf("REPO_URL=%s", j.RepoCloneURL),
  194. fmt.Sprintf("JOB_PORT=%s", j.Port),
  195. )
  196. for k, v := range q.Environment {
  197. env = append(env, fmt.Sprintf("%s=%s", k, v))
  198. }
  199. os.MkdirAll(q.Workdir, os.ModePerm)
  200. cmd := exec.Cmd{
  201. Dir: q.Workdir,
  202. Env: env,
  203. Path: "/usr/bin/make",
  204. Args: []string{
  205. "make",
  206. "-C", q.Workdir,
  207. "-f", fmt.Sprintf("%s.mk", q.Recipe),
  208. "-I", c.cfg.Workdir,
  209. "all",
  210. },
  211. }
  212. output, err := cmd.CombinedOutput()
  213. if err != nil {
  214. b.Status = "failure"
  215. } else {
  216. b.Status = "success"
  217. }
  218. b.Enddate = time.Now()
  219. j.Enddate = time.Now()
  220. b.Logfile = path.Join(c.cfg.Logdir, j.ID, b.ID+".log")
  221. os.MkdirAll(filepath.Dir(b.Logfile), os.ModePerm)
  222. ioutil.WriteFile(b.Logfile, output, 0600)
  223. log.Printf("ID %s on %s finished %s\n", j.ID, q.Name, b.Status)
  224. c.sendStatusUpdate(j, b)
  225. c.renderBuildTemplate(j)
  226. case <-time.After(time.Second * 1):
  227. }
  228. }
  229. }
  230. func (c *controller) handleWebhook(w http.ResponseWriter, r *http.Request) {
  231. payload, err := ioutil.ReadAll(r.Body)
  232. if err != nil {
  233. http.Error(w, "Internal Error", http.StatusInternalServerError)
  234. return
  235. }
  236. if r.Header.Get("X-GitHub-Event") != "" {
  237. if r.Header.Get("X-GitHub-Event") != "push" {
  238. http.Error(w, "Invalid webhook", http.StatusBadRequest)
  239. return
  240. }
  241. }
  242. data := gitPushEventData{}
  243. if err = json.Unmarshal(payload, &data); err != nil {
  244. http.Error(w, "Failed to parse webhook data", http.StatusBadRequest)
  245. return
  246. }
  247. if c.cfg.Webhook.Secret != "" {
  248. if r.Header.Get("X-Hub-Signature") != "" {
  249. if calcSignature(&payload, c.cfg.Webhook.Secret) != r.Header.Get("X-Hub-Signature") {
  250. http.Error(w, "Invalid secret", http.StatusBadRequest)
  251. return
  252. }
  253. } else {
  254. if data.Secret != c.cfg.Webhook.Secret {
  255. http.Error(w, "Invalid secret", http.StatusBadRequest)
  256. return
  257. }
  258. }
  259. }
  260. port := getPortFromMessage(data.Commits[0].Message)
  261. if port == "" {
  262. fmt.Fprint(w, "No category/port detected in commit message")
  263. return
  264. }
  265. job := job{
  266. ID: time.Now().Format("20060102150405.000"),
  267. Port: port,
  268. CommitID: data.CommitID,
  269. CommitMsg: data.Commits[0].Message,
  270. CommitURL: data.Commits[0].URL,
  271. RepoName: data.Repository.Name,
  272. RepoFullName: data.Repository.FullName,
  273. RepoHTMLURL: data.Repository.HTMLURL,
  274. RepoCloneURL: data.Repository.CloneURL,
  275. Startdate: time.Now(),
  276. Build: make(map[string]*build),
  277. }
  278. cnt := 0
  279. for _, q := range c.getQueueInfoFromMessage(data.Commits[0].Message) {
  280. b := build{
  281. ID: fmt.Sprintf("%03d", cnt+1),
  282. Queue: q.Name,
  283. Status: "pending",
  284. }
  285. job.Build[q.Name] = &b
  286. select {
  287. case q.queue <- &job:
  288. cnt++
  289. log.Printf("ID %s Port %s queued on %s (pos %d)\n", job.ID, job.Port, q.Name, len(q.queue))
  290. default:
  291. log.Printf("ID %s Queue limit reached on queue %s\n", job.ID, q.Name)
  292. }
  293. }
  294. fmt.Fprintf(w, "ID %s has %d Jobs queued", job.ID, cnt)
  295. }
  296. func (c *controller) startHTTPD() {
  297. defer c.wg.Done()
  298. mux := http.NewServeMux()
  299. staticHandlerGz := gziphandler.GzipHandler(http.StripPrefix("/static/", http.FileServer(http.Dir(c.cfg.Staticdir))))
  300. mux.Handle("/static/", staticHandlerGz)
  301. buildHandlerGz := gziphandler.GzipHandler(http.StripPrefix("/builds/", http.FileServer(http.Dir(c.cfg.Logdir))))
  302. mux.Handle("/builds/", buildHandlerGz)
  303. mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
  304. if r.Method == "GET" {
  305. fmt.Fprint(w, "nothing to see here")
  306. } else {
  307. c.handleWebhook(w, r)
  308. }
  309. })
  310. var err error
  311. if c.cfg.Server.TLScert != "" && c.cfg.Server.TLSkey != "" {
  312. cfg := &tls.Config{
  313. MinVersion: tls.VersionTLS12,
  314. CurvePreferences: []tls.CurveID{tls.CurveP521, tls.CurveP384, tls.CurveP256},
  315. PreferServerCipherSuites: true,
  316. CipherSuites: []uint16{
  317. tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
  318. tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
  319. tls.TLS_RSA_WITH_AES_256_GCM_SHA384,
  320. tls.TLS_RSA_WITH_AES_256_CBC_SHA,
  321. },
  322. }
  323. srv := &http.Server{
  324. Addr: c.cfg.Server.Host,
  325. Handler: mux,
  326. TLSConfig: cfg,
  327. TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler), 0),
  328. }
  329. log.Printf("Listening on %s (https)\n", c.cfg.Server.Host)
  330. err = srv.ListenAndServeTLS(c.cfg.Server.TLScert, c.cfg.Server.TLSkey)
  331. } else {
  332. srv := &http.Server{
  333. Addr: c.cfg.Server.Host,
  334. Handler: mux,
  335. }
  336. log.Printf("Listening on %s (http)\n", c.cfg.Server.Host)
  337. err = srv.ListenAndServe()
  338. }
  339. if err != nil {
  340. log.Printf("Listen failed: %s\n", err)
  341. }
  342. }
  343. func parseConfig(file string) config {
  344. f, err := os.Open(file)
  345. if err != nil {
  346. log.Fatalf("Error: %v", err)
  347. }
  348. defer f.Close()
  349. dec := yaml.NewDecoder(f)
  350. cfg := config{}
  351. err = dec.Decode(&cfg)
  352. if err != nil {
  353. log.Fatalf("Error: %v", err)
  354. }
  355. cfg.Workdir, _ = filepath.Abs(cfg.Workdir)
  356. cfg.Logdir, _ = filepath.Abs(cfg.Logdir)
  357. cfg.Server.BaseURL = strings.TrimSuffix(cfg.Server.BaseURL, "/")
  358. cfg.Repository.APIURL = strings.TrimSuffix(cfg.Repository.APIURL, "/")
  359. return cfg
  360. }
  361. func main() {
  362. var cfgfile string
  363. flag.StringVar(&cfgfile, "config", "caronade.yaml", "Path to config file")
  364. flag.Parse()
  365. cfg := parseConfig(cfgfile)
  366. wg := sync.WaitGroup{}
  367. ctrl := controller{
  368. wg: &wg,
  369. cfg: &cfg,
  370. queues: make(map[string]*queue),
  371. }
  372. reg := regexp.MustCompile("[^a-zA-Z0-9]+")
  373. for i := range cfg.Queues {
  374. log.Printf("Adding queue %s\n", cfg.Queues[i].Name)
  375. cfg.Queues[i].Workdir = path.Join(cfg.Workdir, reg.ReplaceAllString(cfg.Queues[i].Name, ""))
  376. cfg.Queues[i].queue = make(chan *job, 10)
  377. ctrl.queues[cfg.Queues[i].Name] = &cfg.Queues[i]
  378. wg.Add(1)
  379. go ctrl.startWorker(&cfg.Queues[i])
  380. }
  381. wg.Add(1)
  382. go ctrl.startHTTPD()
  383. wg.Wait()
  384. }