WIP using fixed test data

see model.go:36 onwards for fixed test data

core/section/github/auth.go

@@ -46,7 +46,7 @@ func validateToken(ptoken string) error {
 	return err
 }
 
-func (*Provider) githubClient(config githubConfig) *gogithub.Client {
+func (*Provider) githubClient(config *githubConfig) *gogithub.Client {
 	ts := oauth2.StaticTokenSource(
 		&oauth2.Token{AccessToken: config.Token},
 	)

core/section/github/commits.go

@@ -12,21 +12,25 @@
 package github
 
 import (
-	"encoding/json"
 	"fmt"
 	"html/template"
-	"net/http"
+	"sort"
 
 	"github.com/documize/community/core/log"
-	"github.com/documize/community/core/section/provider"
 
 	gogithub "github.com/google/go-github/github"
 )
 
 type githubBranchCommits struct {
-	Name    string `json:"name"`
+	Name        string             `json:"name"`
-	Day     string `json:"day"`
+	URL         string             `json:"url"`
-	Commits []githubCommit
+	CommitCount int                `json:"commitCount"`
+	Days        []githubDayCommits `json:"days"`
+}
+
+type githubDayCommits struct {
+	Day     string         `json:"day"`
+	Commits []githubCommit `json:"commits"`
 }
 
 type githubCommit struct {
@@ -37,159 +41,223 @@ type githubCommit struct {
 	Avatar  string       `json:"avatar"`
 }
 
+type githubAuthorStats struct {
+	Author      string `json:"author"`
+	Avatar      string `json:"avatar"`
+	CommitCount int    `json:"commitCount"`
+	//TotalChanges int    `json:"totalChanges"`
+}
+
+// sort stats in order that that should be presented.
+type asToSort []githubAuthorStats
+
+func (s asToSort) Len() int      { return len(s) }
+func (s asToSort) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+func (s asToSort) Less(i, j int) bool {
+	return s[i].CommitCount > s[j].CommitCount
+}
+
 const tagCommitsData = "commitsData"
 
 func init() {
-	reports[tagCommitsData] = report{commandCommitsData, refreshCommits, renderCommits, `
+	reports[tagCommitsData] = report{refreshCommits, renderCommits, `
+<h3>Commits</h3>
 <div class="section-github-render">
-	<p>
+	<table style="width:80%">
-		There are {{ .CommitCount }} commits for branch <a href="{{.Config.BranchURL}}">{{.Config.Branch}}</a> of repository <a href="{{ .Repo.URL }}">{{.Repo.Name}}.</a>
+		<tr>
-		Showing {{ .Limit }} items {{ .DateMessage }}.
+			<th>{{.CommitCount}} commits since {{.Config.Since}}{{.Config.DateMessage}}</th>
-	</p>
+    		<th>Author</th>
-	<div class="github-board">
+    		<th>#commits</th>
-		{{range $data := .BranchCommits}}
+  		</tr>
-			<div class="github-group-title">
+		{{range $stats := .AuthorStats}}
-				Commits on {{ $data.Day }}
+			<tr>
-			</div>
+				<td>
-			<ul class="github-list">
+					<div class="github-avatar">
-				{{range $commit := $data.Commits}}
+						<img alt="@{{$stats.Author}}" src="{{$stats.Avatar}}" height="36" width="36">
-					<li class="github-commit-item">
+					</div>
-						<a class="link" href="{{$commit.URL}}">
+				</td>
-							<div class="github-avatar">
+				<td>{{$stats.Author}}</td>
-								<img alt="@{{$commit.Name}}" src="{{$commit.Avatar}}" height="36" width="36">
+				<td>{{$stats.CommitCount}}</td>
-							</div>
+			</tr>
-							<div class="github-commit-body">
-								<div class="github-commit-title">{{$commit.Message}}</div>
-								<div class="github-commit-meta">{{$commit.Name}} committed on {{$commit.Date}}</div>
-							</div>
-						</a>
-						<div class="clearfix" />
-					</li>
-				{{end}}
-			</ul>
 		{{end}}
-	</div>
+	</table>
+	{{range $branch := .BranchCommits}}
+		<h4>
+			There are {{ $branch.CommitCount }} commits for branch <a href="{{$branch.URL}}">{{$branch.Name}}</a>.
+		</h4>
+		<div class="github-board">
+			{{range $data := $branch.Days}}
+				<div class="github-group-title">
+					Commits on {{ $data.Day }}
+				</div>
+				<ul class="github-list">
+					{{range $commit := $data.Commits}}
+						<li class="github-commit-item">
+							<a class="link" href="{{$commit.URL}}">
+								<div class="github-avatar">
+									<img alt="@{{$commit.Name}}" src="{{$commit.Avatar}}" height="36" width="36">
+								</div>
+								<div class="github-commit-body">
+									<div class="github-commit-title">{{$commit.Message}}</div>
+									<div class="github-commit-meta">{{$commit.Name}} committed on {{$commit.Date}}</div>
+								</div>
+							</a>
+							<div class="clearfix" />
+						</li>
+					{{end}}
+				</ul>
+			{{end}}
+		</div>
+	{{end}}
 </div>
 `}
 }
 
-func (*Provider) getCommits(client *gogithub.Client, config githubConfig) ([]githubBranchCommits, error) {
+func getCommits(client *gogithub.Client, config *githubConfig) ([]githubBranchCommits, []githubAuthorStats, error) {
 
-	opts := &gogithub.CommitsListOptions{
+	authorStats := make(map[string]githubAuthorStats)
-		SHA:         config.Branch,
-		ListOptions: gogithub.ListOptions{PerPage: config.BranchLines}}
 
-	if config.SincePtr != nil {
+	overall := []githubBranchCommits{}
-		opts.Since = *config.SincePtr
-	}
 
-	guff, _, err := client.Repositories.ListCommits(config.Owner, config.Repo, opts)
+	for _, orb := range config.Lists {
 
-	if err != nil {
+		opts := &gogithub.CommitsListOptions{
-		return nil, err
+			SHA:         config.Branch,
-	}
+			ListOptions: gogithub.ListOptions{PerPage: config.BranchLines}}
 
-	if len(guff) == 0 {
+		if config.SincePtr != nil {
-		return []githubBranchCommits{}, nil
+			opts.Since = *config.SincePtr
-	}
-
-	day := ""
-	newDay := ""
-	ret := []githubBranchCommits{}
-
-	for k, v := range guff {
-
-		if guff[k].Commit != nil {
-			if guff[k].Commit.Committer.Date != nil {
-				y, m, d := (*guff[k].Commit.Committer.Date).Date()
-				newDay = fmt.Sprintf("%s %d, %d", m.String(), d, y)
-			}
 		}
-		if day != newDay {
+
-			day = newDay
+		guff, _, err := client.Repositories.ListCommits(orb.Owner, orb.Repo, opts)
-			ret = append(ret, githubBranchCommits{
+
-				Name: fmt.Sprintf("%s/%s:%s", config.Owner, config.Repo, config.Branch),
+		if err != nil {
-				Day:  day,
+			return nil, nil, err
+		}
+
+		if len(guff) == 0 {
+			return []githubBranchCommits{}, []githubAuthorStats{}, nil
+		}
+
+		day := ""
+		newDay := ""
+		ret := []githubDayCommits{}
+
+		for k, v := range guff {
+
+			if guff[k].Commit != nil {
+				if guff[k].Commit.Committer.Date != nil {
+					y, m, d := (*guff[k].Commit.Committer.Date).Date()
+					newDay = fmt.Sprintf("%s %d, %d", m.String(), d, y)
+				}
+			}
+			if day != newDay {
+				day = newDay
+				ret = append(ret, githubDayCommits{
+					Day: day,
+				})
+			}
+
+			var d, m, u string
+			if v.Commit != nil {
+				if v.Commit.Committer.Date != nil {
+					// d = fmt.Sprintf("%v", *v.Commit.Committer.Date)
+					d = v.Commit.Committer.Date.Format("January 2 2006, 15:04")
+				}
+				if v.Commit.Message != nil {
+					m = *v.Commit.Message
+				}
+			}
+			/* Use author rather than committer
+			var a, l string
+			if v.Committer != nil {
+				if v.Committer.Login != nil {
+					l = *v.Committer.Login
+				}
+				if v.Committer.AvatarURL != nil {
+					a = *v.Committer.AvatarURL
+				}
+			}
+			if a == "" {
+				a = githubGravatar
+			}
+			*/
+
+			if v.HTMLURL != nil {
+				u = *v.HTMLURL
+			}
+
+			//  update of author commits
+			al, aa := "", githubGravatar
+			if v.Author != nil {
+				if v.Author.Login != nil {
+					al = *v.Author.Login
+				}
+				if v.Author.AvatarURL != nil {
+					aa = *v.Author.AvatarURL
+				}
+				cum := authorStats[al]
+				cum.Author = al
+				cum.Avatar = aa
+				cum.CommitCount++
+				/* TODO review, this code removed as too slow
+				cmt, _, err := client.Repositories.GetCommit(orb.Owner, orb.Repo, *v.SHA)
+				if err == nil {
+					if cmt.Stats != nil {
+						if cmt.Stats.Total != nil {
+							cum.TotalChanges += (*cmt.Stats.Total)
+						}
+					}
+				}
+				*/
+				authorStats[al] = cum
+			}
+
+			ret[len(ret)-1].Commits = append(ret[len(ret)-1].Commits, githubCommit{
+				Name:    al,
+				Message: m,
+				Date:    d,
+				Avatar:  aa,
+				URL:     template.URL(u),
 			})
 		}
 
-		var a, d, l, m, u string
+		overall = append(overall, githubBranchCommits{
-		if v.Commit != nil {
+			Name: fmt.Sprintf("%s/%s:%s", orb.Owner, orb.Repo, orb.Name),
-			if v.Commit.Committer.Date != nil {
+			URL:  fmt.Sprintf("https://github.com/%s/%s/tree/%s", orb.Owner, orb.Repo, orb.Name),
-				// d = fmt.Sprintf("%v", *v.Commit.Committer.Date)
+			Days: ret,
-				d = v.Commit.Committer.Date.Format("January 2 2006, 15:04")
-			}
-			if v.Commit.Message != nil {
-				m = *v.Commit.Message
-			}
-		}
-		if v.Committer != nil {
-			if v.Committer.Login != nil {
-				l = *v.Committer.Login
-			}
-			if v.Committer.AvatarURL != nil {
-				a = *v.Committer.AvatarURL
-			}
-		}
-		if a == "" {
-			a = githubGravatar
-		}
-		if v.HTMLURL != nil {
-			u = *v.HTMLURL
-		}
-		ret[len(ret)-1].Commits = append(ret[len(ret)-1].Commits, githubCommit{
-			Name:    l,
-			Message: m,
-			Date:    d,
-			Avatar:  a,
-			URL:     template.URL(u),
 		})
+
 	}
 
-	return ret, nil
+	retStats := make([]githubAuthorStats, 0, len(authorStats))
+	for _, v := range authorStats {
+		retStats = append(retStats, v)
+	}
+	sort.Stable(asToSort(retStats))
+
+	return overall, retStats, nil
 
 }
 
-func commandCommitsData(p *Provider, client *gogithub.Client, config githubConfig, w http.ResponseWriter) {
+func refreshCommits(gr *githubRender, config *githubConfig, client *gogithub.Client) (err error) {
 
-	render, err := p.getCommits(client, config)
+	gr.BranchCommits, gr.AuthorStats, err = getCommits(client, config)
 	if err != nil {
-		log.Error("github getCommits:", err)
+		log.Error("github refreshCommits:", err)
-		provider.WriteError(w, "github", err)
-		return
-	}
-
-	provider.WriteJSON(w, render)
-
-}
-
-func refreshCommits(p *Provider, c githubConfig, data string) string {
-
-	refreshed, err := p.getCommits(p.githubClient(c), c)
-	if err != nil {
-		log.Error("unable to get github commits", err)
-		return data
-	}
-	j, err := json.Marshal(refreshed)
-	if err != nil {
-		log.Error("unable to marshall github commits", err)
-		return data
-	}
-	return string(j)
-
-}
-
-func renderCommits(c *githubConfig, payload *githubRender, data string) error {
-
-	raw := []githubBranchCommits{}
-	if err := json.Unmarshal([]byte(data), &raw); err != nil {
 		return err
 	}
-	c.ReportInfo.ID = tagCommitsData
+	return nil
-	payload.BranchCommits = raw
+}
-	for _, list := range raw {
+
-		payload.CommitCount += len(list.Commits)
+func renderCommits(payload *githubRender, c *githubConfig) error {
+	payload.CommitCount = 0
+	for l, list := range payload.BranchCommits {
+		payload.BranchCommits[l].CommitCount = 0
+		for _, day := range list.Days {
+			payload.BranchCommits[l].CommitCount += len(day.Commits)
+			payload.CommitCount += len(day.Commits)
+		}
 	}
 	return nil
-
 }

core/section/github/github.go

@@ -108,7 +108,7 @@ func (p *Provider) Command(ctx *provider.Context, w http.ResponseWriter, r *http
 	// always use DB version of the token
 	config.Token = ctx.GetSecrets("token") // get the secret token in the database
 
-	client := p.githubClient(config)
+	client := p.githubClient(&config)
 
 	switch method {
 
@@ -132,12 +132,12 @@ func (p *Provider) Command(ctx *provider.Context, w http.ResponseWriter, r *http
 
 		if listFailed(method, config, client, w) {
 
-			rep, ok := reports[method]
+			// TODO refactor - currently treats all remaining commands as triggering a report-set
-			if !ok {
+			gr := githubRender{}
-				log.ErrorString("Github connector unknown method: " + method)
+			for _, rep := range reports {
-				provider.WriteEmpty(w)
+				log.IfErr(rep.refresh(&gr, &config, client))
 			}
-			rep.command(p, client, config, w)
+			provider.WriteJSON(w, &gr)
 
 		}
 
@@ -158,14 +158,19 @@ func (p *Provider) Refresh(ctx *provider.Context, configJSON, data string) strin
 	c.Clean()
 	c.Token = ctx.GetSecrets("token")
 
-	rep, ok := reports[c.ReportInfo.ID]
+	var gr = githubRender{}
-	if !ok {
+	client := p.githubClient(&c)
-		msg := "github report not found for: " + c.ReportInfo.ID
+	for _, rep := range reports {
-		log.ErrorString(msg)
+		log.IfErr(rep.refresh(&gr, &c, client))
-		return "Documize internal error: " + msg
 	}
 
-	return rep.refresh(p, c, data)
+	byts, err := json.Marshal(&gr)
+	if err != nil {
+		log.Error("unable to marshall github data", err)
+		return "internal configuration error '" + err.Error() + "'"
+	}
+
+	return string(byts)
 
 }
 
@@ -186,40 +191,50 @@ func (p *Provider) Render(ctx *provider.Context, config, data string) string {
 	c.Clean()
 	c.Token = ctx.GetSecrets("token")
 
+	err = json.Unmarshal([]byte(data), &payload)
+
+	if err != nil {
+		log.Error("unable to unmarshall github data", err)
+		return "Please delete and recreate this Github section."
+	}
+
 	payload.Config = c
 	payload.Repo = c.RepoInfo
 	payload.Limit = c.BranchLines
-	if len(c.BranchSince) > 0 {
+
-		payload.DateMessage = "created after " + c.BranchSince
+	ret := ""
+	for _, repID := range c.ReportOrder {
+
+		rep, ok := reports[repID]
+		if !ok {
+			msg := "github report not found for: " + repID
+			log.ErrorString(msg)
+			return "Documize internal error: " + msg
+		}
+
+		if err = rep.render(&payload, &c); err != nil {
+			log.Error("unable to render "+repID, err)
+			return "Documize internal github render " + repID + " error: " + err.Error() + "<BR>" + data
+		}
+
+		t := template.New("github")
+
+		t, err = t.Parse(rep.template)
+
+		if err != nil {
+			log.Error("github render template.Parse error:", err)
+			return "Documize internal github template.Parse error: " + err.Error()
+		}
+
+		buffer := new(bytes.Buffer)
+		err = t.Execute(buffer, payload)
+		if err != nil {
+			log.Error("github render template.Execute error:", err)
+			return "Documize internal github template.Execute error: " + err.Error()
+		}
+
+		ret += buffer.String()
+
 	}
-
+	return ret
-	rep, ok := reports[c.ReportInfo.ID]
-	if !ok {
-		msg := "github report not found for: " + c.ReportInfo.ID
-		log.ErrorString(msg)
-		return "Documize internal error: " + msg
-	}
-
-	if err = rep.render(&c, &payload, data); err != nil {
-		log.Error("unable to render "+c.ReportInfo.ID, err)
-		return "Documize internal github render " + c.ReportInfo.ID + " error: " + err.Error() + "<BR>" + data
-	}
-
-	t := template.New("github")
-
-	t, err = t.Parse(rep.template)
-
-	if err != nil {
-		log.Error("github render template.Parse error:", err)
-		return "Documize internal github template.Parse error: " + err.Error()
-	}
-
-	buffer := new(bytes.Buffer)
-	err = t.Execute(buffer, payload)
-	if err != nil {
-		log.Error("github render template.Execute error:", err)
-		return "Documize internal github template.Execute error: " + err.Error()
-	}
-
-	return buffer.String()
 }

core/section/github/issues.go

@@ -12,18 +12,20 @@
 package github
 
 import (
-	"encoding/json"
 	"html/template"
-	"net/http"
+	"sort"
-	"strconv"
+	"time"
-	"strings"
 
 	"github.com/documize/community/core/log"
-	"github.com/documize/community/core/section/provider"
 
 	gogithub "github.com/google/go-github/github"
 )
 
+const (
+	tagIssuesData    = "issuesData"
+	issuesTimeFormat = "January 2 2006, 15:04"
+)
+
 type githubIssue struct {
 	ID      int           `json:"id"`
 	Date    string        `json:"date"`
@@ -34,33 +36,43 @@ type githubIssue struct {
 	Avatar  string        `json:"avatar"`
 	Labels  template.HTML `json:"labels"`
 	IsOpen  bool          `json:"isopen"`
+	Repo    string        `json:"repo"`
 }
 
-const tagIssuesData = "issuesData"
+// sort issues in order that that should be presented - by date updated.
+type issuesToSort []githubIssue
+
+func (s issuesToSort) Len() int      { return len(s) }
+func (s issuesToSort) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+func (s issuesToSort) Less(i, j int) bool {
+	if !s[i].IsOpen && s[j].IsOpen {
+		return true
+	}
+	if s[i].IsOpen && !s[j].IsOpen {
+		return false
+	}
+	// TODO this seems a very slow approach
+	iDate, iErr := time.Parse(issuesTimeFormat, s[i].Updated)
+	log.IfErr(iErr)
+	jDate, jErr := time.Parse(issuesTimeFormat, s[j].Updated)
+	log.IfErr(jErr)
+	return iDate.Before(jDate)
+}
 
 func init() {
-	reports[tagIssuesData] = report{commandIssuesData, refreshIssues, renderIssues, `
+	reports[tagIssuesData] = report{refreshIssues, renderIssues, `
 <div class="section-github-render">
+	<h3>Issues</h3>
 	<p>
-		{{if .ShowIssueNumbers}}
+		During the period since {{.Config.Since}}{{.Config.DateMessage}}, {{.ClosedIssues}} issues were closed, while {{.OpenIssues}} remain open. 
-			Showing Selected Issues
-		{{else}}
-			{{ .Config.IssueState.Name }}
-		{{end}}
-		    for repository <a href="{{ .Repo.URL }}/issues">{{.Repo.Name}}</a>
 		{{if .ShowList}}
-			labelled
+			Labelled
 			{{range $label := .List}}
 				{{if $label.Included}}
 					<span class="github-issue-label" style="background-color:#{{$label.Color}}">{{$label.Name}}</span>
 				{{end}}
 			{{end}}
 		{{end}}
-		{{if .ShowIssueNumbers}}
-			issue(s) {{ .DateMessage }}.
-		{{else}}
-			up to {{ .Limit }} items are shown{{ .DateMessage }}.
-		{{end}}
 	</p>
 	<div class="github-board">
 	<ul class="github-list">
@@ -81,7 +93,7 @@ func init() {
 					<div class="github-commit-body">
 						<div class="github-commit-title"><span class="label-name">{{$data.Message}}</span> {{$data.Labels}}</div>
 						<div class="github-commit-meta">
-							#{{$data.ID}} opened on {{$data.Date}} by {{$data.Name}}, last updated {{$data.Updated}}
+							#{{$data.ID}} opened on {{$data.Date}} by {{$data.Name}} in {{$data.Repo}}, last updated {{$data.Updated}}
 						</div>
 					</div>
 				</a>
@@ -102,142 +114,96 @@ func wrapLabels(labels []gogithub.Label) string {
 	return l
 }
 
-func (*Provider) getIssues(client *gogithub.Client, config githubConfig) ([]githubIssue, error) {
+func getIssues(client *gogithub.Client, config *githubConfig) ([]githubIssue, error) {
 
 	ret := []githubIssue{}
 
-	isRequired := make([]int, 0, 10)
+	hadRepo := make(map[string]bool)
-	for _, s := range strings.Split(strings.Replace(config.IssuesText, "#", "", -1), ",") {
-		i, err := strconv.Atoi(strings.TrimSpace(s))
-		if err == nil {
-			isRequired = append(isRequired, i)
-		}
-	}
-	if len(isRequired) > 0 {
 
-		for _, i := range isRequired {
+	for _, orb := range config.Lists {
 
-			issue, _, err := client.Issues.Get(config.Owner, config.Repo, i)
+		rName := orb.Owner + "/" + orb.Repo
 
-			if err == nil {
+		if !hadRepo[rName] {
-				n := ""
+
-				p := issue.User
+			for _, state := range []string{"open", "closed"} {
-				if p != nil {
+
-					if p.Login != nil {
+				opts := &gogithub.IssueListByRepoOptions{
-						n = *p.Login
+					Sort:        "updated",
+					State:       state,
+					ListOptions: gogithub.ListOptions{PerPage: config.BranchLines}}
+
+				if config.SincePtr != nil && state == "closed" /* we want all the open ones */ {
+					opts.Since = *config.SincePtr
+				}
+
+				/* TODO refactor to select certain lables
+				for _, lab := range config.Lists {
+					if lab.Included {
+						opts.Labels = append(opts.Labels, lab.Name)
 					}
 				}
-				l := wrapLabels(issue.Labels)
+				*/
-				ret = append(ret, githubIssue{
-					Name:    n,
-					Message: *issue.Title,
-					Date:    issue.CreatedAt.Format("January 2 2006, 15:04"),
-					Updated: issue.UpdatedAt.Format("January 2 2006, 15:04"),
-					URL:     template.URL(*issue.HTMLURL),
-					Labels:  template.HTML(l),
-					ID:      *issue.Number,
-					IsOpen:  *issue.State == "open",
-				})
-			}
-		}
 
-	} else {
+				guff, _, err := client.Issues.ListByRepo(orb.Owner, orb.Repo, opts)
 
-		opts := &gogithub.IssueListByRepoOptions{
+				if err != nil {
-			Sort:        "updated",
+					return ret, err
-			State:       config.IssueState.ID,
+				}
-			ListOptions: gogithub.ListOptions{PerPage: config.BranchLines}}
 
-		if config.SincePtr != nil {
+				for _, v := range guff {
-			opts.Since = *config.SincePtr
+					n := ""
-		}
+					ptr := v.User
-
+					if ptr != nil {
-		for _, lab := range config.Lists {
+						if ptr.Login != nil {
-			if lab.Included {
+							n = *ptr.Login
-				opts.Labels = append(opts.Labels, lab.Name)
+						}
-			}
+					}
-		}
+					l := wrapLabels(v.Labels)
-
+					ret = append(ret, githubIssue{
-		guff, _, err := client.Issues.ListByRepo(config.Owner, config.Repo, opts)
+						Name:    n,
-
+						Message: *v.Title,
-		if err != nil {
+						Date:    v.CreatedAt.Format(issuesTimeFormat),
-			return ret, err
+						Updated: v.UpdatedAt.Format(issuesTimeFormat),
-		}
+						URL:     template.URL(*v.HTMLURL),
-
+						Labels:  template.HTML(l),
-		for _, v := range guff {
+						ID:      *v.Number,
-			n := ""
+						IsOpen:  *v.State == "open",
-			ptr := v.User
+						Repo:    rName,
-			if ptr != nil {
+					})
-				if ptr.Login != nil {
-					n = *ptr.Login
 				}
 			}
-			l := wrapLabels(v.Labels)
-			ret = append(ret, githubIssue{
-				Name:    n,
-				Message: *v.Title,
-				Date:    v.CreatedAt.Format("January 2 2006, 15:04"),
-				Updated: v.UpdatedAt.Format("January 2 2006, 15:04"),
-				URL:     template.URL(*v.HTMLURL),
-				Labels:  template.HTML(l),
-				ID:      *v.Number,
-				IsOpen:  *v.State == "open",
-			})
 		}
+		hadRepo[rName] = true
+
 	}
 
+	sort.Stable(issuesToSort(ret))
+
 	return ret, nil
 
 }
 
-func commandIssuesData(p *Provider, client *gogithub.Client, config githubConfig, w http.ResponseWriter) {
+func refreshIssues(gr *githubRender, config *githubConfig, client *gogithub.Client) (err error) {
-	render, err := p.getIssues(client, config)
+	gr.Issues, err = getIssues(client, config)
 	if err != nil {
-		log.Error("github getIssues:", err)
+		log.Error("unable to get github issues (cmd)", err)
-		provider.WriteError(w, "github", err)
+		return err
-		return
 	}
 
-	provider.WriteJSON(w, render)
+	gr.OpenIssues = 0
-}
+	gr.ClosedIssues = 0
-
+	for _, v := range gr.Issues {
-func refreshIssues(p *Provider, c githubConfig, data string) string {
+		if v.IsOpen {
-	refreshed, err := p.getIssues(p.githubClient(c), c)
+			gr.OpenIssues++
-	if err != nil {
+		} else {
-		log.Error("unable to get github issues", err)
+			gr.ClosedIssues++
-		return data
-	}
-	j, err := json.Marshal(refreshed)
-	if err != nil {
-		log.Error("unable to marshall github issues", err)
-		return data
-	}
-	return string(j)
-}
-
-func renderIssues(c *githubConfig, payload *githubRender, data string) error {
-	raw := []githubIssue{}
-
-	if len(data) > 0 {
-		err := json.Unmarshal([]byte(data), &raw)
-		if err != nil {
-			return err
-		}
-	}
-	payload.Issues = raw
-	if strings.TrimSpace(c.IssuesText) != "" {
-		payload.ShowIssueNumbers = true
-		payload.DateMessage = c.IssuesText
-	} else {
-		if len(c.Lists) > 0 {
-			for _, v := range c.Lists {
-				if v.Included {
-					payload.ShowList = true
-					break
-				}
-			}
-			payload.List = c.Lists
 		}
 	}
+
+	return nil
+}
+
+func renderIssues(payload *githubRender, c *githubConfig) error {
 	return nil
 }

core/section/github/lists.go

@@ -114,6 +114,8 @@ func listFailed(method string, config githubConfig, client *gogithub.Client, w h
 		render := make([]githubBranch, len(branches))
 		for kc, vb := range branches {
 			render[kc] = githubBranch{
+				Owner:    config.Owner,
+				Repo:     config.Repo,
 				Name:     *vb.Name,
 				ID:       fmt.Sprintf("%s:%s:%s", config.Owner, config.Repo, *vb.Name),
 				Included: false,

core/section/github/milestones.go

@@ -0,0 +1,183 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>.
+//
+// https://documize.com
+
+package github
+
+import (
+	"fmt"
+	"sort"
+	"time"
+
+	"github.com/documize/community/core/log"
+
+	gogithub "github.com/google/go-github/github"
+)
+
+const (
+	tagMilestonesData    = "milestonesData"
+	milestonesTimeFormat = "January 2 2006"
+)
+
+type githubMilestone struct {
+	Repo         string `json:"repo"`
+	Name         string `json:"name"`
+	URL          string `json:"url"`
+	IsOpen       bool   `json:"isopen"`
+	OpenIssues   int    `json:"openIssues"`
+	ClosedIssues int    `json:"closedIssues"`
+	CompleteMsg  string `json:"completeMsg"`
+	DueDate      string `json:"dueDate"`
+	UpdatedAt    string `json:"updatedAt"`
+}
+
+// sort milestones in order that that should be presented - by date updated.
+
+type milestonesToSort []githubMilestone
+
+func (s milestonesToSort) Len() int      { return len(s) }
+func (s milestonesToSort) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+func (s milestonesToSort) Less(i, j int) bool {
+	if !s[i].IsOpen && s[j].IsOpen {
+		return true
+	}
+	if s[i].IsOpen && !s[j].IsOpen {
+		return false
+	}
+	// TODO this seems a very slow approach
+	iDate, iErr := time.Parse(milestonesTimeFormat, s[i].UpdatedAt)
+	log.IfErr(iErr)
+	jDate, jErr := time.Parse(milestonesTimeFormat, s[j].UpdatedAt)
+	log.IfErr(jErr)
+	return iDate.Before(jDate)
+
+}
+
+func init() {
+	reports[tagMilestonesData] = report{refreshMilestones, renderMilestones, `
+<div class="section-github-render">
+	<h3>Milestones</h3>
+	<div class="github-board">
+	<ul class="github-list">
+		{{range $data := .Milestones}}
+			<li class="github-commit-item">
+				<a class="link" href="{{$data.URL}}">
+					<div class="issue-avatar">
+						{{if $data.IsOpen}}
+							<span title="Open issue">
+								<svg height="16" version="1.1" viewBox="0 0 14 16" width="14"><path d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg>
+							</span>
+						{{else}}
+							<span title="Closed issue">
+								<svg height="16" version="1.1" viewBox="0 0 16 16" width="16"><path d="M7 10h2v2H7v-2zm2-6H7v5h2V4zm1.5 1.5l-1 1L12 9l4-4.5-1-1L12 7l-1.5-1.5zM8 13.7A5.71 5.71 0 0 1 2.3 8c0-3.14 2.56-5.7 5.7-5.7 1.83 0 3.45.88 4.5 2.2l.92-.92A6.947 6.947 0 0 0 8 1C4.14 1 1 4.14 1 8s3.14 7 7 7 7-3.14 7-7l-1.52 1.52c-.66 2.41-2.86 4.19-5.48 4.19v-.01z"></path></svg>
+							</span>
+						{{end}}
+				  	</div>
+					<div class="github-commit-body">
+						<div class="github-commit-title"><span class="label-name">{{$data.Repo}} - {{$data.Name}}</span> </div>
+						<div class="github-commit-meta">
+						  {{$data.DueDate}} Last updated {{$data.UpdatedAt}}. 
+						  {{$data.CompleteMsg}} complete {{$data.OpenIssues}} open {{$data.ClosedIssues}} closed
+						</div>
+					</div>
+				</a>
+				<div class="clearfix" />
+			</li>
+		{{end}}
+	</ul>
+	</div>
+</div>
+`}
+}
+
+func getMilestones(client *gogithub.Client, config *githubConfig) ([]githubMilestone, error) {
+
+	ret := []githubMilestone{}
+
+	hadRepo := make(map[string]bool)
+
+	for _, orb := range config.Lists {
+		rName := orb.Owner + "/" + orb.Repo
+
+		if !hadRepo[rName] {
+
+			for _, state := range []string{"open", "closed"} {
+
+				opts := &gogithub.MilestoneListOptions{
+					Sort:        "updated",
+					State:       state,
+					ListOptions: gogithub.ListOptions{PerPage: config.BranchLines}}
+
+				guff, _, err := client.Issues.ListMilestones(orb.Owner, orb.Repo, opts)
+
+				if err != nil {
+					return ret, err
+				}
+
+				for _, v := range guff {
+					include := true
+					if state == "closed" {
+						if config.SincePtr != nil {
+							if (*config.SincePtr).After(*v.ClosedAt) {
+								include = false
+							}
+						}
+					}
+					if include {
+						dd := "No due date."
+						if v.DueOn != nil {
+							// TODO refactor to add message in red if the milestone is overdue
+							dd = "Due on " + (*v.DueOn).Format(milestonesTimeFormat) + "."
+						}
+						up := ""
+						if v.UpdatedAt != nil {
+							up = (*v.UpdatedAt).Format(milestonesTimeFormat)
+						}
+
+						ret = append(ret, githubMilestone{
+							Repo:         rName,
+							Name:         *v.Title,
+							URL:          *v.HTMLURL,
+							IsOpen:       *v.State == "open",
+							OpenIssues:   *v.OpenIssues,
+							ClosedIssues: *v.ClosedIssues,
+							CompleteMsg:  fmt.Sprintf("%2.0f%%", float64(*v.ClosedIssues*100)/float64(*v.OpenIssues+*v.ClosedIssues)),
+							DueDate:      dd,
+							UpdatedAt:    up,
+						})
+					}
+				}
+
+			}
+
+		}
+		hadRepo[rName] = true
+
+	}
+
+	sort.Stable(milestonesToSort(ret))
+
+	return ret, nil
+
+}
+
+func refreshMilestones(gr *githubRender, config *githubConfig, client *gogithub.Client) (err error) {
+
+	gr.Milestones, err = getMilestones(client, config)
+	if err != nil {
+		log.Error("unable to get github milestones", err)
+		return err
+	}
+	return nil
+}
+
+func renderMilestones(payload *githubRender, c *githubConfig) error {
+	return nil
+}

core/section/github/model.go

@@ -12,7 +12,7 @@
 package github
 
 import (
-	"net/http"
+	"fmt"
 	"time"
 
 	"github.com/documize/community/core/log"
@@ -21,24 +21,27 @@ import (
 )
 
 type githubRender struct {
-	Config           githubConfig
+	Config           githubConfig          `json:"config"`
-	Repo             githubRepo
+	Repo             githubRepo            `json:"repo"`
-	List             []githubBranch
+	List             []githubBranch        `json:"list"`
-	ShowList         bool
+	ShowList         bool                  `json:"showList"`
-	ShowIssueNumbers bool
+	ShowIssueNumbers bool                  `json:"showIssueNumbers"`
-	BranchCommits    []githubBranchCommits
+	BranchCommits    []githubBranchCommits `json:"branchCommits"`
-	CommitCount      int
+	CommitCount      int                   `json:"commitCount"`
-	Issues           []githubIssue
+	Issues           []githubIssue         `json:"issues"`
-	//IssueNum         int
+	OpenIssues       int                   `json:"openIssues"`
-	//IssueNumActivity []githubIssueActivity
+	ClosedIssues     int                   `json:"closedIssues"`
-	Limit       int
+	Limit            int                   `json:"limit"`
-	DateMessage string
+	Milestones       []githubMilestone     `json:"milestones"`
+	PullRequests     []githubPullRequest   `json:"pullRequests"`
+	OpenPRs        int                   `json:"openPRs"`
+	ClosedPRs        int                   `json:"closedPRs"`
+	AuthorStats      []githubAuthorStats   `json:"authorStats"`
 }
 
 type report struct {
-	command  func(*Provider, *gogithub.Client, githubConfig, http.ResponseWriter)
+	refresh  func(*githubRender, *githubConfig, *gogithub.Client) error
-	refresh  func(*Provider, githubConfig, string) string
+	render   func(*githubRender, *githubConfig) error
-	render   func(*githubConfig, *githubRender, string) error
 	template string
 }
 
@@ -66,6 +69,8 @@ type githubRepo struct {
 
 type githubBranch struct {
 	ID       string `json:"id"`
+	Owner    string `json:"owner"`
+	Repo     string `json:"repo"`
 	Name     string `json:"name"`
 	Included bool   `json:"included"`
 	URL      string `json:"url"`
@@ -82,6 +87,7 @@ type githubConfig struct {
 	BranchURL   string         `json:"branchURL"`
 	BranchSince string         `json:"branchSince,omitempty"`
 	SincePtr    *time.Time     `json:"-"`
+	Since       string         `json:"since"`
 	BranchLines int            `json:"branchLines,omitempty,string"`
 	OwnerInfo   githubOwner    `json:"owner"`
 	RepoInfo    githubRepo     `json:"repo"`
@@ -91,7 +97,8 @@ type githubConfig struct {
 	Lists       []githubBranch `json:"lists,omitempty"`
 	IssueState  githubReport   `json:"state,omitempty"`
 	IssuesText  string         `json:"issues,omitempty"`
-	//IssueNum    int            `json:"issueNum,omitempty,string"`
+	ReportOrder []string       `json:"reportOrder,omitempty"`
+	DateMessage string         `json:"dateMessage,omitempty"`
 }
 
 func (c *githubConfig) Clean() {
@@ -117,6 +124,32 @@ func (c *githubConfig) Clean() {
 			c.SincePtr = &since
 		}
 	}
+	if c.SincePtr == nil {
+		c.DateMessage = " (the last 7 days)"
+		since := time.Now().AddDate(0, 0, -7)
+		c.SincePtr = &since
+	} else {
+		c.DateMessage = ""
+	}
+	c.Since = (*c.SincePtr).Format(issuesTimeFormat)
+
+	// TEST DATA INSERTION DEBUG ONLY!
+	debugList := []string{"community", "enterprise", "test-data"}
+	c.Lists = make([]githubBranch, 0, len(debugList))
+	for rid, repo := range debugList {
+		c.Lists = append(c.Lists, githubBranch{
+			ID:       fmt.Sprintf("%d", rid+1),
+			Owner:    "documize",
+			Repo:     repo,
+			Name:     "master",
+			Included: true,
+			URL:      "https://github.com/documize/" + repo + "/tree/master",
+			Color:    "",
+		})
+	}
+	c.ReportOrder = []string{tagMilestonesData, tagIssuesData, tagPullRequestData, tagCommitsData}
+	c.BranchLines = 100 // overide js default of 30 with maximum allowable in one call
+
 }
 
 type githubCallbackT struct {

core/section/github/pullrequests.go

@@ -0,0 +1,182 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>.
+//
+// https://documize.com
+
+package github
+
+import (
+	"sort"
+	"time"
+
+	"github.com/documize/community/core/log"
+
+	gogithub "github.com/google/go-github/github"
+)
+
+const (
+	tagPullRequestData = "pullRequestData"
+	//pullRequestTimeFormat = "January 2 2006"
+)
+
+type githubPullRequest struct {
+	Repo      string `json:"repo"`
+	Name      string `json:"name"`
+	URL       string `json:"url"`
+	IsOpen    bool   `json:"isopen"`
+	UpdatedAt string `json:"updatedAt"`
+}
+
+// sort pull requests in order that that should be presented - by date updated, closed first.
+
+type prToSort []githubPullRequest
+
+func (s prToSort) Len() int      { return len(s) }
+func (s prToSort) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+func (s prToSort) Less(i, j int) bool {
+	if !s[i].IsOpen && s[j].IsOpen {
+		return true
+	}
+	if s[i].IsOpen && !s[j].IsOpen {
+		return false
+	}
+	// TODO this seems a very slow approach
+	iDate, iErr := time.Parse(milestonesTimeFormat, s[i].UpdatedAt)
+	log.IfErr(iErr)
+	jDate, jErr := time.Parse(milestonesTimeFormat, s[j].UpdatedAt)
+	log.IfErr(jErr)
+	return iDate.Before(jDate)
+
+}
+
+func init() {
+	reports[tagPullRequestData] = report{refreshPullReqs, renderPullReqs, `
+<div class="section-github-render">
+	<h3>Pull Requests</h3>
+	<p>
+		During the period since {{.Config.Since}}{{.Config.DateMessage}}, {{.ClosedPRs}} pull requests were closed, while {{.OpenPRs}} remain open. 
+	</p>
+	<div class="github-board">
+	<ul class="github-list">
+		{{range $data := .PullRequests}}
+			<li class="github-commit-item">
+				<a class="link" href="{{$data.URL}}">
+					<div class="issue-avatar">
+						{{if $data.IsOpen}}
+							<span title="Open Pull Request" >
+								<svg aria-hidden="true"  height="16" version="1.1" viewBox="0 0 12 16" width="12"><path d="M11 11.28V5c-.03-.78-.34-1.47-.94-2.06C9.46 2.35 8.78 2.03 8 2H7V0L4 3l3 3V4h1c.27.02.48.11.69.31.21.2.3.42.31.69v6.28A1.993 1.993 0 0 0 10 15a1.993 1.993 0 0 0 1-3.72zm-1 2.92c-.66 0-1.2-.55-1.2-1.2 0-.65.55-1.2 1.2-1.2.65 0 1.2.55 1.2 1.2 0 .65-.55 1.2-1.2 1.2zM4 3c0-1.11-.89-2-2-2a1.993 1.993 0 0 0-1 3.72v6.56A1.993 1.993 0 0 0 2 15a1.993 1.993 0 0 0 1-3.72V4.72c.59-.34 1-.98 1-1.72zm-.8 10c0 .66-.55 1.2-1.2 1.2-.65 0-1.2-.55-1.2-1.2 0-.65.55-1.2 1.2-1.2.65 0 1.2.55 1.2 1.2zM2 4.2C1.34 4.2.8 3.65.8 3c0-.65.55-1.2 1.2-1.2.65 0 1.2.55 1.2 1.2 0 .65-.55 1.2-1.2 1.2z"></path></svg>							
+							</span>
+						{{else}}
+							<span title="Closed Pull Request" >
+								<svg aria-hidden="true" height="8" version="1.1" viewBox="0 0 12 16" width="6"><path d="M11 11.28V5c-.03-.78-.34-1.47-.94-2.06C9.46 2.35 8.78 2.03 8 2H7V0L4 3l3 3V4h1c.27.02.48.11.69.31.21.2.3.42.31.69v6.28A1.993 1.993 0 0 0 10 15a1.993 1.993 0 0 0 1-3.72zm-1 2.92c-.66 0-1.2-.55-1.2-1.2 0-.65.55-1.2 1.2-1.2.65 0 1.2.55 1.2 1.2 0 .65-.55 1.2-1.2 1.2zM4 3c0-1.11-.89-2-2-2a1.993 1.993 0 0 0-1 3.72v6.56A1.993 1.993 0 0 0 2 15a1.993 1.993 0 0 0 1-3.72V4.72c.59-.34 1-.98 1-1.72zm-.8 10c0 .66-.55 1.2-1.2 1.2-.65 0-1.2-.55-1.2-1.2 0-.65.55-1.2 1.2-1.2.65 0 1.2.55 1.2 1.2zM2 4.2C1.34 4.2.8 3.65.8 3c0-.65.55-1.2 1.2-1.2.65 0 1.2.55 1.2 1.2 0 .65-.55 1.2-1.2 1.2z"></path></svg>
+							</span>
+						{{end}}
+				  	</div>
+					<div class="github-commit-body">
+						<div class="github-commit-title"><span class="label-name">{{$data.Repo}} - {{$data.Name}}</span> </div>
+						<div class="github-commit-meta">
+						 Last updated {{$data.UpdatedAt}}
+						</div>
+					</div>
+				</a>
+				<div class="clearfix" />
+			</li>
+		{{end}}
+	</ul>
+	</div>
+</div>
+`}
+}
+
+func getPullReqs(client *gogithub.Client, config *githubConfig) ([]githubPullRequest, error) {
+
+	ret := []githubPullRequest{}
+
+	hadRepo := make(map[string]bool)
+
+	for _, orb := range config.Lists {
+		rName := orb.Owner + "/" + orb.Repo
+
+		if !hadRepo[rName] {
+
+			for _, state := range []string{"open", "closed"} {
+
+				opts := &gogithub.PullRequestListOptions{
+					Sort:        "updated",
+					State:       state,
+					ListOptions: gogithub.ListOptions{PerPage: config.BranchLines}}
+
+				guff, _, err := client.PullRequests.List(orb.Owner, orb.Repo, opts)
+
+				if err != nil {
+					return ret, err
+				}
+
+				for _, v := range guff {
+					include := true
+					if state == "closed" {
+						if config.SincePtr != nil {
+							if (*config.SincePtr).After(*v.ClosedAt) {
+								include = false
+							}
+						}
+					}
+					if include {
+						up := ""
+						if v.UpdatedAt != nil {
+							up = (*v.UpdatedAt).Format(milestonesTimeFormat)
+						}
+
+						ret = append(ret, githubPullRequest{
+							Repo:      rName,
+							Name:      *v.Title,
+							URL:       *v.HTMLURL,
+							IsOpen:    *v.State == "open",
+							UpdatedAt: up,
+						})
+					}
+				}
+
+			}
+
+		}
+		hadRepo[rName] = true
+
+	}
+
+	sort.Stable(prToSort(ret))
+
+	return ret, nil
+
+}
+
+func refreshPullReqs(gr *githubRender, config *githubConfig, client *gogithub.Client) (err error) {
+
+	gr.PullRequests, err = getPullReqs(client, config)
+	if err != nil {
+		log.Error("unable to get github Pull Requests", err)
+		return err
+	}
+
+	gr.OpenPRs = 0
+	gr.ClosedPRs = 0
+	for _, v := range gr.PullRequests {
+		if v.IsOpen {
+			gr.OpenPRs++
+		} else {
+			gr.ClosedPRs++
+		}
+	}
+
+	return nil
+}
+
+func renderPullReqs(payload *githubRender, c *githubConfig) error {
+	return nil
+}

gocheck.sh

@@ -3,7 +3,7 @@
 # run github.com/alecthomas/gometalinter to check correctness, style and error handling
 # also check spelling with github.com/client9/misspell
 # Only set up to look at non-vendored code, should be run from top level
-for dir in $(find documize wordsmith sdk plugin-* -type d -print | grep -v -e "web" | grep -v -e "templates" | sort | tr '\n' ' ') 
+for dir in $(find core sdk plugin-* -type d -print | grep -v -e "web" | grep -v -e "templates" | sort | tr '\n' ' ') 
 do
 	echo "*** " $dir
     gometalinter --vendor --disable='gotype' --deadline=30s $dir | sort 
@@ -12,7 +12,7 @@ done
 
 # run github.com/FiloSottile/vendorcheck (including tests)
 echo "*** vendorcheck"
-for dir in documize sdk 
+for dir in core sdk 
 do 
     cd $dir
     vendorcheck -t . | grep -v 'github.com/documize/community'