major package structure refactoring

cmd/community/main.go

@@ -14,7 +14,7 @@ package main
 
 import (
 	"github.com/documize/community/core/api/endpoint"
-	"github.com/documize/community/core/environment"
+	"github.com/documize/community/core/env"
 	"github.com/documize/community/core/section"
 
 	_ "github.com/documize/community/embed" // the compressed front-end code and static data
@@ -22,7 +22,7 @@ import (
 )
 
 func main() {
-	environment.Parse("db")         // process the db value first
+	env.Parse("db")                 // process the db value first
 	ready := make(chan struct{}, 1) // channel is used for testing
 
 	section.Register()

core/api/convert/html/html.go

@@ -18,12 +18,10 @@ import (
 
 	api "github.com/documize/community/core/convapi"
 	"github.com/documize/community/core/log"
-	"github.com/documize/community/core/utility"
+	"github.com/documize/community/core/stringutil"
-
+	"golang.org/x/net/context"
 	"golang.org/x/net/html"
 	"golang.org/x/net/html/atom"
-
-	"golang.org/x/net/context"
 )
 
 const maxTitle = 2000   // NOTE: must be the same length as database page.title
@@ -75,7 +73,7 @@ func (h *htmlToSplit) testableSplit(request *api.DocumentConversionRequest,
 				if bdy.Type == html.ElementNode && bdy.DataAtom == atom.Body {
 					h.thisSect = api.Page{
 						Level: 1,
-						Title: utility.BeautifyFilename(request.Filename),
+						Title: stringutil.BeautifyFilename(request.Filename),
 						Body:  []byte(``)}
 					err := h.processChildren(bdy)
 					if err != nil {
@@ -147,7 +145,7 @@ func (h *htmlToSplit) renderHeading(c *html.Node, level uint64) error {
 	if err != nil {
 		return err
 	}
-	str, err := utility.HTML(string(byt)).Text(false) // heading text
+	str, err := stringutil.HTML(string(byt)).Text(false) // heading text
 	if err != nil {
 		return err
 	}
@@ -191,7 +189,7 @@ func (h *htmlToSplit) renderAppend(c *html.Node) error {
 	if err != nil {
 		return err
 	}
-	ebyt := utility.EscapeHTMLcomplexCharsByte(byt)
+	ebyt := stringutil.EscapeHTMLcomplexCharsByte(byt)
 	if len(ebyt) > maxBody {
 		msg := fmt.Sprintf("(Documize warning: HTML render element ignored, size of %d exceeded maxBody of %d.)", len(ebyt), maxBody)
 		log.Info(msg)

core/api/endpoint/attachment_endpoint.go

@@ -24,12 +24,10 @@ import (
 	"github.com/documize/community/core/api/request"
 	"github.com/documize/community/core/api/util"
 	"github.com/documize/community/core/log"
-
+	"github.com/documize/community/core/secrets"
-	uuid "github.com/nu7hatch/gouuid"
-
 	"github.com/gorilla/mux"
-
 	_ "github.com/mytrile/mime-ext" // this adds a large number of mime extensions
+	uuid "github.com/nu7hatch/gouuid"
 )
 
 // AttachmentDownload is the end-point that responds to a request for a particular attachment
@@ -198,7 +196,7 @@ func AddAttachments(w http.ResponseWriter, r *http.Request) {
 	a.RefID = refID
 	a.DocumentID = documentID
 	a.Job = job
-	random := util.GenerateSalt()
+	random := secrets.GenerateSalt()
 	a.FileID = random[0:9]
 	a.Filename = filename.Filename
 	a.Data = b.Bytes()

core/api/endpoint/authentication_endpoint.go

@@ -24,8 +24,8 @@ import (
 	"github.com/documize/community/core/api/request"
 	"github.com/documize/community/core/api/util"
 	"github.com/documize/community/core/log"
+	"github.com/documize/community/core/secrets"
 	"github.com/documize/community/core/section/provider"
-	"github.com/documize/community/core/utility"
 	"github.com/documize/community/core/web"
 )
 
@@ -46,7 +46,7 @@ func Authenticate(w http.ResponseWriter, r *http.Request) {
 	// decode what we received
 	data := strings.Replace(authHeader, "Basic ", "", 1)
 
-	decodedBytes, err := utility.DecodeBase64([]byte(data))
+	decodedBytes, err := secrets.DecodeBase64([]byte(data))
 	if err != nil {
 		writeBadRequestError(w, method, "Unable to decode authentication token")
 		return
@@ -85,7 +85,7 @@ func Authenticate(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Password correct and active user
-	if email != strings.TrimSpace(strings.ToLower(user.Email)) || !util.MatchPassword(user.Password, password, user.Salt) {
+	if email != strings.TrimSpace(strings.ToLower(user.Email)) || !secrets.MatchPassword(user.Password, password, user.Salt) {
 		writeUnauthorizedError(w)
 		return
 	}

core/api/endpoint/document_endpoint.go

@@ -23,10 +23,10 @@ import (
 	"github.com/documize/community/core/api/plugins"
 	"github.com/documize/community/core/api/request"
 	"github.com/documize/community/core/api/store"
-	"github.com/documize/community/core/log"
-	"github.com/documize/community/core/utility"
-
 	"github.com/documize/community/core/api/util"
+	"github.com/documize/community/core/log"
+	"github.com/documize/community/core/streamutil"
+	"github.com/documize/community/core/stringutil"
 	"github.com/gorilla/mux"
 )
 
@@ -49,8 +49,8 @@ func SearchDocuments(w http.ResponseWriter, r *http.Request) {
 
 	// Put in slugs for easy UI display of search URL
 	for key, result := range results {
-		result.DocumentSlug = utility.MakeSlug(result.DocumentTitle)
+		result.DocumentSlug = stringutil.MakeSlug(result.DocumentTitle)
-		result.FolderSlug = utility.MakeSlug(result.LabelName)
+		result.FolderSlug = stringutil.MakeSlug(result.LabelName)
 		results[key] = result
 	}
 
@@ -362,16 +362,16 @@ func GetDocumentAsDocx(w http.ResponseWriter, r *http.Request) {
 	html = append(html, []byte("<html><head></head><body>")...)
 	for _, page := range pages {
 		html = append(html, []byte(fmt.Sprintf("<h%d>", page.Level))...)
-		html = append(html, utility.EscapeHTMLcomplexCharsByte([]byte(page.Title))...)
+		html = append(html, stringutil.EscapeHTMLcomplexCharsByte([]byte(page.Title))...)
 		html = append(html, []byte(fmt.Sprintf("</h%d>", page.Level))...)
-		html = append(html, utility.EscapeHTMLcomplexCharsByte([]byte(page.Body))...)
+		html = append(html, stringutil.EscapeHTMLcomplexCharsByte([]byte(page.Body))...)
 	}
 	html = append(html, []byte("</body></html>")...)
 
 	export, err := store.ExportAs(xtn, string(html))
 	log.Error("store.ExportAs()", err)
 
-	w.Header().Set("Content-Disposition", "attachment; filename="+utility.MakeSlug(document.Title)+"."+xtn)
+	w.Header().Set("Content-Disposition", "attachment; filename="+stringutil.MakeSlug(document.Title)+"."+xtn)
 	w.Header().Set("Content-Type", "application/octet-stream")
 	w.Header().Set("Content-Length", fmt.Sprintf("%d", len(export.File)))
 
@@ -402,7 +402,7 @@ func UpdateDocument(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	defer utility.Close(r.Body)
+	defer streamutil.Close(r.Body)
 	body, err := ioutil.ReadAll(r.Body)
 
 	if err != nil {

core/api/endpoint/jwt.go

@@ -21,14 +21,14 @@ import (
 	jwt "github.com/dgrijalva/jwt-go"
 
 	"github.com/documize/community/core/api/request"
-	"github.com/documize/community/core/environment"
+	"github.com/documize/community/core/env"
 	"github.com/documize/community/core/log"
 )
 
 var jwtKey string
 
 func init() {
-	environment.GetString(&jwtKey, "salt", false, "the salt string used to encode JWT tokens, if not set a random value will be generated",
+	env.GetString(&jwtKey, "salt", false, "the salt string used to encode JWT tokens, if not set a random value will be generated",
 		func(t *string, n string) bool {
 			if jwtKey == "" {
 				b := make([]byte, 17)

core/api/endpoint/keycloak.go

@@ -12,24 +12,27 @@
 package endpoint
 
 import (
+	"bytes"
 	"database/sql"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"obiwan/utility"
+	"sort"
+	"strconv"
 	"strings"
 
-	"bytes"
-	"errors"
 	"github.com/documize/community/core/api/endpoint/models"
 	"github.com/documize/community/core/api/entity"
 	"github.com/documize/community/core/api/request"
 	"github.com/documize/community/core/api/util"
 	"github.com/documize/community/core/log"
-	"github.com/documize/community/core/utility"
+	"github.com/documize/community/core/secrets"
-	"sort"
+	"github.com/documize/community/core/streamutil"
-	"strconv"
+	"github.com/documize/community/core/stringutil"
 )
 
 // AuthenticateKeycloak checks Keycloak authentication credentials.
@@ -37,7 +40,7 @@ func AuthenticateKeycloak(w http.ResponseWriter, r *http.Request) {
 	method := "AuthenticateKeycloak"
 	p := request.GetPersister(r)
 
-	defer utility.Close(r.Body)
+	defer streamutil.Close(r.Body)
 	body, err := ioutil.ReadAll(r.Body)
 	if err != nil {
 		writeBadRequestError(w, method, "Bad payload")
@@ -118,9 +121,9 @@ func AuthenticateKeycloak(w http.ResponseWriter, r *http.Request) {
 		user.Firstname = a.Firstname
 		user.Lastname = a.Lastname
 		user.Email = a.Email
-		user.Initials = utility.MakeInitials(user.Firstname, user.Lastname)
+		user.Initials = stringutil.MakeInitials(user.Firstname, user.Lastname)
-		user.Salt = util.GenerateSalt()
+		user.Salt = secrets.GenerateSalt()
-		user.Password = util.GeneratePassword(util.GenerateRandomPassword(), user.Salt)
+		user.Password = secrets.GeneratePassword(secrets.GenerateRandomPassword(), user.Salt)
 
 		err = addUser(p, &user, ac.DefaultPermissionAddSpace)
 		if err != nil {
@@ -429,7 +432,7 @@ func KeycloakUsers(c keycloakConfig) (users []entity.User, err error) {
 		u.Email = kc.Email
 		u.Firstname = kc.Firstname
 		u.Lastname = kc.Lastname
-		u.Initials = utility.MakeInitials(u.Firstname, u.Lastname)
+		u.Initials = stringutil.MakeInitials(u.Firstname, u.Lastname)
 		u.Active = kc.Enabled
 		u.Editor = false
 

core/api/endpoint/label_endpoint.go

@@ -27,7 +27,9 @@ import (
 	"github.com/documize/community/core/api/request"
 	"github.com/documize/community/core/api/util"
 	"github.com/documize/community/core/log"
-	"github.com/documize/community/core/utility"
+	"github.com/documize/community/core/secrets"
+	"github.com/documize/community/core/streamutil"
+	"github.com/documize/community/core/stringutil"
 )
 
 // AddFolder creates a new folder.
@@ -45,7 +47,7 @@ func AddFolder(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	defer utility.Close(r.Body)
+	defer streamutil.Close(r.Body)
 	body, err := ioutil.ReadAll(r.Body)
 
 	if err != nil {
@@ -221,7 +223,7 @@ func UpdateFolder(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	defer utility.Close(r.Body)
+	defer streamutil.Close(r.Body)
 	body, err := ioutil.ReadAll(r.Body)
 
 	if err != nil {
@@ -432,7 +434,7 @@ func SetFolderPermissions(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	defer utility.Close(r.Body)
+	defer streamutil.Close(r.Body)
 	body, err := ioutil.ReadAll(r.Body)
 
 	if err != nil {
@@ -490,7 +492,7 @@ func SetFolderPermissions(w http.ResponseWriter, r *http.Request) {
 	hasEveryoneRole := false
 	roleCount := 0
 
-	url := p.Context.GetAppURL(fmt.Sprintf("s/%s/%s", label.RefID, utility.MakeSlug(label.Name)))
+	url := p.Context.GetAppURL(fmt.Sprintf("s/%s/%s", label.RefID, stringutil.MakeSlug(label.Name)))
 
 	for _, role := range model.Roles {
 		role.OrgID = p.Context.OrgID
@@ -625,7 +627,7 @@ func AcceptSharedFolder(w http.ResponseWriter, r *http.Request) {
 
 	p.Context.OrgID = org.RefID
 
-	defer utility.Close(r.Body)
+	defer streamutil.Close(r.Body)
 	body, err := ioutil.ReadAll(r.Body)
 
 	if err != nil {
@@ -666,7 +668,7 @@ func AcceptSharedFolder(w http.ResponseWriter, r *http.Request) {
 
 	user.Firstname = model.Firstname
 	user.Lastname = model.Lastname
-	user.Initials = utility.MakeInitials(user.Firstname, user.Lastname)
+	user.Initials = stringutil.MakeInitials(user.Firstname, user.Lastname)
 
 	tx, err := request.Db.Beginx()
 
@@ -687,9 +689,9 @@ func AcceptSharedFolder(w http.ResponseWriter, r *http.Request) {
 
 	p.Context.UserID = user.RefID
 
-	salt := util.GenerateSalt()
+	salt := secrets.GenerateSalt()
 
-	log.IfErr(p.UpdateUserPassword(user.RefID, salt, util.GeneratePassword(model.Password, salt)))
+	log.IfErr(p.UpdateUserPassword(user.RefID, salt, secrets.GeneratePassword(model.Password, salt)))
 
 	if err != nil {
 		log.IfErr(tx.Rollback())
@@ -735,7 +737,7 @@ func InviteToFolder(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	defer utility.Close(r.Body)
+	defer streamutil.Close(r.Body)
 	body, err := ioutil.ReadAll(r.Body)
 
 	if err != nil {
@@ -831,13 +833,13 @@ func InviteToFolder(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 
-			url := p.Context.GetAppURL(fmt.Sprintf("s/%s/%s", label.RefID, utility.MakeSlug(label.Name)))
+			url := p.Context.GetAppURL(fmt.Sprintf("s/%s/%s", label.RefID, stringutil.MakeSlug(label.Name)))
 			go mail.ShareFolderExistingUser(email, inviter.Fullname(), url, label.Name, model.Message)
 			log.Info(fmt.Sprintf("%s is sharing space %s with existing user %s", inviter.Email, label.Name, email))
 		} else {
 			// On-board new user
 			if strings.Contains(email, "@") {
-				url := p.Context.GetAppURL(fmt.Sprintf("auth/share/%s/%s", label.RefID, utility.MakeSlug(label.Name)))
+				url := p.Context.GetAppURL(fmt.Sprintf("auth/share/%s/%s", label.RefID, stringutil.MakeSlug(label.Name)))
 				err = inviteNewUserToSharedFolder(p, email, inviter, url, label, model.Message)
 
 				if err != nil {
@@ -882,9 +884,9 @@ func inviteNewUserToSharedFolder(p request.Persister, email string, invitedBy en
 	user.Email = email
 	user.Firstname = email
 	user.Lastname = ""
-	user.Salt = util.GenerateSalt()
+	user.Salt = secrets.GenerateSalt()
-	requestedPassword := util.GenerateRandomPassword()
+	requestedPassword := secrets.GenerateRandomPassword()
-	user.Password = util.GeneratePassword(requestedPassword, user.Salt)
+	user.Password = secrets.GeneratePassword(requestedPassword, user.Salt)
 	userID := util.UniqueID()
 	user.RefID = userID
 

core/api/endpoint/meta_endpoint.go

@@ -21,7 +21,7 @@ import (
 	"github.com/documize/community/core/api/entity"
 	"github.com/documize/community/core/api/request"
 	"github.com/documize/community/core/log"
-	"github.com/documize/community/core/utility"
+	"github.com/documize/community/core/stringutil"
 )
 
 // GetMeta provides org meta data based upon request domain (e.g. acme.documize.com).
@@ -141,7 +141,7 @@ func GetSitemap(w http.ResponseWriter, r *http.Request) {
 
 		for _, folder := range folders {
 			var item sitemapItem
-			item.URL = p.Context.GetAppURL(fmt.Sprintf("s/%s/%s", folder.RefID, utility.MakeSlug(folder.Name)))
+			item.URL = p.Context.GetAppURL(fmt.Sprintf("s/%s/%s", folder.RefID, stringutil.MakeSlug(folder.Name)))
 			item.Date = folder.Revised.Format("2006-01-02T15:04:05.999999-07:00")
 			items = append(items, item)
 		}
@@ -156,7 +156,7 @@ func GetSitemap(w http.ResponseWriter, r *http.Request) {
 		for _, document := range documents {
 			var item sitemapItem
 			item.URL = p.Context.GetAppURL(fmt.Sprintf("s/%s/%s/d/%s/%s",
-				document.FolderID, utility.MakeSlug(document.Folder), document.DocumentID, utility.MakeSlug(document.Document)))
+				document.FolderID, stringutil.MakeSlug(document.Folder), document.DocumentID, stringutil.MakeSlug(document.Document)))
 			item.Date = document.Revised.Format("2006-01-02T15:04:05.999999-07:00")
 			items = append(items, item)
 		}

core/api/endpoint/org_endpoint.go

@@ -21,8 +21,7 @@ import (
 	"github.com/documize/community/core/api/entity"
 	"github.com/documize/community/core/api/request"
 	"github.com/documize/community/core/log"
-	"github.com/documize/community/core/utility"
+	"github.com/documize/community/core/streamutil"
-
 	"github.com/gorilla/mux"
 )
 
@@ -66,7 +65,7 @@ func UpdateOrganization(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	defer utility.Close(r.Body)
+	defer streamutil.Close(r.Body)
 	body, err := ioutil.ReadAll(r.Body)
 
 	if err != nil {

core/api/endpoint/page_endpoint.go

@@ -25,9 +25,8 @@ import (
 	"github.com/documize/community/core/api/util"
 	"github.com/documize/community/core/log"
 	"github.com/documize/community/core/section/provider"
-	"github.com/documize/community/core/utility"
+	"github.com/documize/community/core/streamutil"
 	htmldiff "github.com/documize/html-diff"
-
 	"github.com/gorilla/mux"
 )
 
@@ -54,7 +53,7 @@ func AddDocumentPage(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	defer utility.Close(r.Body)
+	defer streamutil.Close(r.Body)
 	body, err := ioutil.ReadAll(r.Body)
 
 	if err != nil {
@@ -260,7 +259,7 @@ func GetDocumentPagesBatch(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	defer utility.Close(r.Body)
+	defer streamutil.Close(r.Body)
 	body, err := ioutil.ReadAll(r.Body)
 
 	if err != nil {
@@ -389,7 +388,7 @@ func DeleteDocumentPages(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	defer utility.Close(r.Body)
+	defer streamutil.Close(r.Body)
 	body, err := ioutil.ReadAll(r.Body)
 	if err != nil {
 		writeBadRequestError(w, method, "Bad body")
@@ -480,7 +479,7 @@ func UpdateDocumentPage(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	defer utility.Close(r.Body)
+	defer streamutil.Close(r.Body)
 	body, err := ioutil.ReadAll(r.Body)
 	if err != nil {
 		writeBadRequestError(w, method, "Bad request body")
@@ -585,7 +584,7 @@ func ChangeDocumentPageSequence(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	defer utility.Close(r.Body)
+	defer streamutil.Close(r.Body)
 	body, err := ioutil.ReadAll(r.Body)
 
 	if err != nil {
@@ -650,7 +649,7 @@ func ChangeDocumentPageLevel(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	defer utility.Close(r.Body)
+	defer streamutil.Close(r.Body)
 	body, err := ioutil.ReadAll(r.Body)
 
 	if err != nil {

core/api/endpoint/sections_endpoint.go

@@ -22,7 +22,7 @@ import (
 	"github.com/documize/community/core/api/util"
 	"github.com/documize/community/core/log"
 	"github.com/documize/community/core/section/provider"
-	"github.com/documize/community/core/utility"
+	"github.com/documize/community/core/streamutil"
 	"github.com/gorilla/mux"
 )
 
@@ -198,7 +198,7 @@ func AddBlock(w http.ResponseWriter, r *http.Request) {
 	method := "AddBlock"
 	p := request.GetPersister(r)
 
-	defer utility.Close(r.Body)
+	defer streamutil.Close(r.Body)
 	body, err := ioutil.ReadAll(r.Body)
 
 	if err != nil {
@@ -329,7 +329,7 @@ func UpdateBlock(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	defer utility.Close(r.Body)
+	defer streamutil.Close(r.Body)
 	body, err := ioutil.ReadAll(r.Body)
 	if err != nil {
 		writeBadRequestError(w, method, "Bad payload")

core/api/endpoint/server.go

@@ -21,7 +21,7 @@ import (
 	"github.com/documize/community/core"
 	"github.com/documize/community/core/api/plugins"
 	"github.com/documize/community/core/database"
-	"github.com/documize/community/core/environment"
+	"github.com/documize/community/core/env"
 	"github.com/documize/community/core/log"
 	"github.com/documize/community/core/web"
 	"github.com/gorilla/mux"
@@ -45,10 +45,10 @@ func init() {
 	Product.License.Trial = false
 	Product.License.Edition = "Community"
 
-	environment.GetString(&certFile, "cert", false, "the cert.pem file used for https", nil)
+	env.GetString(&certFile, "cert", false, "the cert.pem file used for https", nil)
-	environment.GetString(&keyFile, "key", false, "the key.pem file used for https", nil)
+	env.GetString(&keyFile, "key", false, "the key.pem file used for https", nil)
-	environment.GetString(&port, "port", false, "http/https port number", nil)
+	env.GetString(&port, "port", false, "http/https port number", nil)
-	environment.GetString(&forcePort2SSL, "forcesslport", false, "redirect given http port number to TLS", nil)
+	env.GetString(&forcePort2SSL, "forcesslport", false, "redirect given http port number to TLS", nil)
 
 	log.Info("Server.Init complete")
 }

core/api/endpoint/templates_endpoint.go

@@ -12,16 +12,13 @@
 package endpoint
 
 import (
+	"database/sql"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 
-	"database/sql"
-
-	"github.com/gorilla/mux"
-
 	"github.com/documize/community/core/api/convert"
 	"github.com/documize/community/core/api/endpoint/models"
 	"github.com/documize/community/core/api/entity"
@@ -30,8 +27,10 @@ import (
 	api "github.com/documize/community/core/convapi"
 	"github.com/documize/community/core/event"
 	"github.com/documize/community/core/log"
-	"github.com/documize/community/core/utility"
+	"github.com/documize/community/core/secrets"
-
+	"github.com/documize/community/core/streamutil"
+	"github.com/documize/community/core/stringutil"
+	"github.com/gorilla/mux"
 	uuid "github.com/nu7hatch/gouuid"
 )
 
@@ -51,7 +50,7 @@ func SaveAsTemplate(w http.ResponseWriter, r *http.Request) {
 		Excerpt    string
 	}
 
-	defer utility.Close(r.Body)
+	defer streamutil.Close(r.Body)
 	body, err := ioutil.ReadAll(r.Body)
 
 	if err != nil {
@@ -331,7 +330,7 @@ func StartDocumentFromSavedTemplate(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	defer utility.Close(r.Body)
+	defer streamutil.Close(r.Body)
 	body, err := ioutil.ReadAll(r.Body)
 	if err != nil {
 		writeBadRequestError(w, method, "Bad payload")
@@ -345,7 +344,7 @@ func StartDocumentFromSavedTemplate(w http.ResponseWriter, r *http.Request) {
 	d.Title = docTitle
 	d.Location = fmt.Sprintf("template-%s", templateID)
 	d.Excerpt = "A new document"
-	d.Slug = utility.MakeSlug(d.Title)
+	d.Slug = stringutil.MakeSlug(d.Title)
 	d.Tags = ""
 	d.LabelID = folderID
 	documentID := util.UniqueID()
@@ -439,7 +438,7 @@ func StartDocumentFromSavedTemplate(w http.ResponseWriter, r *http.Request) {
 	for _, a := range attachments {
 		a.DocumentID = documentID
 		a.Job = newUUID.String()
-		random := util.GenerateSalt()
+		random := secrets.GenerateSalt()
 		a.FileID = random[0:9]
 		attachmentID := util.UniqueID()
 		a.RefID = attachmentID

core/api/endpoint/user_endpoint.go

@@ -18,6 +18,8 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"obiwan/utility"
+	"strconv"
 	"strings"
 
 	"github.com/documize/community/core/api/entity"
@@ -25,12 +27,12 @@ import (
 	"github.com/documize/community/core/api/request"
 	"github.com/documize/community/core/api/util"
 	api "github.com/documize/community/core/convapi"
-	"github.com/documize/community/core/log"
-	"github.com/documize/community/core/utility"
-
 	"github.com/documize/community/core/event"
+	"github.com/documize/community/core/log"
+	"github.com/documize/community/core/secrets"
+	"github.com/documize/community/core/streamutil"
+	"github.com/documize/community/core/stringutil"
 	"github.com/gorilla/mux"
-	"strconv"
 )
 
 // AddUser is the endpoint that enables an administrator to add a new user for their orgaisation.
@@ -48,7 +50,7 @@ func AddUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	defer utility.Close(r.Body)
+	defer streamutil.Close(r.Body)
 	body, err := ioutil.ReadAll(r.Body)
 
 	if err != nil {
@@ -85,12 +87,12 @@ func AddUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	userModel.Initials = utility.MakeInitials(userModel.Firstname, userModel.Lastname)
+	userModel.Initials = stringutil.MakeInitials(userModel.Firstname, userModel.Lastname)
 
 	// generate secrets
-	requestedPassword := util.GenerateRandomPassword()
+	requestedPassword := secrets.GenerateRandomPassword()
-	userModel.Salt = util.GenerateSalt()
+	userModel.Salt = secrets.GenerateSalt()
-	userModel.Password = util.GeneratePassword(requestedPassword, userModel.Salt)
+	userModel.Password = secrets.GeneratePassword(requestedPassword, userModel.Salt)
 
 	// only create account if not dupe
 	addUser := true
@@ -419,7 +421,7 @@ func UpdateUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	defer utility.Close(r.Body)
+	defer streamutil.Close(r.Body)
 	body, err := ioutil.ReadAll(r.Body)
 
 	if err != nil {
@@ -457,7 +459,7 @@ func UpdateUser(w http.ResponseWriter, r *http.Request) {
 	p.Context.Transaction = tx
 
 	user.RefID = userID
-	user.Initials = utility.MakeInitials(user.Firstname, user.Lastname)
+	user.Initials = stringutil.MakeInitials(user.Firstname, user.Lastname)
 
 	err = p.UpdateUser(user)
 
@@ -516,7 +518,7 @@ func ChangeUserPassword(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	defer utility.Close(r.Body)
+	defer streamutil.Close(r.Body)
 	body, err := ioutil.ReadAll(r.Body)
 
 	if err != nil {
@@ -548,9 +550,9 @@ func ChangeUserPassword(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	user.Salt = util.GenerateSalt()
+	user.Salt = secrets.GenerateSalt()
 
-	err = p.UpdateUserPassword(userID, user.Salt, util.GeneratePassword(newPassword, user.Salt))
+	err = p.UpdateUserPassword(userID, user.Salt, secrets.GeneratePassword(newPassword, user.Salt))
 
 	if err != nil {
 		writeGeneralSQLError(w, method, err)
@@ -604,7 +606,7 @@ func ForgotUserPassword(w http.ResponseWriter, r *http.Request) {
 	method := "ForgotUserPassword"
 	p := request.GetPersister(r)
 
-	defer utility.Close(r.Body)
+	defer streamutil.Close(r.Body)
 	body, err := ioutil.ReadAll(r.Body)
 
 	if err != nil {
@@ -629,7 +631,7 @@ func ForgotUserPassword(w http.ResponseWriter, r *http.Request) {
 
 	p.Context.Transaction = tx
 
-	token := util.GenerateSalt()
+	token := secrets.GenerateSalt()
 
 	err = p.ForgotUserPassword(user.Email, token)
 
@@ -667,7 +669,7 @@ func ResetUserPassword(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	defer utility.Close(r.Body)
+	defer streamutil.Close(r.Body)
 	body, err := ioutil.ReadAll(r.Body)
 
 	if err != nil {
@@ -696,9 +698,9 @@ func ResetUserPassword(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	user.Salt = util.GenerateSalt()
+	user.Salt = secrets.GenerateSalt()
 
-	err = p.UpdateUserPassword(user.RefID, user.Salt, util.GeneratePassword(newPassword, user.Salt))
+	err = p.UpdateUserPassword(user.RefID, user.Salt, secrets.GeneratePassword(newPassword, user.Salt))
 
 	if err != nil {
 		log.IfErr(tx.Rollback())

core/api/plugins/glick.go

@@ -23,7 +23,7 @@ import (
 	"github.com/documize/community/core/api/convert/md"
 	"github.com/documize/community/core/api/request"
 	api "github.com/documize/community/core/convapi"
-	"github.com/documize/community/core/environment"
+	"github.com/documize/community/core/env"
 	"github.com/documize/community/core/log"
 	"github.com/documize/glick"
 )
@@ -33,9 +33,9 @@ var PluginFile = "DB" // this points to the database
 var insecure = "false"
 
 func init() {
-	environment.GetString(&PluginFile, "plugin", false,
+	env.GetString(&PluginFile, "plugin", false,
 		"the JSON file describing plugins, default 'DB' uses the database config table 'FILEPLUGINS' entry", nil)
-	environment.GetString(&insecure, "insecure", false,
+	env.GetString(&insecure, "insecure", false,
 		"if 'true' allow https endpoints with invalid certificates (only for testing)", nil)
 }
 

core/api/request/account.go

@@ -18,7 +18,7 @@ import (
 
 	"github.com/documize/community/core/api/entity"
 	"github.com/documize/community/core/log"
-	"github.com/documize/community/core/utility"
+	"github.com/documize/community/core/streamutil"
 	"github.com/pkg/errors"
 )
 
@@ -28,7 +28,7 @@ func (p *Persister) AddAccount(account entity.Account) (err error) {
 	account.Revised = time.Now().UTC()
 
 	stmt, err := p.Context.Transaction.Preparex("INSERT INTO account (refid, orgid, userid, admin, editor, active, created, revised) VALUES (?, ?, ?, ?, ?, ?, ?, ?)")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		errors.Wrap(err, "Unable to prepare insert for account")
@@ -48,7 +48,7 @@ func (p *Persister) AddAccount(account entity.Account) (err error) {
 // GetUserAccount returns the database account record corresponding to the given userID, using the client's current organizaion.
 func (p *Persister) GetUserAccount(userID string) (account entity.Account, err error) {
 	stmt, err := Db.Preparex("SELECT a.*, b.company, b.title, b.message, b.domain FROM account a, organization b WHERE b.refid=a.orgid and a.orgid=? and a.userid=?")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare select for account by user %s", userID), err)
@@ -109,7 +109,7 @@ func (p *Persister) UpdateAccount(account entity.Account) (err error) {
 	account.Revised = time.Now().UTC()
 
 	stmt, err := p.Context.Transaction.PrepareNamed("UPDATE account SET userid=:userid, admin=:admin, editor=:editor, active=:active, revised=:revised WHERE orgid=:orgid AND refid=:refid")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare update for account %s", account.RefID), err)

core/api/request/activity.go

@@ -19,7 +19,7 @@ import (
 	"github.com/documize/community/core/api/endpoint/models"
 	"github.com/documize/community/core/api/entity"
 	"github.com/documize/community/core/log"
-	"github.com/documize/community/core/utility"
+	"github.com/documize/community/core/streamutil"
 )
 
 // RecordUserActivity logs user initiated data changes.
@@ -29,7 +29,7 @@ func (p *Persister) RecordUserActivity(activity entity.UserActivity) (err error)
 	activity.Created = time.Now().UTC()
 
 	stmt, err := p.Context.Transaction.Preparex("INSERT INTO useractivity (orgid, userid, labelid, sourceid, sourcetype, activitytype, created) VALUES (?, ?, ?, ?, ?, ?, ?)")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error("Unable to prepare insert RecordUserActivity", err)

core/api/request/attachment.go

@@ -18,7 +18,7 @@ import (
 
 	"github.com/documize/community/core/api/entity"
 	"github.com/documize/community/core/log"
-	"github.com/documize/community/core/utility"
+	"github.com/documize/community/core/streamutil"
 )
 
 // AddAttachment inserts the given record into the database attachement table.
@@ -30,7 +30,7 @@ func (p *Persister) AddAttachment(a entity.Attachment) (err error) {
 	a.Extension = bits[len(bits)-1]
 
 	stmt, err := p.Context.Transaction.Preparex("INSERT INTO attachment (refid, orgid, documentid, job, fileid, filename, data, extension, created, revised) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error("Unable to prepare insert for attachment", err)
@@ -50,7 +50,7 @@ func (p *Persister) AddAttachment(a entity.Attachment) (err error) {
 // GetAttachment returns the database attachment record specified by the parameters.
 func (p *Persister) GetAttachment(orgID, attachmentID string) (attachment entity.Attachment, err error) {
 	stmt, err := Db.Preparex("SELECT id, refid, orgid, documentid, job, fileid, filename, data, extension, created, revised FROM attachment WHERE orgid=? and refid=?")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare select for attachment %s", attachmentID), err)

core/api/request/block.go

@@ -18,7 +18,7 @@ import (
 
 	"github.com/documize/community/core/api/entity"
 	"github.com/documize/community/core/log"
-	"github.com/documize/community/core/utility"
+	"github.com/documize/community/core/streamutil"
 	"github.com/jmoiron/sqlx"
 )
 
@@ -30,7 +30,7 @@ func (p *Persister) AddBlock(b entity.Block) (err error) {
 	b.Revised = time.Now().UTC()
 
 	stmt, err := p.Context.Transaction.Preparex("INSERT INTO block (refid, orgid, labelid, userid, contenttype, pagetype, title, body, excerpt, rawbody, config, externalsource, used, created, revised) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error("Unable to prepare insert AddBlock", err)
@@ -50,7 +50,7 @@ func (p *Persister) AddBlock(b entity.Block) (err error) {
 // GetBlock returns requested reusable content block.
 func (p *Persister) GetBlock(id string) (b entity.Block, err error) {
 	stmt, err := Db.Preparex("SELECT a.id, a.refid, a.orgid, a.labelid, a.userid, a.contenttype, a.pagetype, a.title, a.body, a.excerpt, a.rawbody, a.config, a.externalsource, a.used, a.created, a.revised, b.firstname, b.lastname FROM block a LEFT JOIN user b ON a.userid = b.refid WHERE a.orgid=? AND a.refid=?")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare select GetBlock %s", id), err)
@@ -81,7 +81,7 @@ func (p *Persister) GetBlocksForSpace(labelID string) (b []entity.Block, err err
 // IncrementBlockUsage increments usage counter for content block.
 func (p *Persister) IncrementBlockUsage(id string) (err error) {
 	stmt, err := p.Context.Transaction.Preparex("UPDATE block SET used=used+1, revised=? WHERE orgid=? AND refid=?")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare update IncrementBlockUsage id %s", id), err)
 		return
@@ -99,7 +99,7 @@ func (p *Persister) IncrementBlockUsage(id string) (err error) {
 // DecrementBlockUsage decrements usage counter for content block.
 func (p *Persister) DecrementBlockUsage(id string) (err error) {
 	stmt, err := p.Context.Transaction.Preparex("UPDATE block SET used=used-1, revised=? WHERE orgid=? AND refid=?")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare update DecrementBlockUsage id %s", id), err)
 		return
@@ -117,7 +117,7 @@ func (p *Persister) DecrementBlockUsage(id string) (err error) {
 // RemoveBlockReference clears page.blockid for given blockID.
 func (p *Persister) RemoveBlockReference(id string) (err error) {
 	stmt, err := p.Context.Transaction.Preparex("UPDATE page SET blockid='', revised=? WHERE orgid=? AND blockid=?")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare update RemoveBlockReference id %s", id), err)
 		return
@@ -144,7 +144,7 @@ func (p *Persister) UpdateBlock(b entity.Block) (err error) {
 
 	var stmt *sqlx.NamedStmt
 	stmt, err = p.Context.Transaction.PrepareNamed("UPDATE block SET title=:title, body=:body, excerpt=:excerpt, rawbody=:rawbody, config=:config, revised=:revised WHERE orgid=:orgid AND refid=:refid")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare update UpdateBlock %s", b.RefID), err)

core/api/request/config.go

@@ -15,7 +15,7 @@ import (
 	"bytes"
 	"errors"
 
-	"github.com/documize/community/core/utility"
+	"github.com/documize/community/core/streamutil"
 )
 
 /* NOT CURRENTLY USED
@@ -47,7 +47,7 @@ func ConfigString(area, path string) (ret string) {
 		//fmt.Printf("DEBUG: Unable to prepare select SQL for ConfigString: %s -- error: %v\n", sql, err)
 		return ""
 	}
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	var item = make([]uint8, 0)
 
@@ -84,7 +84,7 @@ func ConfigSet(area, json string) error {
 		//fmt.Printf("DEBUG: Unable to prepare select SQL for ConfigSet: %s -- error: %v\n", sql, err)
 		return err
 	}
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	_, err = stmt.Exec()
 	return err
@@ -107,7 +107,7 @@ func UserConfigGetJSON(orgid, userid, area, path string) (ret string) {
 		//fmt.Printf("DEBUG: Unable to prepare select SQL for ConfigString: %s -- error: %v\n", sql, err)
 		return ""
 	}
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	var item = make([]uint8, 0)
 
@@ -145,7 +145,7 @@ func UserConfigSetJSON(orgid, userid, area, json string) error {
 		//fmt.Printf("DEBUG: Unable to prepare select SQL for UserConfigSetJSON: %s -- error: %v\n", sql, err)
 		return err
 	}
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	_, err = stmt.Exec()
 	return err

core/api/request/context.go

@@ -15,13 +15,13 @@ import (
 	"database/sql"
 	"fmt"
 	"net/http"
+	"strings"
 	"time"
 
 	"github.com/gorilla/context"
 	"github.com/jmoiron/sqlx"
 
 	"github.com/documize/community/core/log"
-	"github.com/documize/community/core/utility"
 )
 
 var rc = Context{}
@@ -76,7 +76,12 @@ func SetContext(r *http.Request, c Context) {
 	c.SSL = r.TLS != nil
 
 	// get user IP from request
-	c.ClientIP = utility.GetRemoteIP(r.RemoteAddr)
+	i := strings.LastIndex(r.RemoteAddr, ":")
+	if i == -1 {
+		c.ClientIP = r.RemoteAddr
+	} else {
+		c.ClientIP = r.RemoteAddr[:i]
+	}
 
 	fip := r.Header.Get("X-Forwarded-For")
 	if len(fip) > 0 {

core/api/request/document.go

@@ -20,7 +20,7 @@ import (
 
 	"github.com/documize/community/core/api/entity"
 	"github.com/documize/community/core/log"
-	"github.com/documize/community/core/utility"
+	"github.com/documize/community/core/streamutil"
 )
 
 // AddDocument inserts the given document record into the document table and audits that it has been done.
@@ -30,7 +30,7 @@ func (p *Persister) AddDocument(document entity.Document) (err error) {
 	document.Revised = document.Created // put same time in both fields
 
 	stmt, err := p.Context.Transaction.Preparex("INSERT INTO document (refid, orgid, labelid, userid, job, location, title, excerpt, slug, tags, template, created, revised) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error("Unable to prepare insert for document", err)
@@ -50,7 +50,7 @@ func (p *Persister) AddDocument(document entity.Document) (err error) {
 // GetDocument fetches the document record with the given id fromt the document table and audits that it has been got.
 func (p *Persister) GetDocument(id string) (document entity.Document, err error) {
 	stmt, err := Db.Preparex("SELECT id, refid, orgid, labelid, userid, job, location, title, excerpt, slug, tags, template, layout, created, revised FROM document WHERE orgid=? and refid=?")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare select for document %s", id), err)
@@ -302,7 +302,7 @@ func (p *Persister) UpdateDocument(document entity.Document) (err error) {
 	document.Revised = time.Now().UTC()
 
 	stmt, err := p.Context.Transaction.PrepareNamed("UPDATE document SET labelid=:labelid, userid=:userid, job=:job, location=:location, title=:title, excerpt=:excerpt, slug=:slug, tags=:tags, template=:template, layout=:layout, revised=:revised WHERE orgid=:orgid AND refid=:refid")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare update for document %s", document.RefID), err)
@@ -339,7 +339,7 @@ func (p *Persister) ChangeDocumentLabel(document, label string) (err error) {
 	revised := time.Now().UTC()
 
 	stmt, err := p.Context.Transaction.Preparex("UPDATE document SET labelid=?, revised=? WHERE orgid=? AND refid=?")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare update for document label change %s", document), err)
@@ -368,7 +368,7 @@ func (p *Persister) ChangeDocumentLabel(document, label string) (err error) {
 // Then audits that move.
 func (p *Persister) MoveDocumentLabel(id, move string) (err error) {
 	stmt, err := p.Context.Transaction.Preparex("UPDATE document SET labelid=? WHERE orgid=? AND labelid=?")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare update for document label move %s", id), err)

core/api/request/init.go

@@ -20,9 +20,9 @@ import (
 	"github.com/jmoiron/sqlx"
 
 	"github.com/documize/community/core/database"
-	"github.com/documize/community/core/environment"
+	"github.com/documize/community/core/env"
 	"github.com/documize/community/core/log"
-	"github.com/documize/community/core/utility"
+	"github.com/documize/community/core/streamutil"
 	"github.com/documize/community/core/web"
 )
 
@@ -49,7 +49,7 @@ func (dr *databaseRequest) MakeTx() (err error) {
 func init() {
 	var err error
 
-	environment.GetString(&connectionString, "db", true,
+	env.GetString(&connectionString, "db", true,
 		`'username:password@protocol(hostname:port)/databasename" for example "fred:bloggs@tcp(localhost:3306)/documize"`,
 		func(*string, string) bool {
 
@@ -134,7 +134,7 @@ func (m *baseManager) Delete(tx *sqlx.Tx, table string, id string) (rows int64,
 		log.Error(fmt.Sprintf("Unable to prepare delete of row in table %s", table), err)
 		return
 	}
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	result, err := stmt.Exec(id)
 
@@ -154,7 +154,7 @@ func (m *baseManager) DeleteConstrained(tx *sqlx.Tx, table string, orgID, id str
 		log.Error(fmt.Sprintf("Unable to prepare constrained delete of row in table %s", table), err)
 		return
 	}
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	result, err := stmt.Exec(orgID, id)
 
@@ -174,7 +174,7 @@ func (m *baseManager) DeleteConstrainedWithID(tx *sqlx.Tx, table string, orgID,
 		log.Error(fmt.Sprintf("Unable to prepare ConstrainedWithID delete of row in table %s", table), err)
 		return
 	}
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	result, err := stmt.Exec(orgID, id)
 

core/api/request/label.go

@@ -17,7 +17,7 @@ import (
 
 	"github.com/documize/community/core/api/entity"
 	"github.com/documize/community/core/log"
-	"github.com/documize/community/core/utility"
+	"github.com/documize/community/core/streamutil"
 )
 
 // AddLabel adds new folder into the store.
@@ -27,7 +27,7 @@ func (p *Persister) AddLabel(l entity.Label) (err error) {
 	l.Revised = time.Now().UTC()
 
 	stmt, err := p.Context.Transaction.Preparex("INSERT INTO label (refid, label, orgid, userid, type, created, revised) VALUES (?, ?, ?, ?, ?, ?, ?)")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error("Unable to prepare insert for label", err)
@@ -47,7 +47,7 @@ func (p *Persister) AddLabel(l entity.Label) (err error) {
 // GetLabel returns a folder from the store.
 func (p *Persister) GetLabel(id string) (label entity.Label, err error) {
 	stmt, err := Db.Preparex("SELECT id,refid,label as name,orgid,userid,type,created,revised FROM label WHERE orgid=? and refid=?")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare select for label %s", id), err)
@@ -113,7 +113,7 @@ func (p *Persister) UpdateLabel(label entity.Label) (err error) {
 	label.Revised = time.Now().UTC()
 
 	stmt, err := p.Context.Transaction.PrepareNamed("UPDATE label SET label=:name, type=:type, userid=:userid, revised=:revised WHERE orgid=:orgid AND refid=:refid")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare update for label %s", label.RefID), err)
@@ -133,7 +133,7 @@ func (p *Persister) UpdateLabel(label entity.Label) (err error) {
 // ChangeLabelOwner transfer folder ownership.
 func (p *Persister) ChangeLabelOwner(currentOwner, newOwner string) (err error) {
 	stmt, err := p.Context.Transaction.Preparex("UPDATE label SET userid=? WHERE userid=? AND orgid=?")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare change label owner for  %s", currentOwner), err)

core/api/request/labelrole.go

@@ -18,7 +18,7 @@ import (
 
 	"github.com/documize/community/core/api/entity"
 	"github.com/documize/community/core/log"
-	"github.com/documize/community/core/utility"
+	"github.com/documize/community/core/streamutil"
 )
 
 // AddLabelRole inserts the given record into the labelrole database table.
@@ -27,7 +27,7 @@ func (p *Persister) AddLabelRole(l entity.LabelRole) (err error) {
 	l.Revised = time.Now().UTC()
 
 	stmt, err := p.Context.Transaction.Preparex("INSERT INTO labelrole (refid, labelid, orgid, userid, canview, canedit, created, revised) VALUES (?, ?, ?, ?, ?, ?, ?, ?)")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error("Unable to prepare insert for label role", err)
@@ -116,7 +116,7 @@ func (p *Persister) DeleteUserFolderRoles(labelID, userID string) (rows int64, e
 // MoveLabelRoles changes the labelid for an organization's labelrole records from previousLabel to newLabel.
 func (p *Persister) MoveLabelRoles(previousLabel, newLabel string) (err error) {
 	stmt, err := p.Context.Transaction.Preparex("UPDATE labelrole SET labelid=? WHERE labelid=? AND orgid=?")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare move label roles for label  %s", previousLabel), err)

core/api/request/link.go

@@ -18,7 +18,7 @@ import (
 	"github.com/documize/community/core/api/entity"
 	"github.com/documize/community/core/api/util"
 	"github.com/documize/community/core/log"
-	"github.com/documize/community/core/utility"
+	"github.com/documize/community/core/streamutil"
 )
 
 // AddContentLink inserts wiki-link into the store.
@@ -28,7 +28,7 @@ func (p *Persister) AddContentLink(l entity.Link) (err error) {
 	l.Revised = time.Now().UTC()
 
 	stmt, err := p.Context.Transaction.Preparex("INSERT INTO link (refid, orgid, folderid, userid, sourcedocumentid, sourcepageid, targetdocumentid, targetid, linktype, orphan, created, revised) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error("Unable to prepare insert for link", err)
@@ -228,7 +228,7 @@ func (p *Persister) MarkOrphanDocumentLink(documentID string) (err error) {
 		return
 	}
 
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	_, err = stmt.Exec(revised, p.Context.OrgID, documentID)
 
@@ -245,7 +245,7 @@ func (p *Persister) MarkOrphanPageLink(pageID string) (err error) {
 		return
 	}
 
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	_, err = stmt.Exec(revised, p.Context.OrgID, pageID)
 
@@ -262,7 +262,7 @@ func (p *Persister) MarkOrphanAttachmentLink(attachmentID string) (err error) {
 		return
 	}
 
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	_, err = stmt.Exec(revised, p.Context.OrgID, attachmentID)
 

core/api/request/organization.go

@@ -19,7 +19,7 @@ import (
 
 	"github.com/documize/community/core/api/entity"
 	"github.com/documize/community/core/log"
-	"github.com/documize/community/core/utility"
+	"github.com/documize/community/core/streamutil"
 	"github.com/documize/community/core/web"
 	"github.com/jmoiron/sqlx"
 )
@@ -31,7 +31,7 @@ func (p *Persister) AddOrganization(org entity.Organization) error {
 
 	stmt, err := p.Context.Transaction.Preparex(
 		"INSERT INTO organization (refid, company, title, message, url, domain, email, allowanonymousaccess, serial, created, revised) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error("Unable to prepare insert for org", err)
@@ -60,7 +60,7 @@ func (p *Persister) AddOrganization(org entity.Organization) error {
 // GetOrganization returns the Organization reocrod from the organization database table with the given id.
 func (p *Persister) GetOrganization(id string) (org entity.Organization, err error) {
 	stmt, err := Db.Preparex("SELECT id, refid, company, title, message, url, domain, service as conversionendpoint, email, serial, active, allowanonymousaccess, authprovider, coalesce(authconfig,JSON_UNQUOTE('{}')) as authconfig, created, revised FROM organization WHERE refid=?")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare select for org %s", id), err)
@@ -87,7 +87,7 @@ func (p *Persister) GetOrganizationByDomain(subdomain string) (org entity.Organi
 		var stmt *sqlx.Stmt
 
 		stmt, err = Db.Preparex("SELECT id, refid, company, title, message, url, domain, service as conversionendpoint, email, serial, active, allowanonymousaccess, authprovider, coalesce(authconfig,JSON_UNQUOTE('{}')) as authconfig, created, revised FROM organization WHERE domain=? AND active=1")
-		defer utility.Close(stmt)
+		defer streamutil.Close(stmt)
 
 		if err != nil {
 			log.Error(fmt.Sprintf("Unable to prepare select for subdomain %s", subdomain), err)
@@ -111,7 +111,7 @@ func (p *Persister) UpdateOrganization(org entity.Organization) (err error) {
 	org.Revised = time.Now().UTC()
 
 	stmt, err := p.Context.Transaction.PrepareNamed("UPDATE organization SET title=:title, message=:message, service=:conversionendpoint, email=:email, allowanonymousaccess=:allowanonymousaccess, revised=:revised WHERE refid=:refid")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare update for org %s", org.RefID), err)
@@ -144,7 +144,7 @@ func (p *Persister) DeleteOrganization(orgID string) (rows int64, err error) {
 // RemoveOrganization sets the orgID organization to be inactive, thus executing a "soft delete" operation.
 func (p *Persister) RemoveOrganization(orgID string) (err error) {
 	stmt, err := p.Context.Transaction.Preparex("UPDATE organization SET active=0 WHERE refid=?")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare soft delete for org %s", orgID), err)
@@ -179,7 +179,7 @@ func (p *Persister) UpdateAuthConfig(org entity.Organization) (err error) {
 		return
 	}
 
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	_, err = stmt.Exec(&org)
 	if err != nil {

core/api/request/page.go

@@ -18,9 +18,9 @@ import (
 
 	"github.com/documize/community/core/api/endpoint/models"
 	"github.com/documize/community/core/api/entity"
-	"github.com/documize/community/core/api/util"
 	"github.com/documize/community/core/log"
-	"github.com/documize/community/core/utility"
+	"github.com/documize/community/core/streamutil"
+	"github.com/documize/community/domain/link"
 	"github.com/jmoiron/sqlx"
 )
 
@@ -51,7 +51,7 @@ func (p *Persister) AddPage(model models.PageModel) (err error) {
 	}
 
 	stmt, err := p.Context.Transaction.Preparex("INSERT INTO page (refid, orgid, documentid, userid, contenttype, pagetype, level, title, body, revisions, sequence, blockid, created, revised) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error("Unable to prepare insert for page", err)
@@ -68,7 +68,7 @@ func (p *Persister) AddPage(model models.PageModel) (err error) {
 	_ = searches.Add(&databaseRequest{OrgID: p.Context.OrgID}, model.Page, model.Page.RefID)
 
 	stmt2, err := p.Context.Transaction.Preparex("INSERT INTO pagemeta (pageid, orgid, userid, documentid, rawbody, config, externalsource, created, revised) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)")
-	defer utility.Close(stmt2)
+	defer streamutil.Close(stmt2)
 
 	if err != nil {
 		log.Error("Unable to prepare insert for page meta", err)
@@ -88,7 +88,7 @@ func (p *Persister) AddPage(model models.PageModel) (err error) {
 // GetPage returns the pageID page record from the page table.
 func (p *Persister) GetPage(pageID string) (page entity.Page, err error) {
 	stmt, err := Db.Preparex("SELECT a.id, a.refid, a.orgid, a.documentid, a.userid, a.contenttype, a.pagetype, a.level, a.sequence, a.title, a.body, a.revisions, a.blockid, a.created, a.revised FROM page a WHERE a.orgid=? AND a.refid=?")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare select for page %s", pageID), err)
@@ -133,7 +133,7 @@ func (p *Persister) GetPagesWhereIn(documentID, inPages string) (pages []entity.
 	args = append(args, inValues...)
 
 	stmt, err := Db.Preparex(sql)
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Failed to prepare select pages for org %s and document %s where in %s", p.Context.OrgID, documentID, inPages), err)
@@ -147,7 +147,7 @@ func (p *Persister) GetPagesWhereIn(documentID, inPages string) (pages []entity.
 		return
 	}
 
-	defer utility.Close(rows)
+	defer streamutil.Close(rows)
 
 	for rows.Next() {
 		page := entity.Page{}
@@ -192,7 +192,7 @@ func (p *Persister) UpdatePage(page entity.Page, refID, userID string, skipRevis
 		var stmt *sqlx.Stmt
 		stmt, err = p.Context.Transaction.Preparex("INSERT INTO revision (refid, orgid, documentid, ownerid, pageid, userid, contenttype, pagetype, title, body, rawbody, config, created, revised) SELECT ? as refid, a.orgid, a.documentid, a.userid as ownerid, a.refid as pageid, ? as userid, a.contenttype, a.pagetype, a.title, a.body, b.rawbody, b.config, ? as created, ? as revised FROM page a, pagemeta b WHERE a.refid=? AND a.refid=b.pageid")
 
-		defer utility.Close(stmt)
+		defer streamutil.Close(stmt)
 
 		if err != nil {
 			log.Error(fmt.Sprintf("Unable to prepare insert for page revision %s", page.RefID), err)
@@ -210,7 +210,7 @@ func (p *Persister) UpdatePage(page entity.Page, refID, userID string, skipRevis
 	// Update page
 	var stmt2 *sqlx.NamedStmt
 	stmt2, err = p.Context.Transaction.PrepareNamed("UPDATE page SET documentid=:documentid, level=:level, title=:title, body=:body, revisions=:revisions, sequence=:sequence, revised=:revised WHERE orgid=:orgid AND refid=:refid")
-	defer utility.Close(stmt2)
+	defer streamutil.Close(stmt2)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare update for page %s", page.RefID), err)
@@ -233,7 +233,7 @@ func (p *Persister) UpdatePage(page entity.Page, refID, userID string, skipRevis
 	// Update revisions counter
 	if !skipRevision {
 		stmt3, err := p.Context.Transaction.Preparex("UPDATE page SET revisions=revisions+1 WHERE orgid=? AND refid=?")
-		defer utility.Close(stmt3)
+		defer streamutil.Close(stmt3)
 
 		if err != nil {
 			log.Error(fmt.Sprintf("Unable to prepare revisions counter update for page %s", page.RefID), err)
@@ -249,7 +249,7 @@ func (p *Persister) UpdatePage(page entity.Page, refID, userID string, skipRevis
 	}
 
 	// find any content links in the HTML
-	links := util.GetContentLinks(page.Body)
+	links := link.GetContentLinks(page.Body)
 
 	// get a copy of previously saved links
 	previousLinks, _ := p.GetPageLinks(page.DocumentID, page.RefID)
@@ -300,7 +300,7 @@ func (p *Persister) UpdatePageMeta(meta entity.PageMeta, updateUserID bool) (err
 
 	var stmt *sqlx.NamedStmt
 	stmt, err = p.Context.Transaction.PrepareNamed("UPDATE pagemeta SET userid=:userid, documentid=:documentid, rawbody=:rawbody, config=:config, externalsource=:externalsource, revised=:revised WHERE orgid=:orgid AND pageid=:pageid")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare update for page meta %s", meta.PageID), err)
@@ -321,7 +321,7 @@ func (p *Persister) UpdatePageMeta(meta entity.PageMeta, updateUserID bool) (err
 // It then propagates that change into the search table and audits that it has occurred.
 func (p *Persister) UpdatePageSequence(documentID, pageID string, sequence float64) (err error) {
 	stmt, err := p.Context.Transaction.Preparex("UPDATE page SET sequence=? WHERE orgid=? AND refid=?")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare update for page %s", pageID), err)
@@ -344,7 +344,7 @@ func (p *Persister) UpdatePageSequence(documentID, pageID string, sequence float
 // It then propagates that change into the search table and audits that it has occurred.
 func (p *Persister) UpdatePageLevel(documentID, pageID string, level int) (err error) {
 	stmt, err := p.Context.Transaction.Preparex("UPDATE page SET level=? WHERE orgid=? AND refid=?")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare update for page %s", pageID), err)
@@ -388,7 +388,7 @@ func (p *Persister) DeletePage(documentID, pageID string) (rows int64, err error
 // GetPageMeta returns the meta information associated with the page.
 func (p *Persister) GetPageMeta(pageID string) (meta entity.PageMeta, err error) {
 	stmt, err := Db.Preparex("SELECT id, pageid, orgid, userid, documentid, rawbody, coalesce(config,JSON_UNQUOTE('{}')) as config, externalsource, created, revised FROM pagemeta WHERE orgid=? AND pageid=?")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare select for pagemeta %s", pageID), err)
@@ -429,7 +429,7 @@ func (p *Persister) GetDocumentPageMeta(documentID string, externalSourceOnly bo
 // GetPageRevision returns the revisionID page revision record.
 func (p *Persister) GetPageRevision(revisionID string) (revision entity.Revision, err error) {
 	stmt, err := Db.Preparex("SELECT id, refid, orgid, documentid, ownerid, pageid, userid, contenttype, pagetype, title, body, coalesce(rawbody, '') as rawbody, coalesce(config,JSON_UNQUOTE('{}')) as config, created, revised FROM revision WHERE orgid=? and refid=?")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare select for revision %s", revisionID), err)

core/api/request/pin.go

@@ -17,7 +17,7 @@ import (
 
 	"github.com/documize/community/core/api/entity"
 	"github.com/documize/community/core/log"
-	"github.com/documize/community/core/utility"
+	"github.com/documize/community/core/streamutil"
 	"github.com/jmoiron/sqlx"
 )
 
@@ -36,7 +36,7 @@ func (p *Persister) AddPin(pin entity.Pin) (err error) {
 	pin.Sequence = maxSeq + 1
 
 	stmt, err := p.Context.Transaction.Preparex("INSERT INTO pin (refid, orgid, userid, labelid, documentid, pin, sequence, created, revised) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error("Unable to prepare insert for pin", err)
@@ -56,7 +56,7 @@ func (p *Persister) AddPin(pin entity.Pin) (err error) {
 // GetPin returns requested pinned item.
 func (p *Persister) GetPin(id string) (pin entity.Pin, err error) {
 	stmt, err := Db.Preparex("SELECT id, refid, orgid, userid, labelid as folderid, documentid, pin, sequence, created, revised FROM pin WHERE orgid=? AND refid=?")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare select for pin %s", id), err)
@@ -91,7 +91,7 @@ func (p *Persister) UpdatePin(pin entity.Pin) (err error) {
 
 	var stmt *sqlx.NamedStmt
 	stmt, err = p.Context.Transaction.PrepareNamed("UPDATE pin SET labelid=:folderid, documentid=:documentid, pin=:pin, sequence=:sequence, revised=:revised WHERE orgid=:orgid AND refid=:refid")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare update for pin %s", pin.RefID), err)
@@ -111,7 +111,7 @@ func (p *Persister) UpdatePin(pin entity.Pin) (err error) {
 // UpdatePinSequence updates existing pinned item sequence number
 func (p *Persister) UpdatePinSequence(pinID string, sequence int) (err error) {
 	stmt, err := p.Context.Transaction.Preparex("UPDATE pin SET sequence=?, revised=? WHERE orgid=? AND userid=? AND refid=?")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare update for pin sequence %s", pinID), err)

core/api/request/search.go

@@ -18,12 +18,12 @@ import (
 	"sync"
 	"time"
 
-	_ "github.com/go-sql-driver/mysql" // required for sqlx but not directly called
-	"github.com/jmoiron/sqlx"
-
 	"github.com/documize/community/core/api/entity"
 	"github.com/documize/community/core/log"
-	"github.com/documize/community/core/utility"
+	"github.com/documize/community/core/streamutil"
+	"github.com/documize/community/core/stringutil"
+	_ "github.com/go-sql-driver/mysql" // required for sqlx but not directly called
+	"github.com/jmoiron/sqlx"
 )
 
 // SearchManager type provides the datastructure for the queues of activity to be serialized through a single background goroutine.
@@ -143,7 +143,7 @@ func (m *SearchManager) Add(request *databaseRequest, page entity.Page, id strin
 func searchAdd(request *databaseRequest, page entity.Page) (err error) {
 	id := page.RefID
 	// translate the html into text for the search
-	nonHTML, err := utility.HTML(page.Body).Text(false)
+	nonHTML, err := stringutil.HTML(page.Body).Text(false)
 	if err != nil {
 		log.Error("Unable to decode the html for searching", err)
 		return
@@ -158,7 +158,7 @@ func searchAdd(request *databaseRequest, page entity.Page) (err error) {
 		log.Error("Unable to prepare insert for search", err)
 		return
 	}
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	_, err = stmt.Exec(nonHTML, id)
 	if err != nil {
@@ -179,7 +179,7 @@ func (m *SearchManager) Update(request *databaseRequest, page entity.Page) (err
 
 func searchUpdate(request *databaseRequest, page entity.Page) (err error) {
 	// translate the html into text for the search
-	nonHTML, err := utility.HTML(page.Body).Text(false)
+	nonHTML, err := stringutil.HTML(page.Body).Text(false)
 	if err != nil {
 		log.Error("Unable to decode the html for searching", err)
 		return
@@ -190,7 +190,7 @@ func searchUpdate(request *databaseRequest, page entity.Page) (err error) {
 		log.Error(fmt.Sprintf("Unable to prepare search update for page %s", page.RefID), err)
 		return err // could have been redefined
 	}
-	defer utility.Close(su)
+	defer streamutil.Close(su)
 
 	_, err = su.Exec(page.Title, nonHTML, page.Sequence, page.Level, page.Revised, page.RefID)
 	if err != nil {
@@ -220,7 +220,7 @@ func searchUpdateDocument(request *databaseRequest, page entity.Page) (err error
 		log.Error(fmt.Sprintf("Unable to prepare search update for document %s", page.DocumentID), err)
 		return err // may have been redefined
 	}
-	defer utility.Close(searchstmt)
+	defer streamutil.Close(searchstmt)
 
 	_, err = searchstmt.Exec(page.Title, page.Body, time.Now().UTC(), page.DocumentID)
 	if err != nil {
@@ -271,7 +271,7 @@ func searchRebuild(request *databaseRequest, page entity.Page) (err error) {
 		log.Error(fmt.Sprintf("Unable to prepare searchRebuild select for docId %s", page.DocumentID), err)
 		return err
 	}
-	defer utility.Close(stmt2)
+	defer streamutil.Close(stmt2)
 	err = stmt2.Select(&pages, page.DocumentID)
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to execute searchRebuild select for docId %s", page.DocumentID), err)
@@ -298,7 +298,7 @@ func searchRebuild(request *databaseRequest, page entity.Page) (err error) {
 			log.Error(fmt.Sprintf("Unable to prepare select from searchRebuild for pageId %s", pages[0].ID), err)
 			return err
 		}
-		defer utility.Close(stmt1)
+		defer streamutil.Close(stmt1)
 
 		err = stmt1.Get(&target, pages[0].ID)
 		if err != nil {
@@ -338,7 +338,7 @@ func searchUpdateSequence(request *databaseRequest, page entity.Page) (err error
 		log.Error(fmt.Sprintf("Unable to prepare search sequence update for page %s", page.RefID), err)
 		return err
 	}
-	defer utility.Close(supdate)
+	defer streamutil.Close(supdate)
 
 	_, err = supdate.Exec(page.Sequence, time.Now().UTC(), page.RefID)
 	if err != nil {
@@ -372,7 +372,7 @@ func searchUpdateLevel(request *databaseRequest, page entity.Page) (err error) {
 		log.Error(fmt.Sprintf("Unable to prepare search level update for page %s", pageID), err)
 		return err
 	}
-	defer utility.Close(supdate)
+	defer streamutil.Close(supdate)
 
 	_, err = supdate.Exec(level, time.Now().UTC(), pageID)
 	if err != nil {
@@ -420,7 +420,7 @@ func (m *SearchManager) GetPageContext(request *databaseRequest, pageID string,
 		log.Error(fmt.Sprintf("Unable to prepare setPageContext select for pageId %s", pageID), err)
 		return nil, err
 	}
-	defer utility.Close(stmt1)
+	defer streamutil.Close(stmt1)
 
 	err = stmt1.Get(&target, pageID)
 	if err != nil {
@@ -440,7 +440,7 @@ func (m *SearchManager) GetPageContext(request *databaseRequest, pageID string,
 			log.Error(fmt.Sprintf("Unable to prepare GetPageContext next select for pageId %s", pageID), err)
 			return nil, err
 		}
-		defer utility.Close(stmt2)
+		defer streamutil.Close(stmt2)
 
 		err = stmt2.Get(&next, target.DocumentID, target.DocumentID, target.Sequence, target.Level-1)
 		if err != nil {

core/api/request/user.go

@@ -20,7 +20,7 @@ import (
 
 	"github.com/documize/community/core/api/entity"
 	"github.com/documize/community/core/log"
-	"github.com/documize/community/core/utility"
+	"github.com/documize/community/core/streamutil"
 )
 
 // AddUser adds the given user record to the user table.
@@ -29,7 +29,7 @@ func (p *Persister) AddUser(user entity.User) (err error) {
 	user.Revised = time.Now().UTC()
 
 	stmt, err := p.Context.Transaction.Preparex("INSERT INTO user (refid, firstname, lastname, email, initials, password, salt, reset, created, revised) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error("Unable to prepare insert for user", err)
@@ -54,7 +54,7 @@ func (p *Persister) AddUser(user entity.User) (err error) {
 // GetUser returns the user record for the given id.
 func (p *Persister) GetUser(id string) (user entity.User, err error) {
 	stmt, err := Db.Preparex("SELECT id, refid, firstname, lastname, email, initials, global, password, salt, reset, created, revised FROM user WHERE refid=?")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare select for user %s", id), err)
@@ -76,7 +76,7 @@ func (p *Persister) GetUserByEmail(email string) (user entity.User, err error) {
 	email = strings.TrimSpace(strings.ToLower(email))
 
 	stmt, err := Db.Preparex("SELECT id, refid, firstname, lastname, email, initials, global, password, salt, reset, created, revised FROM user WHERE TRIM(LOWER(email))=?")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare select for user by email %s", email), err)
@@ -98,7 +98,7 @@ func (p *Persister) GetUserByDomain(domain, email string) (user entity.User, err
 	email = strings.TrimSpace(strings.ToLower(email))
 
 	stmt, err := Db.Preparex("SELECT u.id, u.refid, u.firstname, u.lastname, u.email, u.initials, u.global, u.password, u.salt, u.reset, u.created, u.revised FROM user u, account a, organization o WHERE TRIM(LOWER(u.email))=? AND u.refid=a.userid AND a.orgid=o.refid AND TRIM(LOWER(o.domain))=?")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare GetUserByDomain %s %s", domain, email), err)
@@ -118,7 +118,7 @@ func (p *Persister) GetUserByDomain(domain, email string) (user entity.User, err
 // GetUserByToken returns a user record given a reset token value.
 func (p *Persister) GetUserByToken(token string) (user entity.User, err error) {
 	stmt, err := Db.Preparex("SELECT  id, refid, firstname, lastname, email, initials, global, password, salt, reset, created, revised FROM user WHERE reset=?")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare select for user by token %s", token), err)
@@ -140,7 +140,7 @@ func (p *Persister) GetUserByToken(token string) (user entity.User, err error) {
 // the onboarding process.
 func (p *Persister) GetUserBySerial(serial string) (user entity.User, err error) {
 	stmt, err := Db.Preparex("SELECT id, refid, firstname, lastname, email, initials, global, password, salt, reset, created, revised FROM user WHERE salt=?")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		return
@@ -211,7 +211,7 @@ func (p *Persister) UpdateUser(user entity.User) (err error) {
 
 	stmt, err := p.Context.Transaction.PrepareNamed(
 		"UPDATE user SET firstname=:firstname, lastname=:lastname, email=:email, revised=:revised, initials=:initials WHERE refid=:refid")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to prepare update for user %s", user.RefID), err)
@@ -231,7 +231,7 @@ func (p *Persister) UpdateUser(user entity.User) (err error) {
 // UpdateUserPassword updates a user record with new password and salt values.
 func (p *Persister) UpdateUserPassword(userID, salt, password string) (err error) {
 	stmt, err := p.Context.Transaction.Preparex("UPDATE user SET salt=?, password=?, reset='' WHERE refid=?")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error("Unable to prepare update for user", err)
@@ -257,7 +257,7 @@ func (p *Persister) UpdateUserPassword(userID, salt, password string) (err error
 // DeactiveUser deletes the account record for the given userID and persister.Context.OrgID.
 func (p *Persister) DeactiveUser(userID string) (err error) {
 	stmt, err := p.Context.Transaction.Preparex("DELETE FROM account WHERE userid=? and orgid=?")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error("Unable to prepare update for user", err)
@@ -277,7 +277,7 @@ func (p *Persister) DeactiveUser(userID string) (err error) {
 // ForgotUserPassword sets the password to '' and the reset field to token, for a user identified by email.
 func (p *Persister) ForgotUserPassword(email, token string) (err error) {
 	stmt, err := p.Context.Transaction.Preparex("UPDATE user SET reset=?, password='' WHERE LOWER(email)=?")
-	defer utility.Close(stmt)
+	defer streamutil.Close(stmt)
 
 	if err != nil {
 		log.Error("Unable to prepare update for reset password", err)

core/api/store/store.go

@@ -17,7 +17,7 @@ import (
 
 	"github.com/documize/community/core/api/entity"
 	api "github.com/documize/community/core/convapi"
-	"github.com/documize/community/core/utility"
+	"github.com/documize/community/core/stringutil"
 )
 
 // StorageProvider describes the interface for document conversion and take-on.
@@ -42,7 +42,7 @@ func ConvertFileResult(filename string, fileResult *api.DocumentConversionRespon
 	if fileResult != nil {
 		if len(fileResult.Pages) > 0 {
 			document.Title = fileResult.Pages[0].Title
-			document.Slug = utility.MakeSlug(fileResult.Pages[0].Title)
+			document.Slug = stringutil.MakeSlug(fileResult.Pages[0].Title)
 		}
 		document.Excerpt = fileResult.Excerpt
 	}

core/api/util/encoding.go

@@ -1,31 +0,0 @@
-// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
-//
-// This software (Documize Community Edition) is licensed under
-// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
-//
-// You can operate outside the AGPL restrictions by purchasing
-// Documize Enterprise Edition and obtaining a commercial license
-// by contacting <sales@documize.com>.
-//
-// https://documize.com
-
-package util
-
-import (
-	"bytes"
-	h "html/template"
-	txt "text/template"
-)
-
-// EncodeTextTemplate encodes input using text/template
-func EncodeTextTemplate(html string) (safe string, err error) {
-	var out bytes.Buffer
-	t, err := txt.New("foo").Parse(`{{define "T"}}{{.}}{{end}}`)
-	err = t.ExecuteTemplate(&out, "T", html)
-	return out.String(), err
-}
-
-// EncodeHTMLString encodes HTML string
-func EncodeHTMLString(html string) (safe string) {
-	return h.HTMLEscapeString(html)
-}

core/api/util/encoding_test.go

@@ -1,33 +0,0 @@
-// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
-//
-// This software (Documize Community Edition) is licensed under
-// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
-//
-// You can operate outside the AGPL restrictions by purchasing
-// Documize Enterprise Edition and obtaining a commercial license
-// by contacting <sales@documize.com>.
-//
-// https://documize.com
-
-package util
-
-import "testing"
-
-func TestHTMLEncoding(t *testing.T) {
-	testHTML(t, "<script>alert('test')</script>", "&lt;script&gt;alert(&#39;test&#39;)&lt;/script&gt;")
-	text(t, "<script>alert('test')</script>", "<script>alert('test')</script>")
-}
-
-func testHTML(t *testing.T, in, out string) {
-	got := EncodeHTMLString(in)
-	if got != out {
-		t.Errorf("EncodeHTMLString `%s` got `%s` expected `%s`\n", in, got, out)
-	}
-}
-
-func text(t *testing.T, in, out string) {
-	got, _ := EncodeTextTemplate(in)
-	if got != out {
-		t.Errorf("Html encode `%s` got `%s` expected `%s`\n", in, got, out)
-	}
-}

core/database/check.go

@@ -18,7 +18,7 @@ import (
 	"strings"
 
 	"github.com/documize/community/core/log"
-	"github.com/documize/community/core/utility"
+	"github.com/documize/community/core/streamutil"
 	"github.com/documize/community/core/web"
 	"github.com/jmoiron/sqlx"
 )
@@ -52,7 +52,7 @@ func Check(Db *sqlx.DB, connectionString string) bool {
 		web.SiteMode = web.SiteModeBadDB
 		return false
 	}
-	defer utility.Close(rows)
+	defer streamutil.Close(rows)
 	var version, dbComment, charset, collation string
 	if rows.Next() {
 		err = rows.Scan(&version, &dbComment, &charset, &collation)

core/database/create.go

@@ -20,7 +20,8 @@ import (
 
 	"github.com/documize/community/core/api/util"
 	"github.com/documize/community/core/log"
-	"github.com/documize/community/core/utility"
+	"github.com/documize/community/core/secrets"
+	"github.com/documize/community/core/stringutil"
 	"github.com/documize/community/core/web"
 )
 
@@ -126,7 +127,7 @@ func Create(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	err = setupAccount(details, util.GenerateSalt())
+	err = setupAccount(details, secrets.GenerateSalt())
 	if err != nil {
 		log.Error("database.Create()", err)
 		return
@@ -152,8 +153,8 @@ type onboardRequest struct {
 // Once done, they can then login and use Documize.
 func setupAccount(completion onboardRequest, serial string) (err error) {
 	//accountTitle := "This is where you will find documentation for your all projects. You can customize this message from the settings screen."
-	salt := util.GenerateSalt()
+	salt := secrets.GenerateSalt()
-	password := util.GeneratePassword(completion.Password, salt)
+	password := secrets.GeneratePassword(completion.Password, salt)
 
 	// Allocate organization to the user.
 	orgID := util.UniqueID()
@@ -170,7 +171,7 @@ func setupAccount(completion onboardRequest, serial string) (err error) {
 	userID := util.UniqueID()
 
 	sql = fmt.Sprintf("insert into user (refid, firstname, lastname, email, initials, salt, password, global) values (\"%s\",\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\", 1)",
-		userID, completion.Firstname, completion.Lastname, completion.Email, utility.MakeInitials(completion.Firstname, completion.Lastname), salt, password)
+		userID, completion.Firstname, completion.Lastname, completion.Email, stringutil.MakeInitials(completion.Firstname, completion.Lastname), salt, password)
 	_, err = runSQL(sql)
 
 	if err != nil {

core/database/migrate.go

@@ -22,11 +22,10 @@ import (
 	"strings"
 	"time"
 
-	"github.com/jmoiron/sqlx"
-
 	"github.com/documize/community/core/log"
-	"github.com/documize/community/core/utility"
+	"github.com/documize/community/core/streamutil"
 	"github.com/documize/community/core/web"
+	"github.com/jmoiron/sqlx"
 )
 
 const migrationsDir = "bindata/scripts"
@@ -170,7 +169,7 @@ func getLastMigration(tx *sqlx.Tx) (lastMigration string, err error) {
 	var stmt *sql.Stmt
 	stmt, err = tx.Prepare("SELECT JSON_EXTRACT(`config`,'$.database') FROM `config` WHERE `key` = 'META';")
 	if err == nil {
-		defer utility.Close(stmt)
+		defer streamutil.Close(stmt)
 		var item = make([]uint8, 0)
 
 		row := stmt.QueryRow()

core/env/database.go

@@ -9,5 +9,5 @@
 //
 // https://documize.com
 
-// Package utility contains utility functions used by the whole Documize ecosystem.
+// Package env provides runtime, server level setup and configuration
-package utility
+package env

core/env/flags.go

@@ -9,9 +9,9 @@
 //
 // https://documize.com
 
-// Package environment allow environment variables to be obtained from either the environment or the command line.
+// Package env allow environment variables to be obtained from either the environment or the command line.
 // Environment variables are always uppercase, with the Prefix; flags are always lowercase without.
-package environment
+package env
 
 import (
 	"flag"

core/env/logger.go

@@ -0,0 +1,19 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>.
+//
+// https://documize.com
+
+// Package env provides runtime, server level setup and configuration
+package env
+
+// Logger provides the interface for Documize compatible loggers.
+type Logger interface {
+	Info(message string)
+	Error(message string, err error)
+}

core/env/runtime.go

@@ -0,0 +1,22 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>.
+//
+// https://documize.com
+
+// Package env provides runtime, server level setup and configuration
+package env
+
+import "github.com/jmoiron/sqlx"
+
+// Runtime provides access to database, logger and other server-level scoped objects.
+// Do not share per-request data -- use Context for per-request values.
+type Runtime struct {
+	Db  *sqlx.DB
+	Log Logger
+}

core/log/logger.go

@@ -20,7 +20,7 @@ import (
 
 	log "github.com/Sirupsen/logrus"
 
-	env "github.com/documize/community/core/environment"
+	"github.com/documize/community/core/env"
 )
 
 var environment = "Non-production"

core/osutil/command.go

@@ -9,14 +9,15 @@
 //
 // https://documize.com
 
-package utility
+package osutil
 
 import (
 	"bytes"
 	"errors"
-	"github.com/documize/community/core/log"
 	"os/exec"
 	"time"
+
+	"github.com/documize/community/core/log"
 )
 
 var errTimeout = errors.New("conversion timelimit exceeded")

core/osutil/command_test.go

@@ -9,11 +9,13 @@
 //
 // https://documize.com
 
-package utility
+package osutil
 
-import "testing"
+import (
-import "os/exec"
+	"os/exec"
-import "time"
+	"testing"
+	"time"
+)
 
 func TestCmd(t *testing.T) {
 	cmd := exec.Command("echo", "test")

core/request/context.go

@@ -9,18 +9,5 @@
 //
 // https://documize.com
 
-package utility
+// Package request provides HTTP request parsing functions.
-
+package request
-import (
-	"strings"
-)
-
-// GetRemoteIP returns just the IP and not the port number
-func GetRemoteIP(ip string) string {
-	i := strings.LastIndex(ip, ":")
-	if i == -1 {
-		return ip
-	}
-
-	return ip[:i]
-}

core/request/param.go

@@ -9,7 +9,8 @@
 //
 // https://documize.com
 
-package util
+// Package request provides HTTP request parsing functions.
+package request
 
 import (
 	"net/http"
@@ -17,7 +18,7 @@ import (
 	"github.com/gorilla/mux"
 )
 
-// Params returns the paramaters to a gorilla mux request.
+// Params returns the paramaters from route request.
 func Params(r *http.Request) map[string]string {
 	return mux.Vars(r)
 }

core/request/query.go

@@ -9,17 +9,13 @@
 //
 // https://documize.com
 
-package utility
+// Package request provides HTTP request parsing functions.
+package request
 
-import "testing"
+import "net/http"
-import "os"
-import "github.com/documize/community/core/log"
 
-func TestDefclose(t *testing.T) {
+// Query returns query string from HTTP request.
-	var f *os.File
+func Query(r *http.Request, key string) string {
-	log.TestIfErr = true
+	query := r.URL.Query()
-	Close(f)
+	return query.Get(key)
-	if log.TestIfErr {
-		t.Error("Close() did not error when it should have")
-	}
 }

core/response/write.go

@@ -0,0 +1,130 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>.
+//
+// https://documize.com
+
+// Package response provides functions to write HTTP response.
+package response
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"github.com/documize/community/core/log"
+)
+
+// Helper for writing consistent headers back to HTTP client
+func writeStatus(w http.ResponseWriter, status int) {
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.WriteHeader(status)
+}
+
+// WriteMissingDataError notifies HTTP client of missing data in request.
+func WriteMissingDataError(w http.ResponseWriter, method, parameter string) {
+	writeStatus(w, http.StatusBadRequest)
+	_, err := w.Write([]byte("{Error: 'Missing data'}"))
+	log.IfErr(err)
+	log.Info(fmt.Sprintf("Missing data %s for method %s", parameter, method))
+}
+
+// WriteNotFoundError notifies HTTP client of 'record not found' error.
+func WriteNotFoundError(w http.ResponseWriter, method string, id string) {
+	writeStatus(w, http.StatusNotFound)
+	_, err := w.Write([]byte("{Error: 'Not found'}"))
+	log.IfErr(err)
+	log.Info(fmt.Sprintf("Not found ID %s for method %s", id, method))
+}
+
+// WriteServerError notifies HTTP client of general application error.
+func WriteServerError(w http.ResponseWriter, method string, err error) {
+	writeStatus(w, http.StatusBadRequest)
+	_, err2 := w.Write([]byte("{Error: 'Internal server error'}"))
+	log.IfErr(err2)
+	log.Error(fmt.Sprintf("Internal server error for method %s", method), err)
+}
+
+// WriteDuplicateError notifies HTTP client of duplicate data that has been rejected.
+func WriteDuplicateError(w http.ResponseWriter, method, entity string) {
+	writeStatus(w, http.StatusConflict)
+	_, err := w.Write([]byte("{Error: 'Duplicate record'}"))
+	log.IfErr(err)
+	log.Info(fmt.Sprintf("Duplicate %s record detected for method %s", entity, method))
+}
+
+// WriteUnauthorizedError notifies HTTP client of rejected unauthorized request.
+func WriteUnauthorizedError(w http.ResponseWriter) {
+	writeStatus(w, http.StatusUnauthorized)
+	_, err := w.Write([]byte("{Error: 'Unauthorized'}"))
+	log.IfErr(err)
+}
+
+// WriteForbiddenError notifies HTTP client of request that is not allowed.
+func WriteForbiddenError(w http.ResponseWriter) {
+	writeStatus(w, http.StatusForbidden)
+	_, err := w.Write([]byte("{Error: 'Forbidden'}"))
+	log.IfErr(err)
+}
+
+// WriteBadRequestError notifies HTTP client of rejected request due to bad data within request.
+func WriteBadRequestError(w http.ResponseWriter, method, message string) {
+	writeStatus(w, http.StatusBadRequest)
+	_, err := w.Write([]byte("{Error: 'Bad Request'}"))
+	log.IfErr(err)
+	log.Info(fmt.Sprintf("Bad Request %s for method %s", message, method))
+}
+
+// WriteBadLicense notifies HTTP client of invalid license (402)
+func WriteBadLicense(w http.ResponseWriter) {
+	writeStatus(w, http.StatusPaymentRequired)
+
+	var e struct {
+		Reason string
+	}
+	e.Reason = "invalid or expired Documize license"
+
+	j, _ := json.Marshal(e)
+	_, err := w.Write(j)
+	log.IfErr(err)
+}
+
+// WriteBytes dumps bytes to HTTP response
+func WriteBytes(w http.ResponseWriter, data []byte) {
+	writeStatus(w, http.StatusOK)
+	_, err := w.Write(data)
+	log.IfErr(err)
+}
+
+// WriteString writes string to HTTP response
+func WriteString(w http.ResponseWriter, data string) {
+	writeStatus(w, http.StatusOK)
+	_, err := w.Write([]byte(data))
+	log.IfErr(err)
+}
+
+// WriteEmpty writes empty JSON HTTP response
+func WriteEmpty(w http.ResponseWriter) {
+	writeStatus(w, http.StatusOK)
+	_, err := w.Write([]byte("{}"))
+	log.IfErr(err)
+}
+
+// WriteJSON serializes data as JSON to HTTP response.
+func WriteJSON(w http.ResponseWriter, v interface{}) {
+	writeStatus(w, http.StatusOK)
+
+	j, err := json.Marshal(v)
+
+	if err != nil {
+		log.IfErr(err)
+	}
+
+	_, err = w.Write(j)
+	log.IfErr(err)
+}

core/secrets/aes.go

@@ -9,30 +9,18 @@
 //
 // https://documize.com
 
-package utility
+package secrets
 
 import (
 	"crypto/aes"
 	"crypto/cipher"
 	"crypto/rand"
-	"encoding/base64"
 	"errors"
 	"io"
 )
 
 var key = []byte("8456FHkQW1566etydT46jk39ghjfFhg4") // 32 bytes
 
-// MakeMD5 returns the MD5 hash of a given string, usually a password.
-/*
-func MakeMD5(password string) []byte {
-	hash := md5.New()
-	if _, err := io.WriteString(hash, password); err != nil {
-		log.Error("error in MakeMD5", err)
-	}
-	return hash.Sum(nil)
-}
-*/
-
 // MakeAES creates an AES encryption of of a given string,
 // using a hard-wired key value,
 // suitable for use as an authentication token.
@@ -69,20 +57,3 @@ func DecryptAES(text []byte) ([]byte, error) {
 	cfb.XORKeyStream(text, text)
 	return DecodeBase64(text)
 }
-
-// EncodeBase64 is a convenience function to encode using StdEncoding.
-func EncodeBase64(b []byte) []byte {
-	return []byte(base64.StdEncoding.EncodeToString(b))
-}
-
-// EncodeBase64AsString is a convenience function to encode using StdEncoding.
-/*
-func EncodeBase64AsString(b []byte) string {
-	return base64.StdEncoding.EncodeToString(b)
-}
-*/
-
-// DecodeBase64 is a convenience function to decode using StdEncoding.
-func DecodeBase64(b []byte) ([]byte, error) {
-	return base64.StdEncoding.DecodeString(string(b))
-}

core/secrets/base64.go

@@ -0,0 +1,26 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>.
+//
+// https://documize.com
+
+package secrets
+
+import (
+	"encoding/base64"
+)
+
+// EncodeBase64 is a convenience function to encode using StdEncoding.
+func EncodeBase64(b []byte) []byte {
+	return []byte(base64.StdEncoding.EncodeToString(b))
+}
+
+// DecodeBase64 is a convenience function to decode using StdEncoding.
+func DecodeBase64(b []byte) ([]byte, error) {
+	return base64.StdEncoding.DecodeString(string(b))
+}

core/secrets/password.go

@@ -9,15 +9,14 @@
 //
 // https://documize.com
 
-package util
+package secrets
 
 import (
 	"crypto/rand"
 	"encoding/hex"
 
-	"golang.org/x/crypto/bcrypt"
-
 	"github.com/documize/community/core/log"
+	"golang.org/x/crypto/bcrypt"
 )
 
 // GenerateRandomPassword provides a string suitable for use as a password.

core/secrets/secrets_test.go

@@ -9,7 +9,7 @@
 //
 // https://documize.com
 
-package utility
+package secrets
 
 import "testing"
 

core/streamutil/defclose.go

@@ -9,17 +9,16 @@
 //
 // https://documize.com
 
-package utility
+package streamutil
 
 import "io"
-import "github.com/documize/community/core/log"
 
 // Close is a convenience function to close an io.Closer, usually in a defer.
 func Close(f interface{}) {
 	if f != nil {
 		if ff, ok := f.(io.Closer); ok {
 			if ff != io.Closer(nil) {
-				log.IfErr(ff.Close())
+				// log.IfErr(ff.Close())
 			}
 		}
 	}

core/stringutil/beautify.go

@@ -9,7 +9,7 @@
 //
 // https://documize.com
 
-package utility
+package stringutil
 
 import (
 	"path/filepath"

core/stringutil/beautify_test.go

@@ -9,7 +9,7 @@
 //
 // https://documize.com
 
-package utility
+package stringutil
 
 import "testing"
 

core/stringutil/conjoin.go

@@ -9,7 +9,7 @@
 //
 // https://documize.com
 
-package utility
+package stringutil
 
 import (
 	"strings"

core/stringutil/html.go

@@ -9,7 +9,7 @@
 //
 // https://documize.com
 
-package utility
+package stringutil
 
 import (
 	"bytes"
@@ -20,8 +20,6 @@ import (
 
 	"golang.org/x/net/html"
 	"golang.org/x/net/html/atom"
-
-	"github.com/documize/community/core/log"
 )
 
 // HTML describes a chunk of HTML, Text() method returns plain text.
@@ -34,7 +32,6 @@ func writeText(n *html.Node, b io.Writer, isTest bool) {
 		case html.TextNode:
 			_, err := b.Write([]byte(n.Data + string(rune(0x200B)))) // + http://en.wikipedia.org/wiki/Zero-width_space
 			if err != nil {
-				log.Error("write TextNode", err)
 			}
 			// TODO This use of zero-width-space (subsequently replaced by ' ' or ignored, depending on context)
 			// TODO works well for in-word breaks, but at the expense of concatenating some words in error.
@@ -61,7 +58,6 @@ func writeText(n *html.Node, b io.Writer, isTest bool) {
 									_, err = b.Write([]byte(" [ ] "))
 								}
 								if err != nil {
-									log.Error("write []", err)
 								}
 							}
 							return
@@ -73,7 +69,6 @@ func writeText(n *html.Node, b io.Writer, isTest bool) {
 			default:
 				_, err := b.Write([]byte(" ")) // add a space after each main element
 				if err != nil {
-					log.Error("write space", err)
 				}
 			}
 		}
@@ -155,7 +150,6 @@ func EscapeHTMLcomplexCharsByte(b []byte) []byte {
 		} else {
 			_, err := ret.Write(b[:size])
 			if err != nil {
-				log.Error("EscapeHTMLcomplexCharsByte", err)
 			}
 		}
 		b = b[size:]

core/stringutil/html_test.go

@@ -9,7 +9,7 @@
 //
 // https://documize.com
 
-package utility
+package stringutil
 
 import "testing"
 

core/stringutil/initials.go

@@ -9,7 +9,7 @@
 //
 // https://documize.com
 
-package utility
+package stringutil
 
 import (
 	"strings"

core/stringutil/initials_test.go

@@ -9,7 +9,7 @@
 //
 // https://documize.com
 
-package utility
+package stringutil
 
 import "testing"
 

core/stringutil/slug.go

@@ -9,7 +9,7 @@
 //
 // https://documize.com
 
-package utility
+package stringutil
 
 import (
 	"strings"

core/stringutil/slug_test.go

@@ -9,7 +9,7 @@
 //
 // https://documize.com
 
-package utility
+package stringutil
 
 import "testing"
 

core/stringutil/words.go

@@ -9,13 +9,16 @@
 //
 // https://documize.com
 
-package utility
+package stringutil
 
-import "unicode"
+import (
-import nethtml "golang.org/x/net/html"
+	"unicode"
+
+	nethtml "golang.org/x/net/html"
+)
 
 // Words returns a slice of words, where each word contains no whitespace, and each item of punctuation is its own word.
-// This functionality is provided to enable verification of the text extraction algorithem across different implemntations.
+// This functionality is provided to enable verification of the text extraction algorithm across different implemntations.
 func Words(ch HTML, inSqBr int, testMode bool) ([]string, int, error) {
 	txt, err := ch.Text(testMode)
 	if err != nil {

core/stringutil/words_test.go

@@ -9,7 +9,7 @@
 //
 // https://documize.com
 
-package utility
+package stringutil
 
 import (
 	"sort"

core/web/web.go

@@ -16,8 +16,8 @@ import (
 	"html/template"
 	"net/http"
 
-	"github.com/documize/community/core/api/util"
+	"github.com/documize/community/core/env"
-	"github.com/documize/community/core/environment"
+	"github.com/documize/community/core/secrets"
 )
 
 // SiteMode defines that the web server should show the system to be in a particular state.
@@ -40,8 +40,8 @@ var SiteInfo struct {
 }
 
 func init() {
-	environment.GetString(&SiteMode, "offline", false, "set to '1' for OFFLINE mode", nil) // no sense overriding this setting from the DB
+	env.GetString(&SiteMode, "offline", false, "set to '1' for OFFLINE mode", nil) // no sense overriding this setting from the DB
-	SiteInfo.DBhash = util.GenerateRandomPassword()                                        // do this only once
+	SiteInfo.DBhash = secrets.GenerateRandomPassword()                             // do this only once
 }
 
 // EmbedHandler is defined in each embed directory

domain/link/link.go

@@ -9,14 +9,13 @@
 //
 // https://documize.com
 
-package util
+package link
 
 import (
 	"strings"
 
-	"golang.org/x/net/html"
-
 	"github.com/documize/community/core/api/entity"
+	"golang.org/x/net/html"
 )
 
 // GetContentLinks returns Documize generated <a> links.

domain/space/endpoint.go

@@ -0,0 +1,14 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>.
+//
+// https://documize.com
+
+// Package space handles API calls and persistence for spaces.
+// Spaces in Documize contain documents.
+package space

domain/space/model.go

@@ -0,0 +1,14 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>.
+//
+// https://documize.com
+
+// Package space handles API calls and persistence for spaces.
+// Spaces in Documize contain documents.
+package space

domain/space/permission.go

@@ -0,0 +1,14 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>.
+//
+// https://documize.com
+
+// Package space handles API calls and persistence for spaces.
+// Spaces in Documize contain documents.
+package space

domain/space/store_mysql.go

@@ -0,0 +1,14 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>.
+//
+// https://documize.com
+
+// Package space handles API calls and persistence for spaces.
+// Spaces in Documize contain documents.
+package space

domain/space/storer.go

@@ -0,0 +1,14 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>.
+//
+// https://documize.com
+
+// Package space handles API calls and persistence for spaces.
+// Spaces in Documize contain documents.
+package space