restructure directories

core/utility/beautify.go

@@ -0,0 +1,66 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under 
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>. 
+//
+// https://documize.com
+
+package utility
+
+import (
+	"path/filepath"
+	"strings"
+	"unicode"
+)
+
+// BeautifyFilename takes a filename and attempts to turn it into a readable form,
+// as TitleCase natural language, suitable for the top level of a Document.
+func BeautifyFilename(fn string) string {
+	_, file := filepath.Split(fn)
+	splits := strings.Split(file, ".")
+	r := []rune(strings.Join(splits[:len(splits)-1], "."))
+
+	// make any non-letter/digit characters space
+	for i := range r {
+		if !(unicode.IsLetter(r[i]) || unicode.IsDigit(r[i]) || r[i] == '.') {
+			r[i] = ' '
+		}
+	}
+
+	// insert spaces in front of any Upper/Lowwer 2-letter combinations
+addSpaces:
+	for i := range r {
+		if i > 1 { // do not insert a space at the start of the file name
+			if unicode.IsLower(r[i]) && unicode.IsUpper(r[i-1]) && r[i-2] != ' ' {
+				n := make([]rune, len(r)+1)
+				for j := 0; j < i-1; j++ {
+					n[j] = r[j]
+				}
+				n[i-1] = ' '
+				for j := i - 1; j < len(r); j++ {
+					n[j+1] = r[j]
+				}
+				r = n
+				goto addSpaces
+			}
+		}
+	}
+
+	// make the first letter of each word upper case
+	for i := range r {
+		switch i {
+		case 0:
+			r[i] = unicode.ToUpper(r[i])
+		case 1: // the zero element should never be space
+		default:
+			if r[i-1] == ' ' {
+				r[i] = unicode.ToUpper(r[i])
+			}
+		}
+	}
+	return string(r)
+}

core/utility/beautify_test.go

@@ -0,0 +1,25 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under 
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>. 
+//
+// https://documize.com
+
+package utility
+
+import "testing"
+
+func TestBeautify(t *testing.T) {
+	bs(t, "DooDah$day.zip", "Doo Dah Day")
+}
+
+func bs(t *testing.T, in, out string) {
+	got := BeautifyFilename(in)
+	if got != out {
+		t.Errorf("BeautifyFilename input `%s` got `%s` expected `%s`\n", in, got, out)
+	}
+}

core/utility/command.go

@@ -0,0 +1,59 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under 
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>. 
+//
+// https://documize.com
+
+package utility
+
+import (
+	"bytes"
+	"github.com/documize/community/core/log"
+	"errors"
+	"os/exec"
+	"time"
+)
+
+var errTimeout = errors.New("conversion timelimit exceeded")
+
+// CommandWithTimeout runs a command but stops it if it does not finish within the timout above.
+func CommandWithTimeout(command *exec.Cmd, timeout time.Duration) ([]byte, error) {
+	var output bytes.Buffer
+	//fmt.Println("DEBUG CommandWithTimeout: %v", command.Args)
+	command.Stdout = &output
+	command.Stderr = &output
+	if err := command.Start(); err != nil {
+		return nil, err
+	}
+	done := make(chan error, 1)
+	defer close(done)
+	go func() {
+		done <- command.Wait()
+	}()
+	select {
+	case <-time.After(timeout):
+		if err := command.Process.Kill(); err != nil {
+			log.Error("failed to kill: ", err)
+		}
+		<-done // prevent memory leak
+		//fmt.Println("DEBUG timeout")
+		return nil, errTimeout
+	case err := <-done:
+		if err != nil {
+			//fmt.Println("DEBUG error return")
+			return output.Bytes(), err
+		}
+		if !command.ProcessState.Success() {
+			err = errors.New(string(output.Bytes()))
+			//fmt.Println("DEBUG not successful")
+			return nil, err
+		}
+		//fmt.Println("DEBUG successful return")
+		return output.Bytes(), nil
+	}
+}

core/utility/command_test.go

@@ -0,0 +1,39 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under 
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>. 
+//
+// https://documize.com
+
+package utility
+
+import "testing"
+import "os/exec"
+import "time"
+
+func TestCmd(t *testing.T) {
+	cmd := exec.Command("echo", "test")
+	buf, err := CommandWithTimeout(cmd,time.Second)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if string(buf) != "test\n" {
+		t.Error("command did not return `test` it returned:" + string(buf))
+	}
+	cmd2 := exec.Command("dingbat doodah")
+	_, err2 := CommandWithTimeout(cmd2,time.Second)
+	if err2 == nil {
+		t.Error("bad command did not return an error")
+	}
+	timeout := 5 * time.Second
+	cmd3 := exec.Command("sleep", "50")
+	_, err3 := CommandWithTimeout(cmd3,timeout)
+	if err3 != errTimeout {
+		t.Error("sleep command did not timeout:", err3)
+	}
+}

core/utility/defclose.go

@@ -0,0 +1,26 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under 
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>. 
+//
+// https://documize.com
+
+package utility
+
+import "io"
+import "github.com/documize/community/core/log"
+
+// Close is a convenience function to close an io.Closer, usually in a defer.
+func Close(f interface{}) {
+	if f != nil {
+		if ff, ok := f.(io.Closer); ok {
+			if ff != io.Closer(nil) {
+				log.IfErr(ff.Close())
+			}
+		}
+	}
+}

core/utility/defclose_test.go

@@ -0,0 +1,25 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under 
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>. 
+//
+// https://documize.com
+
+package utility
+
+import "testing"
+import "os"
+import "github.com/documize/community/core/log"
+
+func TestDefclose(t *testing.T) {
+	var f *os.File
+	log.TestIfErr=true
+	Close(f)
+	if log.TestIfErr {
+		t.Error("Close() did not error when it should have")
+	}
+}

core/utility/doc.go

@@ -0,0 +1,13 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under 
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>. 
+//
+// https://documize.com
+
+// Package utility contains utility functions used by the whole Documize ecosystem.
+package utility

core/utility/html.go

@@ -0,0 +1,164 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>.
+//
+// https://documize.com
+
+package utility
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"strings"
+	"unicode/utf8"
+
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+
+	"github.com/documize/community/core/log"
+)
+
+// HTML describes a chunk of HTML, Text() method returns plain text.
+type HTML string
+
+// write out the textual element of the html node, if present, then iterate through the child nodes.
+func writeText(n *html.Node, b io.Writer, isTest bool) {
+	if !excluded(n) {
+		switch n.Type {
+		case html.TextNode:
+			_, err := b.Write([]byte(n.Data + string(rune(0x200B)))) // + http://en.wikipedia.org/wiki/Zero-width_space
+			if err != nil {
+				log.Error("write TextNode", err)
+			}
+			// TODO This use of zero-width-space (subsequently replaced by ' ' or ignored, depending on context)
+			// TODO works well for in-word breaks, but at the expense of concatenating some words in error.
+			// TODO It may be that better examination of the HTML structure could be used to determine
+			// TODO when a space is, or is not, required. In that event we would not use zero-width-space.
+
+		default:
+			for c := n.FirstChild; c != nil; c = c.NextSibling {
+				writeText(c, b, isTest)
+			}
+			switch n.DataAtom {
+			case 0:
+				if n.Data == "documize" {
+					for _, a := range n.Attr {
+						if a.Key == "type" {
+							if isTest {
+								var err error
+								switch a.Val {
+								case "field-start":
+									_, err = b.Write([]byte(" [ "))
+								case "field-end":
+									_, err = b.Write([]byte(" ] "))
+								default:
+									_, err = b.Write([]byte(" [ ] "))
+								}
+								if err != nil {
+									log.Error("write []", err)
+								}
+							}
+							return
+						}
+					}
+				}
+			case atom.Span, atom.U, atom.B, atom.I, atom.Del, atom.Sub, atom.Sup:
+				//NoOp
+			default:
+				_, err := b.Write([]byte(" ")) // add a space after each main element
+				if err != nil {
+					log.Error("write space", err)
+				}
+			}
+		}
+	}
+}
+
+func excluded(n *html.Node) bool {
+	if n.DataAtom == atom.Div {
+		for _, a := range n.Attr {
+			if a.Key == "class" {
+				switch a.Val {
+				case "documize-first-page",
+					"documize-exotic-image",
+					"documize-footnote",
+					"documize-graphictext",
+					"documize-math":
+					return true
+				}
+			}
+		}
+	}
+	return false
+}
+
+// findBody finds the body HTML node if it exists in the tree. Required to bypass the page title text.
+func findBody(n *html.Node) *html.Node {
+	if n.DataAtom == atom.Body {
+		return n
+	}
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		r := findBody(c)
+		if r != nil {
+			return r
+		}
+	}
+	return nil
+}
+
+// Text returns only the plain text elements of the HTML Chunk, concatanated with "\n",
+// for use in the TOC or for text indexing.
+func (ch HTML) Text(isTest bool) (string, error) {
+	var b bytes.Buffer
+	doc, err := html.Parse(strings.NewReader(string(ch)))
+	if err != nil {
+		return "", err
+	}
+	body := findBody(doc)
+	if body == nil {
+		body = doc
+	}
+	writeText(body, &b, isTest)
+	return string(b.Bytes()), nil
+}
+
+// EscapeHTMLcomplexChars looks for "complex" characters within HTML
+// and replaces them with the HTML escape codes which describe them.
+// "Complex" characters are those encoded in more than one byte by UTF8.
+func EscapeHTMLcomplexChars(s string) string {
+	ret := ""
+	for _, r := range s {
+		if utf8.RuneLen(r) > 1 {
+			ret += fmt.Sprintf("&#%d;", r)
+		} else {
+			ret += string(r)
+		}
+	}
+	return ret
+}
+
+// EscapeHTMLcomplexCharsByte looks for "complex" characters within HTML
+// and replaces them with the HTML escape codes which describe them.
+// "Complex" characters are those encoded in more than one byte by UTF8.
+func EscapeHTMLcomplexCharsByte(b []byte) []byte {
+	var ret bytes.Buffer
+	for len(b) > 0 {
+		r, size := utf8.DecodeRune(b)
+		if utf8.RuneLen(r) > 1 {
+			fmt.Fprintf(&ret, "&#%d;", r)
+		} else {
+			_, err := ret.Write(b[:size])
+			if err != nil {
+				log.Error("EscapeHTMLcomplexCharsByte", err)
+			}
+		}
+		b = b[size:]
+	}
+	return ret.Bytes()
+}

core/utility/html_test.go

@@ -0,0 +1,83 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under 
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>. 
+//
+// https://documize.com
+
+package utility
+
+import "testing"
+
+func TestHTML(t *testing.T) {
+	type testConv struct {
+		htm, txt string
+		istest   bool
+	}
+	convTest := []testConv{
+		{
+			`<html><head><title>HTML TITLE</title></head><body><p>This <I>is</I>:</p><ul><li><a href="foo">Example</a><li><a href="/bar/baz">HTML text.</a><div class="documize-math">exclueded</div></ul></body></html>`,
+			"This is : Example HTML text. ", false,
+		},
+		{
+			`<p>This is:</p><ul><li><documize type="field-start"></documize> <documize type="field-end"></documize><documize type="unknown"></documize><li><a href="/bar/baz">HTML text.</a></ul>`,
+			"This is: [ ] [ ] HTML text. ", true,
+		},
+	}
+	for _, tst := range convTest {
+		var ch HTML
+		ch = HTML([]byte(tst.htm))
+		//t.Logf("HTML: %s", ch)
+		txt, err := ch.Text(tst.istest)
+		if err != nil {
+			t.Log(err)
+			t.Fail()
+		}
+		expected := compressSpaces(tst.txt)
+		got := compressSpaces(string(txt))
+		if expected != got {
+			t.Errorf("Conversion to text for `%s`, expected: `%s` got: `%s`\n",
+				ch, expected, got)
+		} //else {
+		//	t.Logf("Text: %s", txt)
+		//}
+	}
+}
+
+func compressSpaces(s string) string {
+	ret := ""
+	inSpace := false
+	for _, r := range s {
+		switch r {
+		case ' ', '\t', '\n', '\u200b' /*zero width space*/ :
+			if !inSpace {
+				ret += " "
+			}
+			inSpace = true
+		default:
+			inSpace = false
+			ret += string(r)
+		}
+	}
+	return ret
+}
+
+func TestHTMLescape(t *testing.T) {
+	tianchao := "兲朝 test"
+	expected := "&#20850;&#26397; test"
+
+	gotString := EscapeHTMLcomplexChars(tianchao)
+	if gotString != expected {
+		t.Errorf("EscapeHTMLcomplexChars error got `%s` expected `%s`\n", gotString, expected)
+	}
+
+	gotBytes := EscapeHTMLcomplexCharsByte([]byte(tianchao))
+	if string(gotBytes) != expected {
+		t.Errorf("EscapeHTMLcomplexCharsByte error got `%s` expected `%s`\n", string(gotBytes), expected)
+	}
+
+}

core/utility/secrets.go

@@ -0,0 +1,88 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under 
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>. 
+//
+// https://documize.com
+
+package utility
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"encoding/base64"
+	"errors"
+	"io"
+)
+
+var key = []byte("8456FHkQW1566etydT46jk39ghjfFhg4") // 32 bytes
+
+// MakeMD5 returns the MD5 hash of a given string, usually a password.
+/*
+func MakeMD5(password string) []byte {
+	hash := md5.New()
+	if _, err := io.WriteString(hash, password); err != nil {
+		log.Error("error in MakeMD5", err)
+	}
+	return hash.Sum(nil)
+}
+*/
+
+// MakeAES creates an AES encryption of of a given string,
+// using a hard-wired key value,
+// suitable for use as an authentication token.
+func MakeAES(secret string) ([]byte, error) {
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+	b := EncodeBase64([]byte(secret))
+	ciphertext := make([]byte, aes.BlockSize+len(b))
+	iv := ciphertext[:aes.BlockSize]
+	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
+		return nil, err
+	}
+	cfb := cipher.NewCFBEncrypter(block, iv)
+	cfb.XORKeyStream(ciphertext[aes.BlockSize:], b)
+	return ciphertext, nil
+}
+
+// DecryptAES decrypts an AES encoded []byte,
+// using a hard-wired key value,
+// suitable for use when reading an authentication token.
+func DecryptAES(text []byte) ([]byte, error) {
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, errors.New("aes.NewCipher failure: " + err.Error())
+	}
+	if len(text) < aes.BlockSize {
+		return nil, errors.New("ciphertext too short")
+	}
+	iv := text[:aes.BlockSize]
+	text = text[aes.BlockSize:]
+	cfb := cipher.NewCFBDecrypter(block, iv)
+	cfb.XORKeyStream(text, text)
+	return DecodeBase64(text)
+}
+
+// EncodeBase64 is a convenience function to encode using StdEncoding.
+func EncodeBase64(b []byte) []byte {
+	return []byte(base64.StdEncoding.EncodeToString(b))
+}
+
+// EncodeBase64AsString is a convenience function to encode using StdEncoding.
+/*
+func EncodeBase64AsString(b []byte) string {
+	return base64.StdEncoding.EncodeToString(b)
+}
+*/
+
+// DecodeBase64 is a convenience function to decode using StdEncoding.
+func DecodeBase64(b []byte) ([]byte, error) {
+	return base64.StdEncoding.DecodeString(string(b))
+}

core/utility/secrets_test.go

@@ -0,0 +1,35 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under 
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>. 
+//
+// https://documize.com
+
+package utility
+
+import "testing"
+
+func TestSecrets(t *testing.T) {
+	mimi := "007"
+	b, e := MakeAES(mimi)
+	if e != nil {
+		t.Fatal(e)
+	}
+	mm, e2 := DecryptAES(b)
+	if e2 != nil {
+		t.Fatal(e2)
+	}
+	if mimi != string(mm) {
+		t.Errorf("wanted %s got %s", mimi, string(mm))
+	}
+	
+	_, ee := DecryptAES([]byte{})
+	if ee == nil {
+		t.Error("should have errored on empty cypher")
+	}
+	
+}

core/utility/slug.go

@@ -0,0 +1,37 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under 
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>. 
+//
+// https://documize.com
+
+package utility
+
+import (
+	"strings"
+	"unicode"
+)
+
+// MakeSlug creates a slug, suitable for use in a URL, from a string
+func MakeSlug(str string) string {
+	slg := strings.Map(
+		func(r rune) rune { // individual mapping of runes into a format suitable for use in a URL
+			r = unicode.ToLower(r)
+			if unicode.IsLower(r) || unicode.IsDigit(r) {
+				return r
+			}
+			return '-'
+		}, str)
+	slg = strings.NewReplacer("---", "-", "--", "-").Replace(slg)
+	for strings.HasSuffix(slg, "-") {
+		slg = strings.TrimSuffix(slg, "-")
+	}
+	for strings.HasPrefix(slg, "-") {
+		slg = strings.TrimPrefix(slg, "-")
+	}
+	return slg
+}

core/utility/slug_test.go

@@ -0,0 +1,25 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under 
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>. 
+//
+// https://documize.com
+
+package utility
+
+import "testing"
+
+func TestSlug(t *testing.T) {
+	st(t, " Zip--up ", "zip-up")
+}
+
+func st(t *testing.T, in, out string) {
+	got := MakeSlug(in)
+	if got != out {
+		t.Errorf("slug input `%s` got `%s` expected `%s`\n", in, got, out)
+	}
+}

core/utility/user.go

@@ -0,0 +1,34 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under 
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>. 
+//
+// https://documize.com
+
+package utility
+
+import (
+	"strings"
+)
+
+// MakeInitials returns user initials from firstname and lastname.
+func MakeInitials(firstname, lastname string) string {
+	firstname = strings.TrimSpace(firstname)
+	lastname = strings.TrimSpace(lastname)
+	a := ""
+	b := ""
+
+	if len(firstname) > 0 {
+		a = firstname[:1]
+	}
+
+	if len(lastname) > 0 {
+		b = lastname[:1]
+	}
+
+	return strings.ToUpper(a + b)
+}

core/utility/user_test.go

@@ -0,0 +1,28 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under 
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>. 
+//
+// https://documize.com
+
+package utility
+
+import "testing"
+
+func TestInitials(t *testing.T) {
+	in(t, "Harvey", "Kandola", "HK")
+	in(t, "Harvey", "", "H")
+	in(t, "", "Kandola", "K")
+	in(t, "", "", "")
+}
+
+func in(t *testing.T, firstname, lastname, expecting string) {
+	initials := MakeInitials(firstname, lastname)
+	if initials != expecting {
+		t.Errorf("expecting initials of `%s` got `%s`\n", expecting, initials)
+	}
+}

core/utility/words.go

@@ -0,0 +1,75 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under 
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>. 
+//
+// https://documize.com
+
+package utility
+
+import "unicode"
+import nethtml "golang.org/x/net/html"
+
+// Words returns a slice of words, where each word contains no whitespace, and each item of punctuation is its own word.
+// This functionality is provided to enable verification of the text extraction algorithem across different implemntations.
+func Words(ch HTML, inSqBr int, testMode bool) ([]string, int, error) {
+	txt, err := ch.Text(testMode)
+	if err != nil {
+		return nil, inSqBr, err
+	}
+	txt = nethtml.UnescapeString(txt)
+
+	words := []string{""}
+
+	for _, c := range txt {
+		if inSqBr > 0 {
+			switch c {
+			case ']':
+				inSqBr--
+			case '[':
+				inSqBr++
+			}
+		} else {
+			if c == rune(0x200B) { // http://en.wikipedia.org/wiki/Zero-width_space
+				if testMode {
+					c = ' ' // NOTE only replace with a space here if we are testing
+				}
+			}
+			if c != rune(0x200B) { // http://en.wikipedia.org/wiki/Zero-width_space
+				if c == '[' {
+					inSqBr = 1
+					words = append(words, "[") // open square bracket means potentially elided text
+					words = append(words, "")
+				} else {
+					inSqBr = 0
+					if unicode.IsPunct(c) || unicode.IsSymbol(c) || unicode.IsDigit(c) {
+						if words[len(words)-1] == "" {
+							words[len(words)-1] = string(c)
+						} else {
+							words = append(words, string(c))
+						}
+						words = append(words, "")
+					} else {
+						if unicode.IsGraphic(c) || unicode.IsSpace(c) {
+							if unicode.IsSpace(c) {
+								if words[len(words)-1] != "" {
+									words = append(words, "")
+								}
+							} else {
+								words[len(words)-1] += string(c)
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+	if !testMode { // add dummy punctuation if not in test mode to avoid incorrect sentance concatanation
+		words = append(words, ".")
+	}
+	return append(words, ""), inSqBr, nil // make sure there is always a blank entry at the end
+}

core/utility/words_test.go

@@ -0,0 +1,57 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under 
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>. 
+//
+// https://documize.com
+
+package utility
+
+import (
+	"sort"
+	"strings"
+	"testing"
+)
+
+func TestWords(t *testing.T) {
+	ws(t, " the quick brown fox jumps over the lazy dog [ ] [" +string(rune(0x200B)), 0, true,
+		"the quick brown fox jumps over the lazy dog [ [", 1)
+	ws(t, "the quick brown [ dog jumps over the lazy ] fox", 0, false,
+		"the quick brown [ fox .", 0)
+	ws(t, "the quick brown;fox;", 0, false,
+		"the quick brown ; fox ; .", 0)
+	ws(t, "the ] quick brown fox ", 1, true,
+		"quick brown fox", 0)
+}
+
+func ws(t *testing.T, in string, bktIn int, isTest bool, out string, bktOut int) {
+	wds := strings.Split(out, " ")
+	gotX, bo, e := Words(HTML(in), bktIn, isTest)
+	if e != nil {
+		t.Fatal(e)
+	}
+	if bo != bktOut {
+		t.Errorf("wrong bracket count returned: input `%s` bktIn %d bktOut %d\n", in, bktIn, bktOut)
+	}
+	got := make([]string, 0, len(gotX))
+	for _, v := range gotX { // remove empty entries
+		if v != "" {
+			got = append(got, v)
+		}
+	}
+	if len(got) != len(wds) {
+		t.Errorf("wrong number of words found: input `%s` got %d %v expected %d %v`\n", in, len(got), got, len(wds), wds)
+	} else {
+		sort.Strings(wds)
+		sort.Strings(got)
+		for i := range wds {
+			if wds[i] != got[i] {
+				t.Errorf("wrong word[%d]: input `%s` got %v expected %v\n", i, in, got, wds)
+			}
+		}
+	}
+}