refactored document & search codebase to new API

core/api/request/document.go

@@ -187,10 +187,10 @@ func (p *Persister) GetDocumentTemplates() (documents []entity.Document, err err
 func (p *Persister) GetPublicDocuments(orgID string) (documents []entity.SitemapDocument, err error) {
 	err = Db.Select(&documents,
 		`SELECT d.refid as documentid, d.title as document, d.revised as revised, l.refid as folderid, l.label as folder
-FROM document d LEFT JOIN label l ON l.refid=d.labelid
+		FROM document d LEFT JOIN label l ON l.refid=d.labelid
-WHERE d.orgid=?
+		WHERE d.orgid=?
-AND l.type=1
+		AND l.type=1
-AND d.template=0`, orgID)
+		AND d.template=0`, orgID)
 
 	if err != nil {
 		log.Error(fmt.Sprintf("Unable to execute GetPublicDocuments for org %s", orgID), err)

domain/activity/mysql/store.go

@@ -0,0 +1,76 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>.
+//
+// https://documize.com
+
+package mysql
+
+import (
+	"database/sql"
+	"time"
+
+	"github.com/documize/community/core/env"
+	"github.com/documize/community/core/streamutil"
+	"github.com/documize/community/domain"
+	"github.com/documize/community/model/activity"
+	"github.com/pkg/errors"
+)
+
+// Scope provides data access to MySQL.
+type Scope struct {
+	Runtime *env.Runtime
+}
+
+// RecordUserActivity logs user initiated data changes.
+func (s Scope) RecordUserActivity(ctx domain.RequestContext, activity activity.UserActivity) (err error) {
+	activity.OrgID = ctx.OrgID
+	activity.UserID = ctx.UserID
+	activity.Created = time.Now().UTC()
+
+	stmt, err := ctx.Transaction.Preparex("INSERT INTO useractivity (orgid, userid, labelid, sourceid, sourcetype, activitytype, created) VALUES (?, ?, ?, ?, ?, ?, ?)")
+	defer streamutil.Close(stmt)
+
+	if err != nil {
+		err = errors.Wrap(err, "prepare record user activity")
+		return
+	}
+
+	_, err = stmt.Exec(activity.OrgID, activity.UserID, activity.LabelID, activity.SourceID, activity.SourceType, activity.ActivityType, activity.Created)
+
+	if err != nil {
+		err = errors.Wrap(err, "execute record user activity")
+		return
+	}
+
+	return
+}
+
+// GetDocumentActivity returns the metadata for a specified document.
+func (s Scope) GetDocumentActivity(ctx domain.RequestContext, id string) (a []activity.DocumentActivity, err error) {
+	qry := `SELECT a.id, a.created, a.orgid, IFNULL(a.userid, '') AS userid, a.labelid, a.sourceid as documentid, a.activitytype,
+		IFNULL(u.firstname, 'Anonymous') AS firstname, IFNULL(u.lastname, 'Viewer') AS lastname
+		FROM useractivity a
+		LEFT JOIN user u ON a.userid=u.refid
+		WHERE a.orgid=? AND a.sourceid=? AND a.sourcetype=2
+		AND a.userid != '0' AND a.userid != ''
+		ORDER BY a.created DESC`
+
+	err = s.Runtime.Db.Select(&a, qry, ctx.OrgID, id)
+
+	if len(a) == 0 {
+		a = []activity.DocumentActivity{}
+	}
+
+	if err != nil && err != sql.ErrNoRows {
+		err = errors.Wrap(err, "select document user activity")
+		return
+	}
+
+	return
+}

domain/document/endpoint.go

@@ -0,0 +1,339 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>.
+//
+// https://documize.com
+
+package document
+
+import (
+	"database/sql"
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+
+	"github.com/documize/community/core/env"
+	"github.com/documize/community/core/request"
+	"github.com/documize/community/core/response"
+	"github.com/documize/community/core/streamutil"
+	"github.com/documize/community/core/stringutil"
+	"github.com/documize/community/domain"
+	"github.com/documize/community/domain/space"
+	"github.com/documize/community/model/activity"
+	"github.com/documize/community/model/audit"
+	"github.com/documize/community/model/doc"
+	"github.com/documize/community/model/link"
+	"github.com/documize/community/model/page"
+	"github.com/documize/community/model/search"
+)
+
+// Handler contains the runtime information such as logging and database.
+type Handler struct {
+	Runtime *env.Runtime
+	Store   *domain.Store
+}
+
+// SearchDocuments endpoint takes a list of keywords and returns a list of document references matching those keywords.
+func (h *Handler) SearchDocuments(w http.ResponseWriter, r *http.Request) {
+	method := "document.search"
+	ctx := domain.GetRequestContext(r)
+
+	keywords := request.Query(r, "keywords")
+	decoded, err := url.QueryUnescape(keywords)
+	if err != nil {
+		response.WriteBadRequestError(w, method, err.Error())
+		return
+	}
+
+	results, err := h.Store.Search.Documents(ctx, decoded)
+	if err != nil {
+		h.Runtime.Log.Error("search failed", err)
+	}
+
+	// Put in slugs for easy UI display of search URL
+	for key, result := range results {
+		result.DocumentSlug = stringutil.MakeSlug(result.DocumentTitle)
+		result.FolderSlug = stringutil.MakeSlug(result.LabelName)
+		results[key] = result
+	}
+
+	if len(results) == 0 {
+		results = []search.DocumentSearch{}
+	}
+
+	h.Store.Audit.Record(ctx, audit.EventTypeSearch)
+
+	response.WriteJSON(w, results)
+}
+
+// Get is an endpoint that returns the document-level information for a given documentID.
+func (h *Handler) Get(w http.ResponseWriter, r *http.Request) {
+	method := "document.get"
+	ctx := domain.GetRequestContext(r)
+
+	id := request.Param(r, "documentID")
+	if len(id) == 0 {
+		response.WriteMissingDataError(w, method, "documentID")
+		return
+	}
+
+	document, err := h.Store.Document.Get(ctx, id)
+	if err == sql.ErrNoRows {
+		response.WriteNotFoundError(w, method, id)
+		return
+	}
+	if err != nil {
+		response.WriteServerError(w, method, err)
+		return
+	}
+
+	if CanViewDocumentInFolder(ctx, *h.Store, document.LabelID) {
+		response.WriteForbiddenError(w)
+		return
+	}
+
+	ctx.Transaction, err = h.Runtime.Db.Beginx()
+	if err != nil {
+		response.WriteServerError(w, method, err)
+		return
+	}
+
+	h.Store.Activity.RecordUserActivity(ctx, activity.UserActivity{
+		LabelID:      document.LabelID,
+		SourceID:     document.RefID,
+		SourceType:   activity.SourceTypeDocument,
+		ActivityType: activity.TypeRead})
+
+	h.Store.Audit.Record(ctx, audit.EventTypeDocumentView)
+
+	ctx.Transaction.Commit()
+
+	response.WriteJSON(w, document)
+}
+
+// Activity is an endpoint returning the activity logs for specified document.
+func (h *Handler) Activity(w http.ResponseWriter, r *http.Request) {
+	method := "document.activity"
+	ctx := domain.GetRequestContext(r)
+
+	id := request.Param(r, "documentID")
+	if len(id) == 0 {
+		response.WriteMissingDataError(w, method, "documentID")
+		return
+	}
+
+	a, err := h.Store.Activity.GetDocumentActivity(ctx, id)
+	if err != nil && err != sql.ErrNoRows {
+		response.WriteServerError(w, method, err)
+		return
+	}
+
+	response.WriteJSON(w, a)
+}
+
+// DocumentLinks is an endpoint returning the links for a document.
+func (h *Handler) DocumentLinks(w http.ResponseWriter, r *http.Request) {
+	method := "document.links"
+	ctx := domain.GetRequestContext(r)
+
+	id := request.Param(r, "documentID")
+	if len(id) == 0 {
+		response.WriteMissingDataError(w, method, "documentID")
+		return
+	}
+
+	l, err := h.Store.Link.GetDocumentOutboundLinks(ctx, id)
+
+	if len(l) == 0 {
+		l = []link.Link{}
+	}
+
+	if err != nil && err != sql.ErrNoRows {
+		response.WriteServerError(w, method, err)
+		return
+	}
+
+	response.WriteJSON(w, l)
+}
+
+// BySpace is an endpoint that returns the documents in a given folder.
+func (h *Handler) BySpace(w http.ResponseWriter, r *http.Request) {
+	method := "document.space"
+	ctx := domain.GetRequestContext(r)
+
+	folderID := request.Query(r, "folder")
+
+	if len(folderID) == 0 {
+		response.WriteMissingDataError(w, method, "folder")
+		return
+	}
+
+	if !space.CanViewSpace(ctx, *h.Store, folderID) {
+		response.WriteForbiddenError(w)
+		return
+	}
+
+	documents, err := h.Store.Document.GetBySpace(ctx, folderID)
+	if err != nil && err != sql.ErrNoRows {
+		response.WriteServerError(w, method, err)
+		return
+	}
+
+	if len(documents) == 0 {
+		documents = []doc.Document{}
+	}
+
+	response.WriteJSON(w, documents)
+}
+
+// ByTag is an endpoint that returns the documents with a given tag.
+func (h *Handler) ByTag(w http.ResponseWriter, r *http.Request) {
+	method := "document.space"
+	ctx := domain.GetRequestContext(r)
+
+	tag := request.Query(r, "tag")
+	if len(tag) == 0 {
+		response.WriteMissingDataError(w, method, "tag")
+		return
+	}
+
+	documents, err := h.Store.Document.GetByTag(ctx, tag)
+	if err != nil && err != sql.ErrNoRows {
+		response.WriteServerError(w, method, err)
+		return
+	}
+
+	if len(documents) == 0 {
+		documents = []doc.Document{}
+	}
+
+	response.WriteJSON(w, documents)
+}
+
+// Update updates an existing document using the
+// format described in NewDocumentModel() encoded as JSON in the request.
+func (h *Handler) Update(w http.ResponseWriter, r *http.Request) {
+	method := "document.space"
+	ctx := domain.GetRequestContext(r)
+
+	documentID := request.Param(r, "documentID")
+	if len(documentID) == 0 {
+		response.WriteMissingDataError(w, method, "documentID")
+		return
+	}
+
+	if !ctx.Editor {
+		response.WriteForbiddenError(w)
+		return
+	}
+
+	if !CanChangeDocument(ctx, *h.Store, documentID) {
+		response.WriteForbiddenError(w)
+		return
+	}
+
+	defer streamutil.Close(r.Body)
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		response.WriteBadRequestError(w, method, err.Error())
+		return
+	}
+
+	d := doc.Document{}
+	err = json.Unmarshal(body, &d)
+	if err != nil {
+		response.WriteBadRequestError(w, method, err.Error())
+		return
+	}
+
+	d.RefID = documentID
+
+	ctx.Transaction, err = h.Runtime.Db.Beginx()
+	if err != nil {
+		response.WriteServerError(w, method, err)
+		return
+	}
+
+	err = h.Store.Document.Update(ctx, d)
+	if err != nil {
+		ctx.Transaction.Rollback()
+		response.WriteServerError(w, method, err)
+		return
+	}
+
+	h.Store.Audit.Record(ctx, audit.EventTypeDocumentUpdate)
+
+	p := page.Page{DocumentID: documentID, OrgID: ctx.OrgID, Body: d.Slug, Title: d.Title}
+	h.Store.Search.UpdateDocument(ctx, p)
+
+	ctx.Transaction.Commit()
+
+	response.WriteEmpty(w)
+}
+
+// Delete is an endpoint that deletes a document specified by documentID.
+func (h *Handler) Delete(w http.ResponseWriter, r *http.Request) {
+	method := "document.delete"
+	ctx := domain.GetRequestContext(r)
+
+	documentID := request.Param(r, "documentID")
+	if len(documentID) == 0 {
+		response.WriteMissingDataError(w, method, "documentID")
+		return
+	}
+
+	if !CanChangeDocument(ctx, *h.Store, documentID) {
+		response.WriteForbiddenError(w)
+		return
+	}
+
+	doc, err := h.Store.Document.Get(ctx, documentID)
+	if err != nil {
+		response.WriteServerError(w, method, err)
+		return
+	}
+
+	ctx.Transaction, err = h.Runtime.Db.Beginx()
+	if err != nil {
+		response.WriteServerError(w, method, err)
+		return
+	}
+
+	_, err = h.Store.Document.Delete(ctx, documentID)
+	if err != nil {
+		ctx.Transaction.Rollback()
+		response.WriteServerError(w, method, err)
+		return
+	}
+
+	_, err = h.Store.Pin.DeletePinnedDocument(ctx, documentID)
+	if err != nil && err != sql.ErrNoRows {
+		ctx.Transaction.Rollback()
+		response.WriteServerError(w, method, err)
+		return
+	}
+
+	h.Store.Link.MarkOrphanDocumentLink(ctx, documentID)
+	h.Store.Link.DeleteSourceDocumentLinks(ctx, documentID)
+
+	h.Store.Activity.RecordUserActivity(ctx, activity.UserActivity{
+		LabelID:      doc.LabelID,
+		SourceID:     documentID,
+		SourceType:   activity.SourceTypeDocument,
+		ActivityType: activity.TypeDeleted})
+
+	h.Store.Audit.Record(ctx, audit.EventTypeDocumentDelete)
+
+	p := page.Page{DocumentID: documentID, OrgID: ctx.OrgID}
+	h.Store.Search.DeleteDocument(ctx, p)
+
+	ctx.Transaction.Commit()
+
+	response.WriteEmpty(w)
+}

domain/document/mysql/store.go

@@ -13,10 +13,12 @@ package document
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/documize/community/core/env"
 	"github.com/documize/community/core/streamutil"
 	"github.com/documize/community/domain"
+	"github.com/documize/community/domain/store/mysql"
 	"github.com/documize/community/model/doc"
 	"github.com/pkg/errors"
 )
@@ -26,6 +28,30 @@ type Scope struct {
 	Runtime *env.Runtime
 }
 
+// Add inserts the given document record into the document table and audits that it has been done.
+func (s Scope) Add(ctx domain.RequestContext, document doc.Document) (err error) {
+	document.OrgID = ctx.OrgID
+	document.Created = time.Now().UTC()
+	document.Revised = document.Created // put same time in both fields
+
+	stmt, err := ctx.Transaction.Preparex("INSERT INTO document (refid, orgid, labelid, userid, job, location, title, excerpt, slug, tags, template, created, revised) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
+	defer streamutil.Close(stmt)
+
+	if err != nil {
+		err = errors.Wrap(err, "prepare insert document")
+		return
+	}
+
+	_, err = stmt.Exec(document.RefID, document.OrgID, document.LabelID, document.UserID, document.Job, document.Location, document.Title, document.Excerpt, document.Slug, document.Tags, document.Template, document.Created, document.Revised)
+
+	if err != nil {
+		err = errors.Wrap(err, "execuet insert document")
+		return
+	}
+
+	return
+}
+
 // Get fetches the document record with the given id fromt the document table and audits that it has been got.
 func (s Scope) Get(ctx domain.RequestContext, id string) (document doc.Document, err error) {
 	stmt, err := s.Runtime.Db.Preparex("SELECT id, refid, orgid, labelid, userid, job, location, title, excerpt, slug, tags, template, layout, created, revised FROM document WHERE orgid=? and refid=?")
@@ -45,7 +71,203 @@ func (s Scope) Get(ctx domain.RequestContext, id string) (document doc.Document,
 	return
 }
 
-// MoveDocumentSpace changes the label for client's organization's documents which have space "id", to "move".
+// DocumentMeta returns the metadata for a specified document.
+func (s Scope) DocumentMeta(ctx domain.RequestContext, id string) (meta doc.DocumentMeta, err error) {
+	sqlViewers := `SELECT MAX(a.created) as created,
+		IFNULL(a.userid, '') AS userid, IFNULL(u.firstname, 'Anonymous') AS firstname, IFNULL(u.lastname, 'Viewer') AS lastname
+		FROM audit a LEFT JOIN user u ON a.userid=u.refid
+		WHERE a.orgid=? AND a.documentid=?
+		AND a.userid != '0' AND a.userid != ''
+		AND action='get-document'
+		GROUP BY a.userid ORDER BY MAX(a.created) DESC`
+
+	err = s.Runtime.Db.Select(&meta.Viewers, sqlViewers, ctx.OrgID, id)
+
+	if err != nil {
+		err = errors.Wrap(err, fmt.Sprintf("select document viewers %s", id))
+		return
+	}
+
+	sqlEdits := `SELECT a.created,
+		IFNULL(a.action, '') AS action, IFNULL(a.userid, '') AS userid, IFNULL(u.firstname, 'Anonymous') AS firstname, IFNULL(u.lastname, 'Viewer') AS lastname, IFNULL(a.pageid, '') AS pageid
+		FROM audit a LEFT JOIN user u ON a.userid=u.refid
+		WHERE a.orgid=? AND a.documentid=? AND a.userid != '0' AND a.userid != ''
+		AND (a.action='update-page' OR a.action='add-page' OR a.action='remove-page')
+		ORDER BY a.created DESC;`
+
+	err = s.Runtime.Db.Select(&meta.Editors, sqlEdits, ctx.OrgID, id)
+
+	if err != nil {
+		err = errors.Wrap(err, fmt.Sprintf("select document editors %s", id))
+		return
+	}
+
+	return
+}
+
+// GetAll returns a slice containg all of the the documents for the client's organisation, with the most recient first.
+func (s Scope) GetAll() (ctx domain.RequestContext, documents []doc.Document, err error) {
+	err = s.Runtime.Db.Select(&documents, "SELECT id, refid, orgid, labelid, userid, job, location, title, excerpt, slug, tags, template, layout, created, revised FROM document WHERE orgid=? AND template=0 ORDER BY revised DESC", ctx.OrgID)
+
+	if err != nil {
+		err = errors.Wrap(err, "select documents")
+		return
+	}
+
+	return
+}
+
+// GetBySpace returns a slice containing the documents for a given space, most recient first.
+func (s Scope) GetBySpace(ctx domain.RequestContext, folderID string) (documents []doc.Document, err error) {
+	err = s.Runtime.Db.Select(&documents, "SELECT id, refid, orgid, labelid, userid, job, location, title, excerpt, slug, tags, template, layout, created, revised FROM document WHERE orgid=? AND template=0 AND labelid=? ORDER BY revised DESC", ctx.OrgID, folderID)
+
+	if err != nil {
+		err = errors.Wrap(err, "select documents by space")
+		return
+	}
+
+	return
+}
+
+// GetByTag returns a slice containing the documents with the specified tag, in title order.
+func (s Scope) GetByTag(ctx domain.RequestContext, tag string) (documents []doc.Document, err error) {
+	tagQuery := "tags LIKE '%#" + tag + "#%'"
+
+	err = s.Runtime.Db.Select(&documents,
+		`SELECT id, refid, orgid, labelid, userid, job, location, title, excerpt, slug, tags, template, layout, created, revised FROM document WHERE orgid=? AND template=0 AND `+tagQuery+` AND labelid IN
+		(SELECT refid from label WHERE orgid=? AND type=2 AND userid=?
+    	UNION ALL SELECT refid FROM label a where orgid=? AND type=1 AND refid IN (SELECT labelid from labelrole WHERE orgid=? AND userid='' AND (canedit=1 OR canview=1))
+		UNION ALL SELECT refid FROM label a where orgid=? AND type=3 AND refid IN (SELECT labelid from labelrole WHERE orgid=? AND userid=? AND (canedit=1 OR canview=1)))
+		ORDER BY title`,
+		ctx.OrgID,
+		ctx.OrgID,
+		ctx.UserID,
+		ctx.OrgID,
+		ctx.OrgID,
+		ctx.OrgID,
+		ctx.OrgID,
+		ctx.UserID)
+
+	if err != nil {
+		err = errors.Wrap(err, "select documents by tag")
+		return
+	}
+
+	return
+}
+
+// Templates returns a slice containing the documents available as templates to the client's organisation, in title order.
+func (s Scope) Templates() (ctx domain.RequestContext, documents []doc.Document, err error) {
+	err = s.Runtime.Db.Select(&documents,
+		`SELECT id, refid, orgid, labelid, userid, job, location, title, excerpt, slug, tags, template, layout, created, revised FROM document WHERE orgid=? AND template=1 AND labelid IN
+		(SELECT refid from label WHERE orgid=? AND type=2 AND userid=?
+    	UNION ALL SELECT refid FROM label a where orgid=? AND type=1 AND refid IN (SELECT labelid from labelrole WHERE orgid=? AND userid='' AND (canedit=1 OR canview=1))
+		UNION ALL SELECT refid FROM label a where orgid=? AND type=3 AND refid IN (SELECT labelid from labelrole WHERE orgid=? AND userid=? AND (canedit=1 OR canview=1)))
+		ORDER BY title`,
+		ctx.OrgID,
+		ctx.OrgID,
+		ctx.UserID,
+		ctx.OrgID,
+		ctx.OrgID,
+		ctx.OrgID,
+		ctx.OrgID,
+		ctx.UserID)
+
+	if err != nil {
+		err = errors.Wrap(err, "select documents templates")
+		return
+	}
+
+	return
+}
+
+// PublicDocuments returns a slice of SitemapDocument records, holding documents in folders of type 1 (entity.TemplateTypePublic).
+func (s Scope) PublicDocuments(ctx domain.RequestContext, orgID string) (documents []doc.SitemapDocument, err error) {
+	err = s.Runtime.Db.Select(&documents,
+		`SELECT d.refid as documentid, d.title as document, d.revised as revised, l.refid as folderid, l.label as folder
+		FROM document d LEFT JOIN label l ON l.refid=d.labelid
+		WHERE d.orgid=?
+		AND l.type=1
+		AND d.template=0`, orgID)
+
+	if err != nil {
+		err = errors.Wrap(err, fmt.Sprintf("execute GetPublicDocuments for org %s%s", orgID))
+		return
+	}
+
+	return
+}
+
+// DocumentList returns a slice containing the documents available as templates to the client's organisation, in title order.
+func (s Scope) DocumentList(ctx domain.RequestContext) (documents []doc.Document, err error) {
+	err = s.Runtime.Db.Select(&documents,
+		`SELECT id, refid, orgid, labelid, userid, job, location, title, excerpt, slug, tags, template, layout, created, revised FROM document WHERE orgid=? AND template=0 AND labelid IN
+		(SELECT refid from label WHERE orgid=? AND type=2 AND userid=?
+    	UNION ALL SELECT refid FROM label a where orgid=? AND type=1 AND refid IN (SELECT labelid from labelrole WHERE orgid=? AND userid='' AND (canedit=1 OR canview=1))
+		UNION ALL SELECT refid FROM label a where orgid=? AND type=3 AND refid IN (SELECT labelid from labelrole WHERE orgid=? AND userid=? AND (canedit=1 OR canview=1)))
+		ORDER BY title`,
+		ctx.OrgID,
+		ctx.OrgID,
+		ctx.UserID,
+		ctx.OrgID,
+		ctx.OrgID,
+		ctx.OrgID,
+		ctx.OrgID,
+		ctx.UserID)
+
+	if err != nil {
+		err = errors.Wrap(err, "select documents list")
+		return
+	}
+
+	return
+}
+
+// Update changes the given document record to the new values, updates search information and audits the action.
+func (s Scope) Update(ctx domain.RequestContext, document doc.Document) (err error) {
+	document.Revised = time.Now().UTC()
+
+	stmt, err := ctx.Transaction.PrepareNamed("UPDATE document SET labelid=:labelid, userid=:userid, job=:job, location=:location, title=:title, excerpt=:excerpt, slug=:slug, tags=:tags, template=:template, layout=:layout, revised=:revised WHERE orgid=:orgid AND refid=:refid")
+	defer streamutil.Close(stmt)
+
+	if err != nil {
+		err = errors.Wrap(err, "prepare update document")
+		return
+	}
+
+	_, err = stmt.Exec(&document)
+
+	if err != nil {
+		err = errors.Wrap(err, "execute update document")
+		return
+	}
+
+	return
+}
+
+// ChangeDocumentSpace assigns the specified space to the document.
+func (s Scope) ChangeDocumentSpace(ctx domain.RequestContext, document, space string) (err error) {
+	revised := time.Now().UTC()
+
+	stmt, err := ctx.Transaction.Preparex("UPDATE document SET labelid=?, revised=? WHERE orgid=? AND refid=?")
+	defer streamutil.Close(stmt)
+
+	if err != nil {
+		err = errors.Wrap(err, fmt.Sprintf("prepare change document space %s", document))
+		return
+	}
+
+	_, err = stmt.Exec(space, revised, ctx.OrgID, document)
+
+	if err != nil {
+		err = errors.Wrap(err, fmt.Sprintf("execute change document space %s", document))
+		return
+	}
+
+	return
+}
+
+// MoveDocumentSpace changes the space for client's organization's documents which have space "id", to "move".
 func (s Scope) MoveDocumentSpace(ctx domain.RequestContext, id, move string) (err error) {
 	stmt, err := ctx.Transaction.Preparex("UPDATE document SET labelid=? WHERE orgid=? AND labelid=?")
 	defer streamutil.Close(stmt)
@@ -64,19 +286,25 @@ func (s Scope) MoveDocumentSpace(ctx domain.RequestContext, id, move string) (er
 	return
 }
 
-// PublicDocuments returns a slice of SitemapDocument records, holding documents in folders of type 1 (entity.TemplateTypePublic).
+// Delete delete the document pages in the database, updates the search subsystem, deletes the associated revisions and attachments,
-func (s Scope) PublicDocuments(ctx domain.RequestContext, orgID string) (documents []doc.SitemapDocument, err error) {
+// audits the deletion, then finally deletes the document itself.
-	err = s.Runtime.Db.Select(&documents,
+func (s Scope) Delete(ctx domain.RequestContext, documentID string) (rows int64, err error) {
-		`SELECT d.refid as documentid, d.title as document, d.revised as revised, l.refid as folderid, l.label as folder
+	b := mysql.BaseQuery{}
-FROM document d LEFT JOIN label l ON l.refid=d.labelid
+	rows, err = b.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE from page WHERE documentid=\"%s\" AND orgid=\"%s\"", documentID, ctx.OrgID))
-WHERE d.orgid=?
-AND l.type=1
-AND d.template=0`, orgID)
 
 	if err != nil {
-		err = errors.Wrap(err, fmt.Sprintf("execute GetPublicDocuments for org %s%s", orgID))
 		return
 	}
 
-	return
+	_, err = b.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE from revision WHERE documentid=\"%s\" AND orgid=\"%s\"", documentID, ctx.OrgID))
+	if err != nil {
+		return
+	}
+
+	_, err = b.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE from attachment WHERE documentid=\"%s\" AND orgid=\"%s\"", documentID, ctx.OrgID))
+	if err != nil {
+		return
+	}
+
+	return b.DeleteConstrained(ctx.Transaction, "document", ctx.OrgID, documentID)
 }

domain/search/mysql/store.go

@@ -0,0 +1,315 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>.
+//
+// https://documize.com
+
+package mysql
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/documize/community/core/env"
+	"github.com/documize/community/core/streamutil"
+	"github.com/documize/community/core/stringutil"
+	"github.com/documize/community/domain"
+	"github.com/documize/community/domain/store/mysql"
+	"github.com/documize/community/model"
+	"github.com/documize/community/model/page"
+	"github.com/documize/community/model/search"
+	"github.com/jmoiron/sqlx"
+	"github.com/pkg/errors"
+)
+
+// Scope provides data access to MySQL.
+type Scope struct {
+	Runtime *env.Runtime
+}
+
+// Add search entry  (legacy name: searchAdd).
+func (s Scope) Add(ctx domain.RequestContext, page page.Page) (err error) {
+	id := page.RefID
+
+	// translate the html into text for the search
+	nonHTML, err := stringutil.HTML(page.Body).Text(false)
+	if err != nil {
+		errors.Wrap(err, "search decode body")
+		return
+	}
+
+	// insert into the search table, getting the document title along the way
+	var stmt *sqlx.Stmt
+	stmt, err = ctx.Transaction.Preparex(
+		"INSERT INTO search (id, orgid, documentid, level, sequence, documenttitle, slug, pagetitle, body, created, revised) " +
+			" SELECT page.refid,page.orgid,document.refid,page.level,page.sequence,document.title,document.slug,page.title,?,page.created,page.revised " +
+			" FROM document,page WHERE page.refid=? AND document.refid=page.documentid")
+
+	defer streamutil.Close(stmt)
+
+	if err != nil {
+		err = errors.Wrap(err, "prepare search insert")
+		return
+	}
+
+	_, err = stmt.Exec(nonHTML, id)
+
+	if err != nil {
+		err = errors.Wrap(err, "execute search insert")
+		return
+	}
+
+	return nil
+}
+
+// Update search entry (legacy name: searchUpdate).
+func (s Scope) Update(ctx domain.RequestContext, page page.Page) (err error) {
+	// translate the html into text for the search
+	nonHTML, err := stringutil.HTML(page.Body).Text(false)
+	if err != nil {
+		err = errors.Wrap(err, "search decode body")
+		return
+	}
+
+	su, err := ctx.Transaction.Preparex("UPDATE search SET pagetitle=?,body=?,sequence=?,level=?,revised=? WHERE id=?")
+	defer streamutil.Close(su)
+
+	if err != nil {
+		err = errors.Wrap(err, "prepare search update")
+		return err
+	}
+
+	_, err = su.Exec(page.Title, nonHTML, page.Sequence, page.Level, page.Revised, page.RefID)
+
+	if err != nil {
+		err = errors.Wrap(err, "execute search update")
+		return
+	}
+
+	return nil
+}
+
+// UpdateDocument search entries for document (legacy name: searchUpdateDocument).
+func (s Scope) UpdateDocument(ctx domain.RequestContext, page page.Page) (err error) {
+	stmt, err := ctx.Transaction.Preparex("UPDATE search SET documenttitle=?, slug=?, revised=? WHERE documentid=?")
+	defer streamutil.Close(stmt)
+
+	if err != nil {
+		err = errors.Wrap(err, "prepare search document update")
+		return err
+	}
+
+	_, err = stmt.Exec(page.Title, page.Body, time.Now().UTC(), page.DocumentID)
+
+	if err != nil {
+		err = errors.Wrap(err, "execute search document update")
+		return err
+	}
+
+	return nil
+}
+
+// DeleteDocument removes document search entries (legacy name: searchDeleteDocument)
+func (s Scope) DeleteDocument(ctx domain.RequestContext, page page.Page) (err error) {
+	var bm = mysql.BaseQuery{}
+
+	_, err = bm.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE from search WHERE documentid='%s'", page.DocumentID))
+
+	if err != nil {
+		err = errors.Wrap(err, "delete document search entries")
+	}
+
+	return nil
+}
+
+// Rebuild ... (legacy name: searchRebuild)
+func (s Scope) Rebuild(ctx domain.RequestContext, p page.Page) (err error) {
+	var bm = mysql.BaseQuery{}
+
+	_, err = bm.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE from search WHERE documentid='%s'", p.DocumentID))
+	if err != nil {
+		err = errors.Wrap(err, err.Error())
+		return err
+	}
+
+	var pages []struct{ ID string }
+
+	stmt2, err := ctx.Transaction.Preparex("SELECT refid as id FROM page WHERE documentid=? ")
+	defer streamutil.Close(stmt2)
+
+	if err != nil {
+		err = errors.Wrap(err, err.Error())
+		return err
+	}
+
+	err = stmt2.Select(&pages, p.DocumentID)
+	if err != nil {
+		err = errors.Wrap(err, err.Error())
+		return err
+	}
+
+	if len(pages) > 0 {
+		for _, pg := range pages {
+			err = s.Add(ctx, page.Page{BaseEntity: model.BaseEntity{RefID: pg.ID}})
+			if err != nil {
+				err = errors.Wrap(err, err.Error())
+				return err
+			}
+		}
+
+		// rebuild doc-level tags & excerpts
+		// get the 0'th page data and rewrite it
+
+		target := page.Page{}
+
+		stmt1, err := ctx.Transaction.Preparex("SELECT * FROM page WHERE refid=?")
+		defer streamutil.Close(stmt1)
+
+		if err != nil {
+			err = errors.Wrap(err, err.Error())
+			return err
+		}
+
+		err = stmt1.Get(&target, pages[0].ID)
+		if err != nil {
+			err = errors.Wrap(err, err.Error())
+			return err
+		}
+
+		err = s.Update(ctx, target) // to rebuild the document-level tags + excerpt
+		if err != nil {
+			err = errors.Wrap(err, err.Error())
+			return err
+		}
+	}
+
+	return
+}
+
+// UpdateSequence ... (legacy name: searchUpdateSequence)
+func (s Scope) UpdateSequence(ctx domain.RequestContext, page page.Page) (err error) {
+	supdate, err := ctx.Transaction.Preparex("UPDATE search SET sequence=?,revised=? WHERE id=?")
+	defer streamutil.Close(supdate)
+
+	if err != nil {
+		err = errors.Wrap(err, "prepare search update sequence")
+		return err
+	}
+
+	_, err = supdate.Exec(page.Sequence, time.Now().UTC(), page.RefID)
+	if err != nil {
+		err = errors.Wrap(err, "execute search update sequence")
+		return
+	}
+
+	return
+}
+
+// UpdateLevel ... legacy name: searchUpdateLevel)
+func (s Scope) UpdateLevel(ctx domain.RequestContext, page page.Page) (err error) {
+	pageID := page.RefID
+	level := page.Level
+
+	supdate, err := ctx.Transaction.Preparex("UPDATE search SET level=?,revised=? WHERE id=?")
+	defer streamutil.Close(supdate)
+
+	if err != nil {
+		err = errors.Wrap(err, "prepare search update level")
+		return err
+	}
+
+	_, err = supdate.Exec(level, time.Now().UTC(), pageID)
+	if err != nil {
+		err = errors.Wrap(err, "execute search update level")
+		return
+	}
+
+	return
+}
+
+// Delete ... (legacy name: searchDelete).
+func (s Scope) Delete(ctx domain.RequestContext, page page.Page) (err error) {
+	var bm = mysql.BaseQuery{}
+	_, err = bm.DeleteConstrainedWithID(ctx.Transaction, "search", ctx.OrgID, page.RefID)
+
+	return
+}
+
+// Documents searches the documents that the client is allowed to see, using the keywords search string, then audits that search.
+// Visible documents include both those in the client's own organisation and those that are public, or whose visibility includes the client.
+func (s Scope) Documents(ctx domain.RequestContext, keywords string) (results []search.DocumentSearch, err error) {
+	if len(keywords) == 0 {
+		return
+	}
+
+	var tagQuery, keywordQuery string
+
+	r, _ := regexp.Compile(`(#[a-z0-9][a-z0-9\-_]*)`)
+	res := r.FindAllString(keywords, -1)
+
+	if len(res) == 0 {
+		tagQuery = " "
+	} else {
+		if len(res) == 1 {
+			tagQuery = " AND document.tags LIKE '%" + res[0] + "#%' "
+		} else {
+			fmt.Println("lots of tags!")
+
+			tagQuery = " AND ("
+
+			for i := 0; i < len(res); i++ {
+				tagQuery += "document.tags LIKE '%" + res[i] + "#%'"
+				if i < len(res)-1 {
+					tagQuery += " OR "
+				}
+			}
+
+			tagQuery += ") "
+		}
+
+		keywords = r.ReplaceAllString(keywords, "")
+		keywords = strings.Replace(keywords, "  ", "", -1)
+	}
+
+	keywords = strings.TrimSpace(keywords)
+
+	if len(keywords) > 0 {
+		keywordQuery = "AND MATCH(pagetitle,body) AGAINST('" + keywords + "' in boolean mode)"
+	}
+
+	sql := `SELECT search.id, documentid, pagetitle, document.labelid, document.title as documenttitle, document.tags,
+   		COALESCE(label.label,'Unknown') AS labelname, document.excerpt as documentexcerpt
+   		FROM search, document LEFT JOIN label ON label.orgid=document.orgid AND label.refid = document.labelid
+		WHERE search.documentid = document.refid AND search.orgid=? AND document.template=0 ` + tagQuery +
+		`AND document.labelid IN
+		(SELECT refid from label WHERE orgid=? AND type=2 AND userid=?
+    	UNION ALL SELECT refid FROM label a where orgid=? AND type=1 AND refid IN (SELECT labelid from labelrole WHERE orgid=? AND userid='' AND (canedit=1 OR canview=1))
+		UNION ALL SELECT refid FROM label a where orgid=? AND type=3 AND refid IN (SELECT labelid from labelrole WHERE orgid=? AND userid=? AND (canedit=1 OR canview=1))) ` + keywordQuery
+	// AND MATCH(pagetitle,body)
+	//  		AGAINST('` + keywords + "' in boolean mode)"
+
+	err = s.Runtime.Db.Select(&results,
+		sql,
+		ctx.OrgID,
+		ctx.OrgID,
+		ctx.UserID,
+		ctx.OrgID,
+		ctx.OrgID,
+		ctx.OrgID,
+		ctx.OrgID,
+		ctx.UserID)
+
+	if err != nil {
+		err = errors.Wrap(err, "search documents")
+		return
+	}
+
+	return
+}

domain/search/search.go

@@ -0,0 +1,239 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>.
+//
+// https://documize.com
+
+package search
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+
+	"github.com/documize/community/core/env"
+	"github.com/documize/community/domain"
+	"github.com/documize/community/model"
+	"github.com/documize/community/model/doc"
+	"github.com/documize/community/model/page"
+)
+
+// Indexer type provides the datastructure for the queues of activity to be serialized through a single background goroutine.
+// NOTE if the queue becomes full, the system will trigger the rebuilding entire files in order to clear the backlog.
+type Indexer struct {
+	queue        chan queueEntry
+	rebuild      map[string]bool
+	rebuildLock  sync.RWMutex
+	givenWarning bool
+	runtime      *env.Runtime
+	store        *domain.Store
+}
+
+type queueEntry struct {
+	action    func(domain.RequestContext, page.Page) error
+	isRebuild bool
+	page.Page
+	ctx domain.RequestContext
+}
+
+var searches *Indexer
+
+const searchQueueLength = 2048 // NOTE the largest 15Mb docx in the test set generates 2142 queue entries, but the queue is constantly emptied
+
+// Start the background indexer
+func Start(rt *env.Runtime, s *domain.Store) {
+	searches = &Indexer{}
+	searches.queue = make(chan queueEntry, searchQueueLength) // provide some decoupling
+	searches.rebuild = make(map[string]bool)
+	searches.runtime = rt
+	searches.store = s
+
+	go searches.searchProcessQueue()
+}
+
+// searchProcessQueue is run as a goroutine, it processes the queue of search index update requests.
+func (m *Indexer) searchProcessQueue() {
+	for {
+		//fmt.Println("DEBUG queue length=", len(Searches.queue))
+		if len(m.queue) <= searchQueueLength/20 { // on a busy server, the queue may never get to zero - so use 5%
+			m.rebuildLock.Lock()
+			for docid := range m.rebuild {
+				m.queue <- queueEntry{
+					action:    m.store.Search.Rebuild,
+					isRebuild: true,
+					Page:      page.Page{DocumentID: docid},
+				}
+				delete(m.rebuild, docid)
+			}
+			m.rebuildLock.Unlock()
+		}
+
+		qe := <-m.queue
+		doit := true
+
+		if len(qe.DocumentID) > 0 {
+			m.rebuildLock.RLock()
+			if m.rebuild[qe.DocumentID] {
+				doit = false // don't execute an action on a document queued to be rebuilt
+			}
+			m.rebuildLock.RUnlock()
+		}
+
+		if doit {
+			tx, err := m.runtime.Db.Beginx()
+			if err != nil {
+			} else {
+				ctx := qe.ctx
+				ctx.Transaction = tx
+				err = qe.action(ctx, qe.Page)
+				if err != nil {
+					tx.Rollback()
+					// This action has failed, so re-build indexes for the entire document,
+					// provided it was not a re-build command that failed and we know the documentId.
+					if !qe.isRebuild && len(qe.DocumentID) > 0 {
+						m.rebuildLock.Lock()
+						m.rebuild[qe.DocumentID] = true
+						m.rebuildLock.Unlock()
+					}
+				} else {
+					tx.Commit()
+				}
+			}
+		}
+	}
+}
+
+func (m *Indexer) addQueue(qe queueEntry) error {
+	lsq := len(m.queue)
+
+	if lsq >= (searchQueueLength - 1) {
+		if qe.DocumentID != "" {
+			m.rebuildLock.Lock()
+			if !m.rebuild[qe.DocumentID] {
+				m.runtime.Log.Info(fmt.Sprintf("WARNING: Search Queue Has No Space! Marked rebuild index for document id %s", qe.DocumentID))
+			}
+			m.rebuild[qe.DocumentID] = true
+			m.rebuildLock.Unlock()
+		} else {
+			m.runtime.Log.Error("addQueue", errors.New("WARNING: Search Queue Has No Space! But unable to index unknown document id"))
+		}
+
+		return nil
+	}
+
+	if lsq > ((8 * searchQueueLength) / 10) {
+		if !m.givenWarning {
+			m.runtime.Log.Info(fmt.Sprintf("WARNING: Searches.queue length %d exceeds 80%% of capacity", lsq))
+			m.givenWarning = true
+		}
+	} else {
+		if m.givenWarning {
+			m.runtime.Log.Info(fmt.Sprintf("INFO: Searches.queue length %d now below 80%% of capacity", lsq))
+			m.givenWarning = false
+		}
+	}
+
+	m.queue <- qe
+
+	return nil
+}
+
+// Add should be called when a new page is added to a document.
+func (m *Indexer) Add(ctx domain.RequestContext, page page.Page, id string) (err error) {
+	page.RefID = id
+
+	err = m.addQueue(queueEntry{
+		action: m.store.Search.Add,
+		Page:   page,
+		ctx:    ctx,
+	})
+
+	return
+}
+
+// Update should be called after a page record has been updated.
+func (m *Indexer) Update(ctx domain.RequestContext, page page.Page) (err error) {
+	err = m.addQueue(queueEntry{
+		action: m.store.Search.Update,
+		Page:   page,
+		ctx:    ctx,
+	})
+
+	return
+}
+
+// UpdateDocument should be called after a document record has been updated.
+func (m *Indexer) UpdateDocument(ctx domain.RequestContext, document doc.Document) (err error) {
+	err = m.addQueue(queueEntry{
+		action: m.store.Search.UpdateDocument,
+		Page: page.Page{
+			DocumentID: document.RefID,
+			Title:      document.Title,
+			Body:       document.Slug, // NOTE body==slug in this context
+		},
+		ctx: ctx,
+	})
+
+	return
+}
+
+// DeleteDocument should be called after a document record has been deleted.
+func (m *Indexer) DeleteDocument(ctx domain.RequestContext, documentID string) (err error) {
+	if len(documentID) > 0 {
+		m.queue <- queueEntry{
+			action: m.store.Search.DeleteDocument,
+			Page:   page.Page{DocumentID: documentID},
+			ctx:    ctx,
+		}
+	}
+	return
+}
+
+// UpdateSequence should be called after a page record has been resequenced.
+func (m *Indexer) UpdateSequence(ctx domain.RequestContext, documentID, pageID string, sequence float64) (err error) {
+	err = m.addQueue(queueEntry{
+		action: m.store.Search.UpdateSequence,
+		Page: page.Page{
+			BaseEntity: model.BaseEntity{RefID: pageID},
+			Sequence:   sequence,
+			DocumentID: documentID,
+		},
+		ctx: ctx,
+	})
+
+	return
+}
+
+// UpdateLevel should be called after the level of a page has been changed.
+func (m *Indexer) UpdateLevel(ctx domain.RequestContext, documentID, pageID string, level int) (err error) {
+	err = m.addQueue(queueEntry{
+		action: m.store.Search.UpdateLevel,
+		Page: page.Page{
+			BaseEntity: model.BaseEntity{RefID: pageID},
+			Level:      uint64(level),
+			DocumentID: documentID,
+		},
+		ctx: ctx,
+	})
+
+	return
+}
+
+// Delete should be called after a page has been deleted.
+func (m *Indexer) Delete(ctx domain.RequestContext, documentID, pageID string) (rows int64, err error) {
+	err = m.addQueue(queueEntry{
+		action: m.store.Search.Delete,
+		Page: page.Page{
+			BaseEntity: model.BaseEntity{RefID: pageID},
+			DocumentID: documentID,
+		},
+		ctx: ctx,
+	})
+
+	return
+}

domain/space/permission.go

@@ -13,15 +13,19 @@
 // Spaces in Documize contain documents.
 package space
 
-/*
+import (
+	"database/sql"
+
+	"github.com/documize/community/domain"
+)
+
 // CanViewSpace returns if the user has permission to view the given spaceID.
-func CanViewSpace(s domain.StoreContext, spaceID string) (hasPermission bool) {
+func CanViewSpace(ctx domain.RequestContext, s domain.Store, spaceID string) (hasPermission bool) {
-	roles, err := GetRoles(s, spaceID)
+	roles, err := s.Space.GetRoles(ctx, spaceID)
 	if err == sql.ErrNoRows {
 		err = nil
 	}
 	if err != nil {
-		s.Runtime.Log.Error(fmt.Sprintf("check space permissions %s", spaceID), err)
 		return false
 	}
 
@@ -35,14 +39,13 @@ func CanViewSpace(s domain.StoreContext, spaceID string) (hasPermission bool) {
 }
 
 // CanViewSpaceDocuments returns if the user has permission to view a document within the specified space.
-func CanViewSpaceDocuments(s domain.StoreContext, spaceID string) (hasPermission bool) {
+func CanViewSpaceDocuments(ctx domain.RequestContext, s domain.Store, spaceID string) (hasPermission bool) {
-	roles, err := GetRoles(s, spaceID)
+	roles, err := s.Space.GetRoles(ctx, spaceID)
 	if err == sql.ErrNoRows {
 		err = nil
 	}
 
 	if err != nil {
-		s.Runtime.Log.Error(fmt.Sprintf("check space permissions %s", spaceID), err)
 		return false
 	}
 
@@ -54,4 +57,3 @@ func CanViewSpaceDocuments(s domain.StoreContext, spaceID string) (hasPermission
 
 	return false
 }
-*/

domain/storer.go

@@ -14,6 +14,7 @@ package domain
 
 import (
 	"github.com/documize/community/model/account"
+	"github.com/documize/community/model/activity"
 	"github.com/documize/community/model/attachment"
 	"github.com/documize/community/model/audit"
 	"github.com/documize/community/model/doc"
@@ -21,6 +22,7 @@ import (
 	"github.com/documize/community/model/org"
 	"github.com/documize/community/model/page"
 	"github.com/documize/community/model/pin"
+	"github.com/documize/community/model/search"
 	"github.com/documize/community/model/space"
 	"github.com/documize/community/model/user"
 )
@@ -38,6 +40,8 @@ type Store struct {
 	Attachment   AttachmentStorer
 	Link         LinkStorer
 	Page         PageStorer
+	Activity     ActivityStorer
+	Search       SearchStorer
 }
 
 // SpaceStorer defines required methods for space management
@@ -120,9 +124,19 @@ type AuditStorer interface {
 
 // DocumentStorer defines required methods for document handling
 type DocumentStorer interface {
+	Add(ctx RequestContext, document doc.Document) (err error)
 	Get(ctx RequestContext, id string) (document doc.Document, err error)
-	MoveDocumentSpace(ctx RequestContext, id, move string) (err error)
+	GetAll() (ctx RequestContext, documents []doc.Document, err error)
+	GetBySpace(ctx RequestContext, folderID string) (documents []doc.Document, err error)
+	GetByTag(ctx RequestContext, tag string) (documents []doc.Document, err error)
+	DocumentList(ctx RequestContext) (documents []doc.Document, err error)
+	Templates() (ctx RequestContext, documents []doc.Document, err error)
+	DocumentMeta(ctx RequestContext, id string) (meta doc.DocumentMeta, err error)
 	PublicDocuments(ctx RequestContext, orgID string) (documents []doc.SitemapDocument, err error)
+	Update(ctx RequestContext, document doc.Document) (err error)
+	ChangeDocumentSpace(ctx RequestContext, document, space string) (err error)
+	MoveDocumentSpace(ctx RequestContext, id, move string) (err error)
+	Delete(ctx RequestContext, documentID string) (rows int64, err error)
 }
 
 // SettingStorer defines required methods for persisting global and user level settings
@@ -156,7 +170,26 @@ type LinkStorer interface {
 	DeleteLink(ctx RequestContext, id string) (rows int64, err error)
 }
 
+// ActivityStorer defines required methods for persisting document activity
+type ActivityStorer interface {
+	RecordUserActivity(ctx RequestContext, activity activity.UserActivity) (err error)
+	GetDocumentActivity(ctx RequestContext, id string) (a []activity.DocumentActivity, err error)
+}
+
 // PageStorer defines required methods for persisting document pages
 type PageStorer interface {
 	GetPagesWithoutContent(ctx RequestContext, documentID string) (pages []page.Page, err error)
 }
+
+// SearchStorer defines required methods for persisting search queries
+type SearchStorer interface {
+	Add(ctx RequestContext, page page.Page) (err error)
+	Update(ctx RequestContext, page page.Page) (err error)
+	UpdateDocument(ctx RequestContext, page page.Page) (err error)
+	DeleteDocument(ctx RequestContext, page page.Page) (err error)
+	Rebuild(ctx RequestContext, p page.Page) (err error)
+	UpdateSequence(ctx RequestContext, page page.Page) (err error)
+	UpdateLevel(ctx RequestContext, page page.Page) (err error)
+	Delete(ctx RequestContext, page page.Page) (err error)
+	Documents(ctx RequestContext, keywords string) (results []search.DocumentSearch, err error)
+}

edition/boot/store.go

@@ -23,6 +23,7 @@ import (
 	org "github.com/documize/community/domain/organization/mysql"
 	page "github.com/documize/community/domain/page/mysql"
 	pin "github.com/documize/community/domain/pin/mysql"
+	search "github.com/documize/community/domain/search/mysql"
 	setting "github.com/documize/community/domain/setting/mysql"
 	space "github.com/documize/community/domain/space/mysql"
 	user "github.com/documize/community/domain/user/mysql"
@@ -41,6 +42,7 @@ func AttachStore(r *env.Runtime, s *domain.Store) {
 	s.Attachment = attachment.Scope{Runtime: r}
 	s.Link = link.Scope{Runtime: r}
 	s.Page = page.Scope{Runtime: r}
+	s.Search = search.Scope{Runtime: r}
 }
 
 // https://github.com/golang-sql/sqlexp/blob/c2488a8be21d20d31abf0d05c2735efd2d09afe4/quoter.go#L46

edition/community.go

@@ -18,13 +18,14 @@ import (
 	"github.com/documize/community/core/api"
 	"github.com/documize/community/core/api/request"
 	"github.com/documize/community/core/env"
+	"github.com/documize/community/domain"
+	"github.com/documize/community/domain/search"
 	"github.com/documize/community/domain/section"
 	"github.com/documize/community/edition/boot"
 	"github.com/documize/community/edition/logging"
 	_ "github.com/documize/community/embed" // the compressed front-end code and static data
 	"github.com/documize/community/server"
 	_ "github.com/go-sql-driver/mysql" // the mysql driver is required behind the scenes
-	"github.com/documize/community/domain"
 )
 
 var rt env.Runtime
@@ -68,6 +69,9 @@ func main() {
 	// Register smart sections
 	section.Register(rt)
 
+	// Search indexer queue
+	search.Start(&rt, &s)
+
 	ready := make(chan struct{}, 1) // channel signals router ready
 	server.Start(&rt, &s, ready)
 }

model/activity/activity.go

@@ -71,3 +71,16 @@ const (
 	// TypeFeedback records user providing document feedback
 	TypeFeedback Type = 10
 )
+
+// DocumentActivity represents an activity taken against a document.
+type DocumentActivity struct {
+	ID           int       `json:"id"`
+	OrgID        string    `json:"orgId"`
+	LabelID      string    `json:"folderId"`
+	DocumentID   string    `json:"documentId"`
+	UserID       string    `json:"userId"`
+	Firstname    string    `json:"firstname"`
+	Lastname     string    `json:"lastname"`
+	ActivityType int       `json:"activityType"`
+	Created      time.Time `json:"created"`
+}

server/routing/entries.go

@@ -19,6 +19,7 @@ import (
 	"github.com/documize/community/domain"
 	"github.com/documize/community/domain/attachment"
 	"github.com/documize/community/domain/auth"
+	"github.com/documize/community/domain/document"
 	"github.com/documize/community/domain/link"
 	"github.com/documize/community/domain/meta"
 	"github.com/documize/community/domain/organization"
@@ -31,7 +32,7 @@ import (
 
 // RegisterEndpoints register routes for serving API endpoints
 func RegisterEndpoints(rt *env.Runtime, s *domain.Store) {
-	// We pass server/application level contextual requirements into HTTP handlers
+	// Pass server/application level contextual requirements into HTTP handlers
 	// DO NOT pass in per request context (that is done by auth middleware per request)
 	pin := pin.Handler{Runtime: rt, Store: s}
 	auth := auth.Handler{Runtime: rt, Store: s}
@@ -40,6 +41,7 @@ func RegisterEndpoints(rt *env.Runtime, s *domain.Store) {
 	link := link.Handler{Runtime: rt, Store: s}
 	space := space.Handler{Runtime: rt, Store: s}
 	setting := setting.Handler{Runtime: rt, Store: s}
+	document := document.Handler{Runtime: rt, Store: s}
 	attachment := attachment.Handler{Runtime: rt, Store: s}
 	organization := organization.Handler{Runtime: rt, Store: s}
 
@@ -65,13 +67,13 @@ func RegisterEndpoints(rt *env.Runtime, s *domain.Store) {
 	// Import & Convert Document
 	Add(rt, RoutePrefixPrivate, "import/folder/{folderID}", []string{"POST", "OPTIONS"}, nil, endpoint.UploadConvertDocument)
 
-	Add(rt, RoutePrefixPrivate, "documents/{documentID}/export", []string{"GET", "OPTIONS"}, nil, endpoint.GetDocumentAsDocx)
+	// Add(rt, RoutePrefixPrivate, "documents/{documentID}/export", []string{"GET", "OPTIONS"}, nil, endpoint.GetDocumentAsDocx)
-	Add(rt, RoutePrefixPrivate, "documents", []string{"GET", "OPTIONS"}, []string{"filter", "tag"}, endpoint.GetDocumentsByTag)
+	Add(rt, RoutePrefixPrivate, "documents", []string{"GET", "OPTIONS"}, []string{"filter", "tag"}, document.ByTag)
-	Add(rt, RoutePrefixPrivate, "documents", []string{"GET", "OPTIONS"}, nil, endpoint.GetDocumentsByFolder)
+	Add(rt, RoutePrefixPrivate, "documents", []string{"GET", "OPTIONS"}, nil, document.BySpace)
-	Add(rt, RoutePrefixPrivate, "documents/{documentID}", []string{"GET", "OPTIONS"}, nil, endpoint.GetDocument)
+	Add(rt, RoutePrefixPrivate, "documents/{documentID}", []string{"GET", "OPTIONS"}, nil, document.Get)
-	Add(rt, RoutePrefixPrivate, "documents/{documentID}", []string{"PUT", "OPTIONS"}, nil, endpoint.UpdateDocument)
+	Add(rt, RoutePrefixPrivate, "documents/{documentID}", []string{"PUT", "OPTIONS"}, nil, document.Update)
-	Add(rt, RoutePrefixPrivate, "documents/{documentID}", []string{"DELETE", "OPTIONS"}, nil, endpoint.DeleteDocument)
+	Add(rt, RoutePrefixPrivate, "documents/{documentID}", []string{"DELETE", "OPTIONS"}, nil, document.Delete)
-	Add(rt, RoutePrefixPrivate, "documents/{documentID}/activity", []string{"GET", "OPTIONS"}, nil, endpoint.GetDocumentActivity)
+	Add(rt, RoutePrefixPrivate, "documents/{documentID}/activity", []string{"GET", "OPTIONS"}, nil, document.Activity)
 
 	Add(rt, RoutePrefixPrivate, "documents/{documentID}/pages/level", []string{"POST", "OPTIONS"}, nil, endpoint.ChangeDocumentPageLevel)
 	Add(rt, RoutePrefixPrivate, "documents/{documentID}/pages/sequence", []string{"POST", "OPTIONS"}, nil, endpoint.ChangeDocumentPageSequence)
@@ -117,7 +119,7 @@ func RegisterEndpoints(rt *env.Runtime, s *domain.Store) {
 	Add(rt, RoutePrefixPrivate, "users/{userID}", []string{"DELETE", "OPTIONS"}, nil, user.Delete)
 	Add(rt, RoutePrefixPrivate, "users/sync", []string{"GET", "OPTIONS"}, nil, endpoint.SyncKeycloak)
 
-	Add(rt, RoutePrefixPrivate, "search", []string{"GET", "OPTIONS"}, nil, endpoint.SearchDocuments)
+	Add(rt, RoutePrefixPrivate, "search", []string{"GET", "OPTIONS"}, nil, document.SearchDocuments)
 
 	Add(rt, RoutePrefixPrivate, "templates", []string{"POST", "OPTIONS"}, nil, endpoint.SaveAsTemplate)
 	Add(rt, RoutePrefixPrivate, "templates", []string{"GET", "OPTIONS"}, nil, endpoint.GetSavedTemplates)
@@ -137,7 +139,7 @@ func RegisterEndpoints(rt *env.Runtime, s *domain.Store) {
 
 	Add(rt, RoutePrefixPrivate, "links/{folderID}/{documentID}/{pageID}", []string{"GET", "OPTIONS"}, nil, link.GetLinkCandidates)
 	Add(rt, RoutePrefixPrivate, "links", []string{"GET", "OPTIONS"}, nil, link.SearchLinkCandidates)
-	Add(rt, RoutePrefixPrivate, "documents/{documentID}/links", []string{"GET", "OPTIONS"}, nil, endpoint.GetDocumentLinks)
+	Add(rt, RoutePrefixPrivate, "documents/{documentID}/links", []string{"GET", "OPTIONS"}, nil, document.DocumentLinks)
 
 	Add(rt, RoutePrefixPrivate, "global/smtp", []string{"GET", "OPTIONS"}, nil, setting.SMTP)
 	Add(rt, RoutePrefixPrivate, "global/smtp", []string{"PUT", "OPTIONS"}, nil, setting.SetSMTP)