mirror of
https://github.com/documize/community.git
synced 2025-07-19 13:19:43 +02:00
refactored document & search codebase to new API
This commit is contained in:
parent
d90b3249c3
commit
65390ab67d
12 changed files with 1288 additions and 35 deletions
|
@ -187,10 +187,10 @@ func (p *Persister) GetDocumentTemplates() (documents []entity.Document, err err
|
|||
func (p *Persister) GetPublicDocuments(orgID string) (documents []entity.SitemapDocument, err error) {
|
||||
err = Db.Select(&documents,
|
||||
`SELECT d.refid as documentid, d.title as document, d.revised as revised, l.refid as folderid, l.label as folder
|
||||
FROM document d LEFT JOIN label l ON l.refid=d.labelid
|
||||
WHERE d.orgid=?
|
||||
AND l.type=1
|
||||
AND d.template=0`, orgID)
|
||||
FROM document d LEFT JOIN label l ON l.refid=d.labelid
|
||||
WHERE d.orgid=?
|
||||
AND l.type=1
|
||||
AND d.template=0`, orgID)
|
||||
|
||||
if err != nil {
|
||||
log.Error(fmt.Sprintf("Unable to execute GetPublicDocuments for org %s", orgID), err)
|
||||
|
|
76
domain/activity/mysql/store.go
Normal file
76
domain/activity/mysql/store.go
Normal file
|
@ -0,0 +1,76 @@
|
|||
// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
|
||||
//
|
||||
// This software (Documize Community Edition) is licensed under
|
||||
// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
|
||||
//
|
||||
// You can operate outside the AGPL restrictions by purchasing
|
||||
// Documize Enterprise Edition and obtaining a commercial license
|
||||
// by contacting <sales@documize.com>.
|
||||
//
|
||||
// https://documize.com
|
||||
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"time"
|
||||
|
||||
"github.com/documize/community/core/env"
|
||||
"github.com/documize/community/core/streamutil"
|
||||
"github.com/documize/community/domain"
|
||||
"github.com/documize/community/model/activity"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// Scope provides data access to MySQL.
|
||||
type Scope struct {
|
||||
Runtime *env.Runtime
|
||||
}
|
||||
|
||||
// RecordUserActivity logs user initiated data changes.
|
||||
func (s Scope) RecordUserActivity(ctx domain.RequestContext, activity activity.UserActivity) (err error) {
|
||||
activity.OrgID = ctx.OrgID
|
||||
activity.UserID = ctx.UserID
|
||||
activity.Created = time.Now().UTC()
|
||||
|
||||
stmt, err := ctx.Transaction.Preparex("INSERT INTO useractivity (orgid, userid, labelid, sourceid, sourcetype, activitytype, created) VALUES (?, ?, ?, ?, ?, ?, ?)")
|
||||
defer streamutil.Close(stmt)
|
||||
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "prepare record user activity")
|
||||
return
|
||||
}
|
||||
|
||||
_, err = stmt.Exec(activity.OrgID, activity.UserID, activity.LabelID, activity.SourceID, activity.SourceType, activity.ActivityType, activity.Created)
|
||||
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "execute record user activity")
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// GetDocumentActivity returns the metadata for a specified document.
|
||||
func (s Scope) GetDocumentActivity(ctx domain.RequestContext, id string) (a []activity.DocumentActivity, err error) {
|
||||
qry := `SELECT a.id, a.created, a.orgid, IFNULL(a.userid, '') AS userid, a.labelid, a.sourceid as documentid, a.activitytype,
|
||||
IFNULL(u.firstname, 'Anonymous') AS firstname, IFNULL(u.lastname, 'Viewer') AS lastname
|
||||
FROM useractivity a
|
||||
LEFT JOIN user u ON a.userid=u.refid
|
||||
WHERE a.orgid=? AND a.sourceid=? AND a.sourcetype=2
|
||||
AND a.userid != '0' AND a.userid != ''
|
||||
ORDER BY a.created DESC`
|
||||
|
||||
err = s.Runtime.Db.Select(&a, qry, ctx.OrgID, id)
|
||||
|
||||
if len(a) == 0 {
|
||||
a = []activity.DocumentActivity{}
|
||||
}
|
||||
|
||||
if err != nil && err != sql.ErrNoRows {
|
||||
err = errors.Wrap(err, "select document user activity")
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
339
domain/document/endpoint.go
Normal file
339
domain/document/endpoint.go
Normal file
|
@ -0,0 +1,339 @@
|
|||
// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
|
||||
//
|
||||
// This software (Documize Community Edition) is licensed under
|
||||
// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
|
||||
//
|
||||
// You can operate outside the AGPL restrictions by purchasing
|
||||
// Documize Enterprise Edition and obtaining a commercial license
|
||||
// by contacting <sales@documize.com>.
|
||||
//
|
||||
// https://documize.com
|
||||
|
||||
package document
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
||||
"github.com/documize/community/core/env"
|
||||
"github.com/documize/community/core/request"
|
||||
"github.com/documize/community/core/response"
|
||||
"github.com/documize/community/core/streamutil"
|
||||
"github.com/documize/community/core/stringutil"
|
||||
"github.com/documize/community/domain"
|
||||
"github.com/documize/community/domain/space"
|
||||
"github.com/documize/community/model/activity"
|
||||
"github.com/documize/community/model/audit"
|
||||
"github.com/documize/community/model/doc"
|
||||
"github.com/documize/community/model/link"
|
||||
"github.com/documize/community/model/page"
|
||||
"github.com/documize/community/model/search"
|
||||
)
|
||||
|
||||
// Handler contains the runtime information such as logging and database.
|
||||
type Handler struct {
|
||||
Runtime *env.Runtime
|
||||
Store *domain.Store
|
||||
}
|
||||
|
||||
// SearchDocuments endpoint takes a list of keywords and returns a list of document references matching those keywords.
|
||||
func (h *Handler) SearchDocuments(w http.ResponseWriter, r *http.Request) {
|
||||
method := "document.search"
|
||||
ctx := domain.GetRequestContext(r)
|
||||
|
||||
keywords := request.Query(r, "keywords")
|
||||
decoded, err := url.QueryUnescape(keywords)
|
||||
if err != nil {
|
||||
response.WriteBadRequestError(w, method, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
results, err := h.Store.Search.Documents(ctx, decoded)
|
||||
if err != nil {
|
||||
h.Runtime.Log.Error("search failed", err)
|
||||
}
|
||||
|
||||
// Put in slugs for easy UI display of search URL
|
||||
for key, result := range results {
|
||||
result.DocumentSlug = stringutil.MakeSlug(result.DocumentTitle)
|
||||
result.FolderSlug = stringutil.MakeSlug(result.LabelName)
|
||||
results[key] = result
|
||||
}
|
||||
|
||||
if len(results) == 0 {
|
||||
results = []search.DocumentSearch{}
|
||||
}
|
||||
|
||||
h.Store.Audit.Record(ctx, audit.EventTypeSearch)
|
||||
|
||||
response.WriteJSON(w, results)
|
||||
}
|
||||
|
||||
// Get is an endpoint that returns the document-level information for a given documentID.
|
||||
func (h *Handler) Get(w http.ResponseWriter, r *http.Request) {
|
||||
method := "document.get"
|
||||
ctx := domain.GetRequestContext(r)
|
||||
|
||||
id := request.Param(r, "documentID")
|
||||
if len(id) == 0 {
|
||||
response.WriteMissingDataError(w, method, "documentID")
|
||||
return
|
||||
}
|
||||
|
||||
document, err := h.Store.Document.Get(ctx, id)
|
||||
if err == sql.ErrNoRows {
|
||||
response.WriteNotFoundError(w, method, id)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
response.WriteServerError(w, method, err)
|
||||
return
|
||||
}
|
||||
|
||||
if CanViewDocumentInFolder(ctx, *h.Store, document.LabelID) {
|
||||
response.WriteForbiddenError(w)
|
||||
return
|
||||
}
|
||||
|
||||
ctx.Transaction, err = h.Runtime.Db.Beginx()
|
||||
if err != nil {
|
||||
response.WriteServerError(w, method, err)
|
||||
return
|
||||
}
|
||||
|
||||
h.Store.Activity.RecordUserActivity(ctx, activity.UserActivity{
|
||||
LabelID: document.LabelID,
|
||||
SourceID: document.RefID,
|
||||
SourceType: activity.SourceTypeDocument,
|
||||
ActivityType: activity.TypeRead})
|
||||
|
||||
h.Store.Audit.Record(ctx, audit.EventTypeDocumentView)
|
||||
|
||||
ctx.Transaction.Commit()
|
||||
|
||||
response.WriteJSON(w, document)
|
||||
}
|
||||
|
||||
// Activity is an endpoint returning the activity logs for specified document.
|
||||
func (h *Handler) Activity(w http.ResponseWriter, r *http.Request) {
|
||||
method := "document.activity"
|
||||
ctx := domain.GetRequestContext(r)
|
||||
|
||||
id := request.Param(r, "documentID")
|
||||
if len(id) == 0 {
|
||||
response.WriteMissingDataError(w, method, "documentID")
|
||||
return
|
||||
}
|
||||
|
||||
a, err := h.Store.Activity.GetDocumentActivity(ctx, id)
|
||||
if err != nil && err != sql.ErrNoRows {
|
||||
response.WriteServerError(w, method, err)
|
||||
return
|
||||
}
|
||||
|
||||
response.WriteJSON(w, a)
|
||||
}
|
||||
|
||||
// DocumentLinks is an endpoint returning the links for a document.
|
||||
func (h *Handler) DocumentLinks(w http.ResponseWriter, r *http.Request) {
|
||||
method := "document.links"
|
||||
ctx := domain.GetRequestContext(r)
|
||||
|
||||
id := request.Param(r, "documentID")
|
||||
if len(id) == 0 {
|
||||
response.WriteMissingDataError(w, method, "documentID")
|
||||
return
|
||||
}
|
||||
|
||||
l, err := h.Store.Link.GetDocumentOutboundLinks(ctx, id)
|
||||
|
||||
if len(l) == 0 {
|
||||
l = []link.Link{}
|
||||
}
|
||||
|
||||
if err != nil && err != sql.ErrNoRows {
|
||||
response.WriteServerError(w, method, err)
|
||||
return
|
||||
}
|
||||
|
||||
response.WriteJSON(w, l)
|
||||
}
|
||||
|
||||
// BySpace is an endpoint that returns the documents in a given folder.
|
||||
func (h *Handler) BySpace(w http.ResponseWriter, r *http.Request) {
|
||||
method := "document.space"
|
||||
ctx := domain.GetRequestContext(r)
|
||||
|
||||
folderID := request.Query(r, "folder")
|
||||
|
||||
if len(folderID) == 0 {
|
||||
response.WriteMissingDataError(w, method, "folder")
|
||||
return
|
||||
}
|
||||
|
||||
if !space.CanViewSpace(ctx, *h.Store, folderID) {
|
||||
response.WriteForbiddenError(w)
|
||||
return
|
||||
}
|
||||
|
||||
documents, err := h.Store.Document.GetBySpace(ctx, folderID)
|
||||
if err != nil && err != sql.ErrNoRows {
|
||||
response.WriteServerError(w, method, err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(documents) == 0 {
|
||||
documents = []doc.Document{}
|
||||
}
|
||||
|
||||
response.WriteJSON(w, documents)
|
||||
}
|
||||
|
||||
// ByTag is an endpoint that returns the documents with a given tag.
|
||||
func (h *Handler) ByTag(w http.ResponseWriter, r *http.Request) {
|
||||
method := "document.space"
|
||||
ctx := domain.GetRequestContext(r)
|
||||
|
||||
tag := request.Query(r, "tag")
|
||||
if len(tag) == 0 {
|
||||
response.WriteMissingDataError(w, method, "tag")
|
||||
return
|
||||
}
|
||||
|
||||
documents, err := h.Store.Document.GetByTag(ctx, tag)
|
||||
if err != nil && err != sql.ErrNoRows {
|
||||
response.WriteServerError(w, method, err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(documents) == 0 {
|
||||
documents = []doc.Document{}
|
||||
}
|
||||
|
||||
response.WriteJSON(w, documents)
|
||||
}
|
||||
|
||||
// Update updates an existing document using the
|
||||
// format described in NewDocumentModel() encoded as JSON in the request.
|
||||
func (h *Handler) Update(w http.ResponseWriter, r *http.Request) {
|
||||
method := "document.space"
|
||||
ctx := domain.GetRequestContext(r)
|
||||
|
||||
documentID := request.Param(r, "documentID")
|
||||
if len(documentID) == 0 {
|
||||
response.WriteMissingDataError(w, method, "documentID")
|
||||
return
|
||||
}
|
||||
|
||||
if !ctx.Editor {
|
||||
response.WriteForbiddenError(w)
|
||||
return
|
||||
}
|
||||
|
||||
if !CanChangeDocument(ctx, *h.Store, documentID) {
|
||||
response.WriteForbiddenError(w)
|
||||
return
|
||||
}
|
||||
|
||||
defer streamutil.Close(r.Body)
|
||||
body, err := ioutil.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
response.WriteBadRequestError(w, method, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
d := doc.Document{}
|
||||
err = json.Unmarshal(body, &d)
|
||||
if err != nil {
|
||||
response.WriteBadRequestError(w, method, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
d.RefID = documentID
|
||||
|
||||
ctx.Transaction, err = h.Runtime.Db.Beginx()
|
||||
if err != nil {
|
||||
response.WriteServerError(w, method, err)
|
||||
return
|
||||
}
|
||||
|
||||
err = h.Store.Document.Update(ctx, d)
|
||||
if err != nil {
|
||||
ctx.Transaction.Rollback()
|
||||
response.WriteServerError(w, method, err)
|
||||
return
|
||||
}
|
||||
|
||||
h.Store.Audit.Record(ctx, audit.EventTypeDocumentUpdate)
|
||||
|
||||
p := page.Page{DocumentID: documentID, OrgID: ctx.OrgID, Body: d.Slug, Title: d.Title}
|
||||
h.Store.Search.UpdateDocument(ctx, p)
|
||||
|
||||
ctx.Transaction.Commit()
|
||||
|
||||
response.WriteEmpty(w)
|
||||
}
|
||||
|
||||
// Delete is an endpoint that deletes a document specified by documentID.
|
||||
func (h *Handler) Delete(w http.ResponseWriter, r *http.Request) {
|
||||
method := "document.delete"
|
||||
ctx := domain.GetRequestContext(r)
|
||||
|
||||
documentID := request.Param(r, "documentID")
|
||||
if len(documentID) == 0 {
|
||||
response.WriteMissingDataError(w, method, "documentID")
|
||||
return
|
||||
}
|
||||
|
||||
if !CanChangeDocument(ctx, *h.Store, documentID) {
|
||||
response.WriteForbiddenError(w)
|
||||
return
|
||||
}
|
||||
|
||||
doc, err := h.Store.Document.Get(ctx, documentID)
|
||||
if err != nil {
|
||||
response.WriteServerError(w, method, err)
|
||||
return
|
||||
}
|
||||
|
||||
ctx.Transaction, err = h.Runtime.Db.Beginx()
|
||||
if err != nil {
|
||||
response.WriteServerError(w, method, err)
|
||||
return
|
||||
}
|
||||
|
||||
_, err = h.Store.Document.Delete(ctx, documentID)
|
||||
if err != nil {
|
||||
ctx.Transaction.Rollback()
|
||||
response.WriteServerError(w, method, err)
|
||||
return
|
||||
}
|
||||
|
||||
_, err = h.Store.Pin.DeletePinnedDocument(ctx, documentID)
|
||||
if err != nil && err != sql.ErrNoRows {
|
||||
ctx.Transaction.Rollback()
|
||||
response.WriteServerError(w, method, err)
|
||||
return
|
||||
}
|
||||
|
||||
h.Store.Link.MarkOrphanDocumentLink(ctx, documentID)
|
||||
h.Store.Link.DeleteSourceDocumentLinks(ctx, documentID)
|
||||
|
||||
h.Store.Activity.RecordUserActivity(ctx, activity.UserActivity{
|
||||
LabelID: doc.LabelID,
|
||||
SourceID: documentID,
|
||||
SourceType: activity.SourceTypeDocument,
|
||||
ActivityType: activity.TypeDeleted})
|
||||
|
||||
h.Store.Audit.Record(ctx, audit.EventTypeDocumentDelete)
|
||||
|
||||
p := page.Page{DocumentID: documentID, OrgID: ctx.OrgID}
|
||||
h.Store.Search.DeleteDocument(ctx, p)
|
||||
|
||||
ctx.Transaction.Commit()
|
||||
|
||||
response.WriteEmpty(w)
|
||||
}
|
|
@ -13,10 +13,12 @@ package document
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/documize/community/core/env"
|
||||
"github.com/documize/community/core/streamutil"
|
||||
"github.com/documize/community/domain"
|
||||
"github.com/documize/community/domain/store/mysql"
|
||||
"github.com/documize/community/model/doc"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
@ -26,6 +28,30 @@ type Scope struct {
|
|||
Runtime *env.Runtime
|
||||
}
|
||||
|
||||
// Add inserts the given document record into the document table and audits that it has been done.
|
||||
func (s Scope) Add(ctx domain.RequestContext, document doc.Document) (err error) {
|
||||
document.OrgID = ctx.OrgID
|
||||
document.Created = time.Now().UTC()
|
||||
document.Revised = document.Created // put same time in both fields
|
||||
|
||||
stmt, err := ctx.Transaction.Preparex("INSERT INTO document (refid, orgid, labelid, userid, job, location, title, excerpt, slug, tags, template, created, revised) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
|
||||
defer streamutil.Close(stmt)
|
||||
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "prepare insert document")
|
||||
return
|
||||
}
|
||||
|
||||
_, err = stmt.Exec(document.RefID, document.OrgID, document.LabelID, document.UserID, document.Job, document.Location, document.Title, document.Excerpt, document.Slug, document.Tags, document.Template, document.Created, document.Revised)
|
||||
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "execuet insert document")
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Get fetches the document record with the given id fromt the document table and audits that it has been got.
|
||||
func (s Scope) Get(ctx domain.RequestContext, id string) (document doc.Document, err error) {
|
||||
stmt, err := s.Runtime.Db.Preparex("SELECT id, refid, orgid, labelid, userid, job, location, title, excerpt, slug, tags, template, layout, created, revised FROM document WHERE orgid=? and refid=?")
|
||||
|
@ -45,7 +71,203 @@ func (s Scope) Get(ctx domain.RequestContext, id string) (document doc.Document,
|
|||
return
|
||||
}
|
||||
|
||||
// MoveDocumentSpace changes the label for client's organization's documents which have space "id", to "move".
|
||||
// DocumentMeta returns the metadata for a specified document.
|
||||
func (s Scope) DocumentMeta(ctx domain.RequestContext, id string) (meta doc.DocumentMeta, err error) {
|
||||
sqlViewers := `SELECT MAX(a.created) as created,
|
||||
IFNULL(a.userid, '') AS userid, IFNULL(u.firstname, 'Anonymous') AS firstname, IFNULL(u.lastname, 'Viewer') AS lastname
|
||||
FROM audit a LEFT JOIN user u ON a.userid=u.refid
|
||||
WHERE a.orgid=? AND a.documentid=?
|
||||
AND a.userid != '0' AND a.userid != ''
|
||||
AND action='get-document'
|
||||
GROUP BY a.userid ORDER BY MAX(a.created) DESC`
|
||||
|
||||
err = s.Runtime.Db.Select(&meta.Viewers, sqlViewers, ctx.OrgID, id)
|
||||
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, fmt.Sprintf("select document viewers %s", id))
|
||||
return
|
||||
}
|
||||
|
||||
sqlEdits := `SELECT a.created,
|
||||
IFNULL(a.action, '') AS action, IFNULL(a.userid, '') AS userid, IFNULL(u.firstname, 'Anonymous') AS firstname, IFNULL(u.lastname, 'Viewer') AS lastname, IFNULL(a.pageid, '') AS pageid
|
||||
FROM audit a LEFT JOIN user u ON a.userid=u.refid
|
||||
WHERE a.orgid=? AND a.documentid=? AND a.userid != '0' AND a.userid != ''
|
||||
AND (a.action='update-page' OR a.action='add-page' OR a.action='remove-page')
|
||||
ORDER BY a.created DESC;`
|
||||
|
||||
err = s.Runtime.Db.Select(&meta.Editors, sqlEdits, ctx.OrgID, id)
|
||||
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, fmt.Sprintf("select document editors %s", id))
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// GetAll returns a slice containg all of the the documents for the client's organisation, with the most recient first.
|
||||
func (s Scope) GetAll() (ctx domain.RequestContext, documents []doc.Document, err error) {
|
||||
err = s.Runtime.Db.Select(&documents, "SELECT id, refid, orgid, labelid, userid, job, location, title, excerpt, slug, tags, template, layout, created, revised FROM document WHERE orgid=? AND template=0 ORDER BY revised DESC", ctx.OrgID)
|
||||
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "select documents")
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// GetBySpace returns a slice containing the documents for a given space, most recient first.
|
||||
func (s Scope) GetBySpace(ctx domain.RequestContext, folderID string) (documents []doc.Document, err error) {
|
||||
err = s.Runtime.Db.Select(&documents, "SELECT id, refid, orgid, labelid, userid, job, location, title, excerpt, slug, tags, template, layout, created, revised FROM document WHERE orgid=? AND template=0 AND labelid=? ORDER BY revised DESC", ctx.OrgID, folderID)
|
||||
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "select documents by space")
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// GetByTag returns a slice containing the documents with the specified tag, in title order.
|
||||
func (s Scope) GetByTag(ctx domain.RequestContext, tag string) (documents []doc.Document, err error) {
|
||||
tagQuery := "tags LIKE '%#" + tag + "#%'"
|
||||
|
||||
err = s.Runtime.Db.Select(&documents,
|
||||
`SELECT id, refid, orgid, labelid, userid, job, location, title, excerpt, slug, tags, template, layout, created, revised FROM document WHERE orgid=? AND template=0 AND `+tagQuery+` AND labelid IN
|
||||
(SELECT refid from label WHERE orgid=? AND type=2 AND userid=?
|
||||
UNION ALL SELECT refid FROM label a where orgid=? AND type=1 AND refid IN (SELECT labelid from labelrole WHERE orgid=? AND userid='' AND (canedit=1 OR canview=1))
|
||||
UNION ALL SELECT refid FROM label a where orgid=? AND type=3 AND refid IN (SELECT labelid from labelrole WHERE orgid=? AND userid=? AND (canedit=1 OR canview=1)))
|
||||
ORDER BY title`,
|
||||
ctx.OrgID,
|
||||
ctx.OrgID,
|
||||
ctx.UserID,
|
||||
ctx.OrgID,
|
||||
ctx.OrgID,
|
||||
ctx.OrgID,
|
||||
ctx.OrgID,
|
||||
ctx.UserID)
|
||||
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "select documents by tag")
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Templates returns a slice containing the documents available as templates to the client's organisation, in title order.
|
||||
func (s Scope) Templates() (ctx domain.RequestContext, documents []doc.Document, err error) {
|
||||
err = s.Runtime.Db.Select(&documents,
|
||||
`SELECT id, refid, orgid, labelid, userid, job, location, title, excerpt, slug, tags, template, layout, created, revised FROM document WHERE orgid=? AND template=1 AND labelid IN
|
||||
(SELECT refid from label WHERE orgid=? AND type=2 AND userid=?
|
||||
UNION ALL SELECT refid FROM label a where orgid=? AND type=1 AND refid IN (SELECT labelid from labelrole WHERE orgid=? AND userid='' AND (canedit=1 OR canview=1))
|
||||
UNION ALL SELECT refid FROM label a where orgid=? AND type=3 AND refid IN (SELECT labelid from labelrole WHERE orgid=? AND userid=? AND (canedit=1 OR canview=1)))
|
||||
ORDER BY title`,
|
||||
ctx.OrgID,
|
||||
ctx.OrgID,
|
||||
ctx.UserID,
|
||||
ctx.OrgID,
|
||||
ctx.OrgID,
|
||||
ctx.OrgID,
|
||||
ctx.OrgID,
|
||||
ctx.UserID)
|
||||
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "select documents templates")
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// PublicDocuments returns a slice of SitemapDocument records, holding documents in folders of type 1 (entity.TemplateTypePublic).
|
||||
func (s Scope) PublicDocuments(ctx domain.RequestContext, orgID string) (documents []doc.SitemapDocument, err error) {
|
||||
err = s.Runtime.Db.Select(&documents,
|
||||
`SELECT d.refid as documentid, d.title as document, d.revised as revised, l.refid as folderid, l.label as folder
|
||||
FROM document d LEFT JOIN label l ON l.refid=d.labelid
|
||||
WHERE d.orgid=?
|
||||
AND l.type=1
|
||||
AND d.template=0`, orgID)
|
||||
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, fmt.Sprintf("execute GetPublicDocuments for org %s%s", orgID))
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// DocumentList returns a slice containing the documents available as templates to the client's organisation, in title order.
|
||||
func (s Scope) DocumentList(ctx domain.RequestContext) (documents []doc.Document, err error) {
|
||||
err = s.Runtime.Db.Select(&documents,
|
||||
`SELECT id, refid, orgid, labelid, userid, job, location, title, excerpt, slug, tags, template, layout, created, revised FROM document WHERE orgid=? AND template=0 AND labelid IN
|
||||
(SELECT refid from label WHERE orgid=? AND type=2 AND userid=?
|
||||
UNION ALL SELECT refid FROM label a where orgid=? AND type=1 AND refid IN (SELECT labelid from labelrole WHERE orgid=? AND userid='' AND (canedit=1 OR canview=1))
|
||||
UNION ALL SELECT refid FROM label a where orgid=? AND type=3 AND refid IN (SELECT labelid from labelrole WHERE orgid=? AND userid=? AND (canedit=1 OR canview=1)))
|
||||
ORDER BY title`,
|
||||
ctx.OrgID,
|
||||
ctx.OrgID,
|
||||
ctx.UserID,
|
||||
ctx.OrgID,
|
||||
ctx.OrgID,
|
||||
ctx.OrgID,
|
||||
ctx.OrgID,
|
||||
ctx.UserID)
|
||||
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "select documents list")
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Update changes the given document record to the new values, updates search information and audits the action.
|
||||
func (s Scope) Update(ctx domain.RequestContext, document doc.Document) (err error) {
|
||||
document.Revised = time.Now().UTC()
|
||||
|
||||
stmt, err := ctx.Transaction.PrepareNamed("UPDATE document SET labelid=:labelid, userid=:userid, job=:job, location=:location, title=:title, excerpt=:excerpt, slug=:slug, tags=:tags, template=:template, layout=:layout, revised=:revised WHERE orgid=:orgid AND refid=:refid")
|
||||
defer streamutil.Close(stmt)
|
||||
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "prepare update document")
|
||||
return
|
||||
}
|
||||
|
||||
_, err = stmt.Exec(&document)
|
||||
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "execute update document")
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// ChangeDocumentSpace assigns the specified space to the document.
|
||||
func (s Scope) ChangeDocumentSpace(ctx domain.RequestContext, document, space string) (err error) {
|
||||
revised := time.Now().UTC()
|
||||
|
||||
stmt, err := ctx.Transaction.Preparex("UPDATE document SET labelid=?, revised=? WHERE orgid=? AND refid=?")
|
||||
defer streamutil.Close(stmt)
|
||||
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, fmt.Sprintf("prepare change document space %s", document))
|
||||
return
|
||||
}
|
||||
|
||||
_, err = stmt.Exec(space, revised, ctx.OrgID, document)
|
||||
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, fmt.Sprintf("execute change document space %s", document))
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// MoveDocumentSpace changes the space for client's organization's documents which have space "id", to "move".
|
||||
func (s Scope) MoveDocumentSpace(ctx domain.RequestContext, id, move string) (err error) {
|
||||
stmt, err := ctx.Transaction.Preparex("UPDATE document SET labelid=? WHERE orgid=? AND labelid=?")
|
||||
defer streamutil.Close(stmt)
|
||||
|
@ -64,19 +286,25 @@ func (s Scope) MoveDocumentSpace(ctx domain.RequestContext, id, move string) (er
|
|||
return
|
||||
}
|
||||
|
||||
// PublicDocuments returns a slice of SitemapDocument records, holding documents in folders of type 1 (entity.TemplateTypePublic).
|
||||
func (s Scope) PublicDocuments(ctx domain.RequestContext, orgID string) (documents []doc.SitemapDocument, err error) {
|
||||
err = s.Runtime.Db.Select(&documents,
|
||||
`SELECT d.refid as documentid, d.title as document, d.revised as revised, l.refid as folderid, l.label as folder
|
||||
FROM document d LEFT JOIN label l ON l.refid=d.labelid
|
||||
WHERE d.orgid=?
|
||||
AND l.type=1
|
||||
AND d.template=0`, orgID)
|
||||
// Delete delete the document pages in the database, updates the search subsystem, deletes the associated revisions and attachments,
|
||||
// audits the deletion, then finally deletes the document itself.
|
||||
func (s Scope) Delete(ctx domain.RequestContext, documentID string) (rows int64, err error) {
|
||||
b := mysql.BaseQuery{}
|
||||
rows, err = b.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE from page WHERE documentid=\"%s\" AND orgid=\"%s\"", documentID, ctx.OrgID))
|
||||
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, fmt.Sprintf("execute GetPublicDocuments for org %s%s", orgID))
|
||||
return
|
||||
}
|
||||
|
||||
_, err = b.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE from revision WHERE documentid=\"%s\" AND orgid=\"%s\"", documentID, ctx.OrgID))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
_, err = b.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE from attachment WHERE documentid=\"%s\" AND orgid=\"%s\"", documentID, ctx.OrgID))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return b.DeleteConstrained(ctx.Transaction, "document", ctx.OrgID, documentID)
|
||||
}
|
||||
|
|
315
domain/search/mysql/store.go
Normal file
315
domain/search/mysql/store.go
Normal file
|
@ -0,0 +1,315 @@
|
|||
// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
|
||||
//
|
||||
// This software (Documize Community Edition) is licensed under
|
||||
// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
|
||||
//
|
||||
// You can operate outside the AGPL restrictions by purchasing
|
||||
// Documize Enterprise Edition and obtaining a commercial license
|
||||
// by contacting <sales@documize.com>.
|
||||
//
|
||||
// https://documize.com
|
||||
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/documize/community/core/env"
|
||||
"github.com/documize/community/core/streamutil"
|
||||
"github.com/documize/community/core/stringutil"
|
||||
"github.com/documize/community/domain"
|
||||
"github.com/documize/community/domain/store/mysql"
|
||||
"github.com/documize/community/model"
|
||||
"github.com/documize/community/model/page"
|
||||
"github.com/documize/community/model/search"
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// Scope provides data access to MySQL.
|
||||
type Scope struct {
|
||||
Runtime *env.Runtime
|
||||
}
|
||||
|
||||
// Add search entry (legacy name: searchAdd).
|
||||
func (s Scope) Add(ctx domain.RequestContext, page page.Page) (err error) {
|
||||
id := page.RefID
|
||||
|
||||
// translate the html into text for the search
|
||||
nonHTML, err := stringutil.HTML(page.Body).Text(false)
|
||||
if err != nil {
|
||||
errors.Wrap(err, "search decode body")
|
||||
return
|
||||
}
|
||||
|
||||
// insert into the search table, getting the document title along the way
|
||||
var stmt *sqlx.Stmt
|
||||
stmt, err = ctx.Transaction.Preparex(
|
||||
"INSERT INTO search (id, orgid, documentid, level, sequence, documenttitle, slug, pagetitle, body, created, revised) " +
|
||||
" SELECT page.refid,page.orgid,document.refid,page.level,page.sequence,document.title,document.slug,page.title,?,page.created,page.revised " +
|
||||
" FROM document,page WHERE page.refid=? AND document.refid=page.documentid")
|
||||
|
||||
defer streamutil.Close(stmt)
|
||||
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "prepare search insert")
|
||||
return
|
||||
}
|
||||
|
||||
_, err = stmt.Exec(nonHTML, id)
|
||||
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "execute search insert")
|
||||
return
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Update search entry (legacy name: searchUpdate).
|
||||
func (s Scope) Update(ctx domain.RequestContext, page page.Page) (err error) {
|
||||
// translate the html into text for the search
|
||||
nonHTML, err := stringutil.HTML(page.Body).Text(false)
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "search decode body")
|
||||
return
|
||||
}
|
||||
|
||||
su, err := ctx.Transaction.Preparex("UPDATE search SET pagetitle=?,body=?,sequence=?,level=?,revised=? WHERE id=?")
|
||||
defer streamutil.Close(su)
|
||||
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "prepare search update")
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = su.Exec(page.Title, nonHTML, page.Sequence, page.Level, page.Revised, page.RefID)
|
||||
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "execute search update")
|
||||
return
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateDocument search entries for document (legacy name: searchUpdateDocument).
|
||||
func (s Scope) UpdateDocument(ctx domain.RequestContext, page page.Page) (err error) {
|
||||
stmt, err := ctx.Transaction.Preparex("UPDATE search SET documenttitle=?, slug=?, revised=? WHERE documentid=?")
|
||||
defer streamutil.Close(stmt)
|
||||
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "prepare search document update")
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = stmt.Exec(page.Title, page.Body, time.Now().UTC(), page.DocumentID)
|
||||
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "execute search document update")
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteDocument removes document search entries (legacy name: searchDeleteDocument)
|
||||
func (s Scope) DeleteDocument(ctx domain.RequestContext, page page.Page) (err error) {
|
||||
var bm = mysql.BaseQuery{}
|
||||
|
||||
_, err = bm.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE from search WHERE documentid='%s'", page.DocumentID))
|
||||
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "delete document search entries")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Rebuild ... (legacy name: searchRebuild)
|
||||
func (s Scope) Rebuild(ctx domain.RequestContext, p page.Page) (err error) {
|
||||
var bm = mysql.BaseQuery{}
|
||||
|
||||
_, err = bm.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE from search WHERE documentid='%s'", p.DocumentID))
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
var pages []struct{ ID string }
|
||||
|
||||
stmt2, err := ctx.Transaction.Preparex("SELECT refid as id FROM page WHERE documentid=? ")
|
||||
defer streamutil.Close(stmt2)
|
||||
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
err = stmt2.Select(&pages, p.DocumentID)
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if len(pages) > 0 {
|
||||
for _, pg := range pages {
|
||||
err = s.Add(ctx, page.Page{BaseEntity: model.BaseEntity{RefID: pg.ID}})
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, err.Error())
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// rebuild doc-level tags & excerpts
|
||||
// get the 0'th page data and rewrite it
|
||||
|
||||
target := page.Page{}
|
||||
|
||||
stmt1, err := ctx.Transaction.Preparex("SELECT * FROM page WHERE refid=?")
|
||||
defer streamutil.Close(stmt1)
|
||||
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
err = stmt1.Get(&target, pages[0].ID)
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
err = s.Update(ctx, target) // to rebuild the document-level tags + excerpt
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, err.Error())
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// UpdateSequence ... (legacy name: searchUpdateSequence)
|
||||
func (s Scope) UpdateSequence(ctx domain.RequestContext, page page.Page) (err error) {
|
||||
supdate, err := ctx.Transaction.Preparex("UPDATE search SET sequence=?,revised=? WHERE id=?")
|
||||
defer streamutil.Close(supdate)
|
||||
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "prepare search update sequence")
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = supdate.Exec(page.Sequence, time.Now().UTC(), page.RefID)
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "execute search update sequence")
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// UpdateLevel ... legacy name: searchUpdateLevel)
|
||||
func (s Scope) UpdateLevel(ctx domain.RequestContext, page page.Page) (err error) {
|
||||
pageID := page.RefID
|
||||
level := page.Level
|
||||
|
||||
supdate, err := ctx.Transaction.Preparex("UPDATE search SET level=?,revised=? WHERE id=?")
|
||||
defer streamutil.Close(supdate)
|
||||
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "prepare search update level")
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = supdate.Exec(level, time.Now().UTC(), pageID)
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "execute search update level")
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Delete ... (legacy name: searchDelete).
|
||||
func (s Scope) Delete(ctx domain.RequestContext, page page.Page) (err error) {
|
||||
var bm = mysql.BaseQuery{}
|
||||
_, err = bm.DeleteConstrainedWithID(ctx.Transaction, "search", ctx.OrgID, page.RefID)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Documents searches the documents that the client is allowed to see, using the keywords search string, then audits that search.
|
||||
// Visible documents include both those in the client's own organisation and those that are public, or whose visibility includes the client.
|
||||
func (s Scope) Documents(ctx domain.RequestContext, keywords string) (results []search.DocumentSearch, err error) {
|
||||
if len(keywords) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
var tagQuery, keywordQuery string
|
||||
|
||||
r, _ := regexp.Compile(`(#[a-z0-9][a-z0-9\-_]*)`)
|
||||
res := r.FindAllString(keywords, -1)
|
||||
|
||||
if len(res) == 0 {
|
||||
tagQuery = " "
|
||||
} else {
|
||||
if len(res) == 1 {
|
||||
tagQuery = " AND document.tags LIKE '%" + res[0] + "#%' "
|
||||
} else {
|
||||
fmt.Println("lots of tags!")
|
||||
|
||||
tagQuery = " AND ("
|
||||
|
||||
for i := 0; i < len(res); i++ {
|
||||
tagQuery += "document.tags LIKE '%" + res[i] + "#%'"
|
||||
if i < len(res)-1 {
|
||||
tagQuery += " OR "
|
||||
}
|
||||
}
|
||||
|
||||
tagQuery += ") "
|
||||
}
|
||||
|
||||
keywords = r.ReplaceAllString(keywords, "")
|
||||
keywords = strings.Replace(keywords, " ", "", -1)
|
||||
}
|
||||
|
||||
keywords = strings.TrimSpace(keywords)
|
||||
|
||||
if len(keywords) > 0 {
|
||||
keywordQuery = "AND MATCH(pagetitle,body) AGAINST('" + keywords + "' in boolean mode)"
|
||||
}
|
||||
|
||||
sql := `SELECT search.id, documentid, pagetitle, document.labelid, document.title as documenttitle, document.tags,
|
||||
COALESCE(label.label,'Unknown') AS labelname, document.excerpt as documentexcerpt
|
||||
FROM search, document LEFT JOIN label ON label.orgid=document.orgid AND label.refid = document.labelid
|
||||
WHERE search.documentid = document.refid AND search.orgid=? AND document.template=0 ` + tagQuery +
|
||||
`AND document.labelid IN
|
||||
(SELECT refid from label WHERE orgid=? AND type=2 AND userid=?
|
||||
UNION ALL SELECT refid FROM label a where orgid=? AND type=1 AND refid IN (SELECT labelid from labelrole WHERE orgid=? AND userid='' AND (canedit=1 OR canview=1))
|
||||
UNION ALL SELECT refid FROM label a where orgid=? AND type=3 AND refid IN (SELECT labelid from labelrole WHERE orgid=? AND userid=? AND (canedit=1 OR canview=1))) ` + keywordQuery
|
||||
// AND MATCH(pagetitle,body)
|
||||
// AGAINST('` + keywords + "' in boolean mode)"
|
||||
|
||||
err = s.Runtime.Db.Select(&results,
|
||||
sql,
|
||||
ctx.OrgID,
|
||||
ctx.OrgID,
|
||||
ctx.UserID,
|
||||
ctx.OrgID,
|
||||
ctx.OrgID,
|
||||
ctx.OrgID,
|
||||
ctx.OrgID,
|
||||
ctx.UserID)
|
||||
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "search documents")
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
239
domain/search/search.go
Normal file
239
domain/search/search.go
Normal file
|
@ -0,0 +1,239 @@
|
|||
// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
|
||||
//
|
||||
// This software (Documize Community Edition) is licensed under
|
||||
// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
|
||||
//
|
||||
// You can operate outside the AGPL restrictions by purchasing
|
||||
// Documize Enterprise Edition and obtaining a commercial license
|
||||
// by contacting <sales@documize.com>.
|
||||
//
|
||||
// https://documize.com
|
||||
|
||||
package search
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/documize/community/core/env"
|
||||
"github.com/documize/community/domain"
|
||||
"github.com/documize/community/model"
|
||||
"github.com/documize/community/model/doc"
|
||||
"github.com/documize/community/model/page"
|
||||
)
|
||||
|
||||
// Indexer type provides the datastructure for the queues of activity to be serialized through a single background goroutine.
|
||||
// NOTE if the queue becomes full, the system will trigger the rebuilding entire files in order to clear the backlog.
|
||||
type Indexer struct {
|
||||
queue chan queueEntry
|
||||
rebuild map[string]bool
|
||||
rebuildLock sync.RWMutex
|
||||
givenWarning bool
|
||||
runtime *env.Runtime
|
||||
store *domain.Store
|
||||
}
|
||||
|
||||
type queueEntry struct {
|
||||
action func(domain.RequestContext, page.Page) error
|
||||
isRebuild bool
|
||||
page.Page
|
||||
ctx domain.RequestContext
|
||||
}
|
||||
|
||||
var searches *Indexer
|
||||
|
||||
const searchQueueLength = 2048 // NOTE the largest 15Mb docx in the test set generates 2142 queue entries, but the queue is constantly emptied
|
||||
|
||||
// Start the background indexer
|
||||
func Start(rt *env.Runtime, s *domain.Store) {
|
||||
searches = &Indexer{}
|
||||
searches.queue = make(chan queueEntry, searchQueueLength) // provide some decoupling
|
||||
searches.rebuild = make(map[string]bool)
|
||||
searches.runtime = rt
|
||||
searches.store = s
|
||||
|
||||
go searches.searchProcessQueue()
|
||||
}
|
||||
|
||||
// searchProcessQueue is run as a goroutine, it processes the queue of search index update requests.
|
||||
func (m *Indexer) searchProcessQueue() {
|
||||
for {
|
||||
//fmt.Println("DEBUG queue length=", len(Searches.queue))
|
||||
if len(m.queue) <= searchQueueLength/20 { // on a busy server, the queue may never get to zero - so use 5%
|
||||
m.rebuildLock.Lock()
|
||||
for docid := range m.rebuild {
|
||||
m.queue <- queueEntry{
|
||||
action: m.store.Search.Rebuild,
|
||||
isRebuild: true,
|
||||
Page: page.Page{DocumentID: docid},
|
||||
}
|
||||
delete(m.rebuild, docid)
|
||||
}
|
||||
m.rebuildLock.Unlock()
|
||||
}
|
||||
|
||||
qe := <-m.queue
|
||||
doit := true
|
||||
|
||||
if len(qe.DocumentID) > 0 {
|
||||
m.rebuildLock.RLock()
|
||||
if m.rebuild[qe.DocumentID] {
|
||||
doit = false // don't execute an action on a document queued to be rebuilt
|
||||
}
|
||||
m.rebuildLock.RUnlock()
|
||||
}
|
||||
|
||||
if doit {
|
||||
tx, err := m.runtime.Db.Beginx()
|
||||
if err != nil {
|
||||
} else {
|
||||
ctx := qe.ctx
|
||||
ctx.Transaction = tx
|
||||
err = qe.action(ctx, qe.Page)
|
||||
if err != nil {
|
||||
tx.Rollback()
|
||||
// This action has failed, so re-build indexes for the entire document,
|
||||
// provided it was not a re-build command that failed and we know the documentId.
|
||||
if !qe.isRebuild && len(qe.DocumentID) > 0 {
|
||||
m.rebuildLock.Lock()
|
||||
m.rebuild[qe.DocumentID] = true
|
||||
m.rebuildLock.Unlock()
|
||||
}
|
||||
} else {
|
||||
tx.Commit()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Indexer) addQueue(qe queueEntry) error {
|
||||
lsq := len(m.queue)
|
||||
|
||||
if lsq >= (searchQueueLength - 1) {
|
||||
if qe.DocumentID != "" {
|
||||
m.rebuildLock.Lock()
|
||||
if !m.rebuild[qe.DocumentID] {
|
||||
m.runtime.Log.Info(fmt.Sprintf("WARNING: Search Queue Has No Space! Marked rebuild index for document id %s", qe.DocumentID))
|
||||
}
|
||||
m.rebuild[qe.DocumentID] = true
|
||||
m.rebuildLock.Unlock()
|
||||
} else {
|
||||
m.runtime.Log.Error("addQueue", errors.New("WARNING: Search Queue Has No Space! But unable to index unknown document id"))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
if lsq > ((8 * searchQueueLength) / 10) {
|
||||
if !m.givenWarning {
|
||||
m.runtime.Log.Info(fmt.Sprintf("WARNING: Searches.queue length %d exceeds 80%% of capacity", lsq))
|
||||
m.givenWarning = true
|
||||
}
|
||||
} else {
|
||||
if m.givenWarning {
|
||||
m.runtime.Log.Info(fmt.Sprintf("INFO: Searches.queue length %d now below 80%% of capacity", lsq))
|
||||
m.givenWarning = false
|
||||
}
|
||||
}
|
||||
|
||||
m.queue <- qe
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Add should be called when a new page is added to a document.
|
||||
func (m *Indexer) Add(ctx domain.RequestContext, page page.Page, id string) (err error) {
|
||||
page.RefID = id
|
||||
|
||||
err = m.addQueue(queueEntry{
|
||||
action: m.store.Search.Add,
|
||||
Page: page,
|
||||
ctx: ctx,
|
||||
})
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Update should be called after a page record has been updated.
|
||||
func (m *Indexer) Update(ctx domain.RequestContext, page page.Page) (err error) {
|
||||
err = m.addQueue(queueEntry{
|
||||
action: m.store.Search.Update,
|
||||
Page: page,
|
||||
ctx: ctx,
|
||||
})
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// UpdateDocument should be called after a document record has been updated.
|
||||
func (m *Indexer) UpdateDocument(ctx domain.RequestContext, document doc.Document) (err error) {
|
||||
err = m.addQueue(queueEntry{
|
||||
action: m.store.Search.UpdateDocument,
|
||||
Page: page.Page{
|
||||
DocumentID: document.RefID,
|
||||
Title: document.Title,
|
||||
Body: document.Slug, // NOTE body==slug in this context
|
||||
},
|
||||
ctx: ctx,
|
||||
})
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// DeleteDocument should be called after a document record has been deleted.
|
||||
func (m *Indexer) DeleteDocument(ctx domain.RequestContext, documentID string) (err error) {
|
||||
if len(documentID) > 0 {
|
||||
m.queue <- queueEntry{
|
||||
action: m.store.Search.DeleteDocument,
|
||||
Page: page.Page{DocumentID: documentID},
|
||||
ctx: ctx,
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// UpdateSequence should be called after a page record has been resequenced.
|
||||
func (m *Indexer) UpdateSequence(ctx domain.RequestContext, documentID, pageID string, sequence float64) (err error) {
|
||||
err = m.addQueue(queueEntry{
|
||||
action: m.store.Search.UpdateSequence,
|
||||
Page: page.Page{
|
||||
BaseEntity: model.BaseEntity{RefID: pageID},
|
||||
Sequence: sequence,
|
||||
DocumentID: documentID,
|
||||
},
|
||||
ctx: ctx,
|
||||
})
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// UpdateLevel should be called after the level of a page has been changed.
|
||||
func (m *Indexer) UpdateLevel(ctx domain.RequestContext, documentID, pageID string, level int) (err error) {
|
||||
err = m.addQueue(queueEntry{
|
||||
action: m.store.Search.UpdateLevel,
|
||||
Page: page.Page{
|
||||
BaseEntity: model.BaseEntity{RefID: pageID},
|
||||
Level: uint64(level),
|
||||
DocumentID: documentID,
|
||||
},
|
||||
ctx: ctx,
|
||||
})
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Delete should be called after a page has been deleted.
|
||||
func (m *Indexer) Delete(ctx domain.RequestContext, documentID, pageID string) (rows int64, err error) {
|
||||
err = m.addQueue(queueEntry{
|
||||
action: m.store.Search.Delete,
|
||||
Page: page.Page{
|
||||
BaseEntity: model.BaseEntity{RefID: pageID},
|
||||
DocumentID: documentID,
|
||||
},
|
||||
ctx: ctx,
|
||||
})
|
||||
|
||||
return
|
||||
}
|
|
@ -13,15 +13,19 @@
|
|||
// Spaces in Documize contain documents.
|
||||
package space
|
||||
|
||||
/*
|
||||
import (
|
||||
"database/sql"
|
||||
|
||||
"github.com/documize/community/domain"
|
||||
)
|
||||
|
||||
// CanViewSpace returns if the user has permission to view the given spaceID.
|
||||
func CanViewSpace(s domain.StoreContext, spaceID string) (hasPermission bool) {
|
||||
roles, err := GetRoles(s, spaceID)
|
||||
func CanViewSpace(ctx domain.RequestContext, s domain.Store, spaceID string) (hasPermission bool) {
|
||||
roles, err := s.Space.GetRoles(ctx, spaceID)
|
||||
if err == sql.ErrNoRows {
|
||||
err = nil
|
||||
}
|
||||
if err != nil {
|
||||
s.Runtime.Log.Error(fmt.Sprintf("check space permissions %s", spaceID), err)
|
||||
return false
|
||||
}
|
||||
|
||||
|
@ -35,14 +39,13 @@ func CanViewSpace(s domain.StoreContext, spaceID string) (hasPermission bool) {
|
|||
}
|
||||
|
||||
// CanViewSpaceDocuments returns if the user has permission to view a document within the specified space.
|
||||
func CanViewSpaceDocuments(s domain.StoreContext, spaceID string) (hasPermission bool) {
|
||||
roles, err := GetRoles(s, spaceID)
|
||||
func CanViewSpaceDocuments(ctx domain.RequestContext, s domain.Store, spaceID string) (hasPermission bool) {
|
||||
roles, err := s.Space.GetRoles(ctx, spaceID)
|
||||
if err == sql.ErrNoRows {
|
||||
err = nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
s.Runtime.Log.Error(fmt.Sprintf("check space permissions %s", spaceID), err)
|
||||
return false
|
||||
}
|
||||
|
||||
|
@ -54,4 +57,3 @@ func CanViewSpaceDocuments(s domain.StoreContext, spaceID string) (hasPermission
|
|||
|
||||
return false
|
||||
}
|
||||
*/
|
||||
|
|
|
@ -14,6 +14,7 @@ package domain
|
|||
|
||||
import (
|
||||
"github.com/documize/community/model/account"
|
||||
"github.com/documize/community/model/activity"
|
||||
"github.com/documize/community/model/attachment"
|
||||
"github.com/documize/community/model/audit"
|
||||
"github.com/documize/community/model/doc"
|
||||
|
@ -21,6 +22,7 @@ import (
|
|||
"github.com/documize/community/model/org"
|
||||
"github.com/documize/community/model/page"
|
||||
"github.com/documize/community/model/pin"
|
||||
"github.com/documize/community/model/search"
|
||||
"github.com/documize/community/model/space"
|
||||
"github.com/documize/community/model/user"
|
||||
)
|
||||
|
@ -38,6 +40,8 @@ type Store struct {
|
|||
Attachment AttachmentStorer
|
||||
Link LinkStorer
|
||||
Page PageStorer
|
||||
Activity ActivityStorer
|
||||
Search SearchStorer
|
||||
}
|
||||
|
||||
// SpaceStorer defines required methods for space management
|
||||
|
@ -120,9 +124,19 @@ type AuditStorer interface {
|
|||
|
||||
// DocumentStorer defines required methods for document handling
|
||||
type DocumentStorer interface {
|
||||
Add(ctx RequestContext, document doc.Document) (err error)
|
||||
Get(ctx RequestContext, id string) (document doc.Document, err error)
|
||||
MoveDocumentSpace(ctx RequestContext, id, move string) (err error)
|
||||
GetAll() (ctx RequestContext, documents []doc.Document, err error)
|
||||
GetBySpace(ctx RequestContext, folderID string) (documents []doc.Document, err error)
|
||||
GetByTag(ctx RequestContext, tag string) (documents []doc.Document, err error)
|
||||
DocumentList(ctx RequestContext) (documents []doc.Document, err error)
|
||||
Templates() (ctx RequestContext, documents []doc.Document, err error)
|
||||
DocumentMeta(ctx RequestContext, id string) (meta doc.DocumentMeta, err error)
|
||||
PublicDocuments(ctx RequestContext, orgID string) (documents []doc.SitemapDocument, err error)
|
||||
Update(ctx RequestContext, document doc.Document) (err error)
|
||||
ChangeDocumentSpace(ctx RequestContext, document, space string) (err error)
|
||||
MoveDocumentSpace(ctx RequestContext, id, move string) (err error)
|
||||
Delete(ctx RequestContext, documentID string) (rows int64, err error)
|
||||
}
|
||||
|
||||
// SettingStorer defines required methods for persisting global and user level settings
|
||||
|
@ -156,7 +170,26 @@ type LinkStorer interface {
|
|||
DeleteLink(ctx RequestContext, id string) (rows int64, err error)
|
||||
}
|
||||
|
||||
// ActivityStorer defines required methods for persisting document activity
|
||||
type ActivityStorer interface {
|
||||
RecordUserActivity(ctx RequestContext, activity activity.UserActivity) (err error)
|
||||
GetDocumentActivity(ctx RequestContext, id string) (a []activity.DocumentActivity, err error)
|
||||
}
|
||||
|
||||
// PageStorer defines required methods for persisting document pages
|
||||
type PageStorer interface {
|
||||
GetPagesWithoutContent(ctx RequestContext, documentID string) (pages []page.Page, err error)
|
||||
}
|
||||
|
||||
// SearchStorer defines required methods for persisting search queries
|
||||
type SearchStorer interface {
|
||||
Add(ctx RequestContext, page page.Page) (err error)
|
||||
Update(ctx RequestContext, page page.Page) (err error)
|
||||
UpdateDocument(ctx RequestContext, page page.Page) (err error)
|
||||
DeleteDocument(ctx RequestContext, page page.Page) (err error)
|
||||
Rebuild(ctx RequestContext, p page.Page) (err error)
|
||||
UpdateSequence(ctx RequestContext, page page.Page) (err error)
|
||||
UpdateLevel(ctx RequestContext, page page.Page) (err error)
|
||||
Delete(ctx RequestContext, page page.Page) (err error)
|
||||
Documents(ctx RequestContext, keywords string) (results []search.DocumentSearch, err error)
|
||||
}
|
||||
|
|
|
@ -23,6 +23,7 @@ import (
|
|||
org "github.com/documize/community/domain/organization/mysql"
|
||||
page "github.com/documize/community/domain/page/mysql"
|
||||
pin "github.com/documize/community/domain/pin/mysql"
|
||||
search "github.com/documize/community/domain/search/mysql"
|
||||
setting "github.com/documize/community/domain/setting/mysql"
|
||||
space "github.com/documize/community/domain/space/mysql"
|
||||
user "github.com/documize/community/domain/user/mysql"
|
||||
|
@ -41,6 +42,7 @@ func AttachStore(r *env.Runtime, s *domain.Store) {
|
|||
s.Attachment = attachment.Scope{Runtime: r}
|
||||
s.Link = link.Scope{Runtime: r}
|
||||
s.Page = page.Scope{Runtime: r}
|
||||
s.Search = search.Scope{Runtime: r}
|
||||
}
|
||||
|
||||
// https://github.com/golang-sql/sqlexp/blob/c2488a8be21d20d31abf0d05c2735efd2d09afe4/quoter.go#L46
|
||||
|
|
|
@ -18,13 +18,14 @@ import (
|
|||
"github.com/documize/community/core/api"
|
||||
"github.com/documize/community/core/api/request"
|
||||
"github.com/documize/community/core/env"
|
||||
"github.com/documize/community/domain"
|
||||
"github.com/documize/community/domain/search"
|
||||
"github.com/documize/community/domain/section"
|
||||
"github.com/documize/community/edition/boot"
|
||||
"github.com/documize/community/edition/logging"
|
||||
_ "github.com/documize/community/embed" // the compressed front-end code and static data
|
||||
"github.com/documize/community/server"
|
||||
_ "github.com/go-sql-driver/mysql" // the mysql driver is required behind the scenes
|
||||
"github.com/documize/community/domain"
|
||||
)
|
||||
|
||||
var rt env.Runtime
|
||||
|
@ -68,6 +69,9 @@ func main() {
|
|||
// Register smart sections
|
||||
section.Register(rt)
|
||||
|
||||
// Search indexer queue
|
||||
search.Start(&rt, &s)
|
||||
|
||||
ready := make(chan struct{}, 1) // channel signals router ready
|
||||
server.Start(&rt, &s, ready)
|
||||
}
|
||||
|
|
|
@ -71,3 +71,16 @@ const (
|
|||
// TypeFeedback records user providing document feedback
|
||||
TypeFeedback Type = 10
|
||||
)
|
||||
|
||||
// DocumentActivity represents an activity taken against a document.
|
||||
type DocumentActivity struct {
|
||||
ID int `json:"id"`
|
||||
OrgID string `json:"orgId"`
|
||||
LabelID string `json:"folderId"`
|
||||
DocumentID string `json:"documentId"`
|
||||
UserID string `json:"userId"`
|
||||
Firstname string `json:"firstname"`
|
||||
Lastname string `json:"lastname"`
|
||||
ActivityType int `json:"activityType"`
|
||||
Created time.Time `json:"created"`
|
||||
}
|
||||
|
|
|
@ -19,6 +19,7 @@ import (
|
|||
"github.com/documize/community/domain"
|
||||
"github.com/documize/community/domain/attachment"
|
||||
"github.com/documize/community/domain/auth"
|
||||
"github.com/documize/community/domain/document"
|
||||
"github.com/documize/community/domain/link"
|
||||
"github.com/documize/community/domain/meta"
|
||||
"github.com/documize/community/domain/organization"
|
||||
|
@ -31,7 +32,7 @@ import (
|
|||
|
||||
// RegisterEndpoints register routes for serving API endpoints
|
||||
func RegisterEndpoints(rt *env.Runtime, s *domain.Store) {
|
||||
// We pass server/application level contextual requirements into HTTP handlers
|
||||
// Pass server/application level contextual requirements into HTTP handlers
|
||||
// DO NOT pass in per request context (that is done by auth middleware per request)
|
||||
pin := pin.Handler{Runtime: rt, Store: s}
|
||||
auth := auth.Handler{Runtime: rt, Store: s}
|
||||
|
@ -40,6 +41,7 @@ func RegisterEndpoints(rt *env.Runtime, s *domain.Store) {
|
|||
link := link.Handler{Runtime: rt, Store: s}
|
||||
space := space.Handler{Runtime: rt, Store: s}
|
||||
setting := setting.Handler{Runtime: rt, Store: s}
|
||||
document := document.Handler{Runtime: rt, Store: s}
|
||||
attachment := attachment.Handler{Runtime: rt, Store: s}
|
||||
organization := organization.Handler{Runtime: rt, Store: s}
|
||||
|
||||
|
@ -65,13 +67,13 @@ func RegisterEndpoints(rt *env.Runtime, s *domain.Store) {
|
|||
// Import & Convert Document
|
||||
Add(rt, RoutePrefixPrivate, "import/folder/{folderID}", []string{"POST", "OPTIONS"}, nil, endpoint.UploadConvertDocument)
|
||||
|
||||
Add(rt, RoutePrefixPrivate, "documents/{documentID}/export", []string{"GET", "OPTIONS"}, nil, endpoint.GetDocumentAsDocx)
|
||||
Add(rt, RoutePrefixPrivate, "documents", []string{"GET", "OPTIONS"}, []string{"filter", "tag"}, endpoint.GetDocumentsByTag)
|
||||
Add(rt, RoutePrefixPrivate, "documents", []string{"GET", "OPTIONS"}, nil, endpoint.GetDocumentsByFolder)
|
||||
Add(rt, RoutePrefixPrivate, "documents/{documentID}", []string{"GET", "OPTIONS"}, nil, endpoint.GetDocument)
|
||||
Add(rt, RoutePrefixPrivate, "documents/{documentID}", []string{"PUT", "OPTIONS"}, nil, endpoint.UpdateDocument)
|
||||
Add(rt, RoutePrefixPrivate, "documents/{documentID}", []string{"DELETE", "OPTIONS"}, nil, endpoint.DeleteDocument)
|
||||
Add(rt, RoutePrefixPrivate, "documents/{documentID}/activity", []string{"GET", "OPTIONS"}, nil, endpoint.GetDocumentActivity)
|
||||
// Add(rt, RoutePrefixPrivate, "documents/{documentID}/export", []string{"GET", "OPTIONS"}, nil, endpoint.GetDocumentAsDocx)
|
||||
Add(rt, RoutePrefixPrivate, "documents", []string{"GET", "OPTIONS"}, []string{"filter", "tag"}, document.ByTag)
|
||||
Add(rt, RoutePrefixPrivate, "documents", []string{"GET", "OPTIONS"}, nil, document.BySpace)
|
||||
Add(rt, RoutePrefixPrivate, "documents/{documentID}", []string{"GET", "OPTIONS"}, nil, document.Get)
|
||||
Add(rt, RoutePrefixPrivate, "documents/{documentID}", []string{"PUT", "OPTIONS"}, nil, document.Update)
|
||||
Add(rt, RoutePrefixPrivate, "documents/{documentID}", []string{"DELETE", "OPTIONS"}, nil, document.Delete)
|
||||
Add(rt, RoutePrefixPrivate, "documents/{documentID}/activity", []string{"GET", "OPTIONS"}, nil, document.Activity)
|
||||
|
||||
Add(rt, RoutePrefixPrivate, "documents/{documentID}/pages/level", []string{"POST", "OPTIONS"}, nil, endpoint.ChangeDocumentPageLevel)
|
||||
Add(rt, RoutePrefixPrivate, "documents/{documentID}/pages/sequence", []string{"POST", "OPTIONS"}, nil, endpoint.ChangeDocumentPageSequence)
|
||||
|
@ -117,7 +119,7 @@ func RegisterEndpoints(rt *env.Runtime, s *domain.Store) {
|
|||
Add(rt, RoutePrefixPrivate, "users/{userID}", []string{"DELETE", "OPTIONS"}, nil, user.Delete)
|
||||
Add(rt, RoutePrefixPrivate, "users/sync", []string{"GET", "OPTIONS"}, nil, endpoint.SyncKeycloak)
|
||||
|
||||
Add(rt, RoutePrefixPrivate, "search", []string{"GET", "OPTIONS"}, nil, endpoint.SearchDocuments)
|
||||
Add(rt, RoutePrefixPrivate, "search", []string{"GET", "OPTIONS"}, nil, document.SearchDocuments)
|
||||
|
||||
Add(rt, RoutePrefixPrivate, "templates", []string{"POST", "OPTIONS"}, nil, endpoint.SaveAsTemplate)
|
||||
Add(rt, RoutePrefixPrivate, "templates", []string{"GET", "OPTIONS"}, nil, endpoint.GetSavedTemplates)
|
||||
|
@ -137,7 +139,7 @@ func RegisterEndpoints(rt *env.Runtime, s *domain.Store) {
|
|||
|
||||
Add(rt, RoutePrefixPrivate, "links/{folderID}/{documentID}/{pageID}", []string{"GET", "OPTIONS"}, nil, link.GetLinkCandidates)
|
||||
Add(rt, RoutePrefixPrivate, "links", []string{"GET", "OPTIONS"}, nil, link.SearchLinkCandidates)
|
||||
Add(rt, RoutePrefixPrivate, "documents/{documentID}/links", []string{"GET", "OPTIONS"}, nil, endpoint.GetDocumentLinks)
|
||||
Add(rt, RoutePrefixPrivate, "documents/{documentID}/links", []string{"GET", "OPTIONS"}, nil, document.DocumentLinks)
|
||||
|
||||
Add(rt, RoutePrefixPrivate, "global/smtp", []string{"GET", "OPTIONS"}, nil, setting.SMTP)
|
||||
Add(rt, RoutePrefixPrivate, "global/smtp", []string{"PUT", "OPTIONS"}, nil, setting.SetSMTP)
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue