diff --git a/domain/attachment/endpoint.go b/domain/attachment/endpoint.go index 04764d29..be5ac482 100644 --- a/domain/attachment/endpoint.go +++ b/domain/attachment/endpoint.go @@ -26,6 +26,7 @@ import ( "github.com/documize/community/core/uniqueid" "github.com/documize/community/domain" "github.com/documize/community/domain/document" + indexer "github.com/documize/community/domain/search" "github.com/documize/community/model/attachment" "github.com/documize/community/model/audit" uuid "github.com/nu7hatch/gouuid" @@ -35,6 +36,7 @@ import ( type Handler struct { Runtime *env.Runtime Store *domain.Store + Indexer indexer.Indexer } // Download is the end-point that responds to a request for a particular attachment @@ -155,6 +157,10 @@ func (h *Handler) Delete(w http.ResponseWriter, r *http.Request) { ctx.Transaction.Commit() + a, _ := h.Store.Attachment.GetAttachments(ctx, documentID) + d, _ := h.Store.Document.Get(ctx, documentID) + go h.Indexer.IndexDocument(ctx, d, a) + response.WriteEmpty(w) } @@ -226,5 +232,9 @@ func (h *Handler) Add(w http.ResponseWriter, r *http.Request) { ctx.Transaction.Commit() + all, _ := h.Store.Attachment.GetAttachments(ctx, documentID) + d, _ := h.Store.Document.Get(ctx, documentID) + go h.Indexer.IndexDocument(ctx, d, all) + response.WriteEmpty(w) } diff --git a/domain/conversion/conversion.go b/domain/conversion/conversion.go index 43f5586a..2d71a0a1 100644 --- a/domain/conversion/conversion.go +++ b/domain/conversion/conversion.go @@ -146,6 +146,9 @@ func (h *Handler) convert(w http.ResponseWriter, r *http.Request, job, folderID return } + a, _ := h.Store.Attachment.GetAttachments(ctx, nd.RefID) + go h.Indexer.IndexDocument(ctx, nd, a) + response.WriteJSON(w, nd) } diff --git a/domain/conversion/endpoint.go b/domain/conversion/endpoint.go index 4ee679ae..50f650b0 100644 --- a/domain/conversion/endpoint.go +++ b/domain/conversion/endpoint.go @@ -17,12 +17,14 @@ import ( api "github.com/documize/community/core/convapi" "github.com/documize/community/core/env" "github.com/documize/community/domain" + indexer "github.com/documize/community/domain/search" ) // Handler contains the runtime information such as logging and database. type Handler struct { Runtime *env.Runtime Store *domain.Store + Indexer indexer.Indexer } // UploadConvert is an endpoint to both upload and convert a document diff --git a/domain/document/endpoint.go b/domain/document/endpoint.go index 579d6643..f947f594 100644 --- a/domain/document/endpoint.go +++ b/domain/document/endpoint.go @@ -252,7 +252,8 @@ func (h *Handler) Update(w http.ResponseWriter, r *http.Request) { ctx.Transaction.Commit() - h.Indexer.UpdateDocument(ctx, d) + a, _ := h.Store.Attachment.GetAttachments(ctx, documentID) + go h.Indexer.IndexDocument(ctx, d, a) response.WriteEmpty(w) } @@ -316,7 +317,7 @@ func (h *Handler) Delete(w http.ResponseWriter, r *http.Request) { ctx.Transaction.Commit() - h.Indexer.DeleteDocument(ctx, documentID) + go h.Indexer.DeleteDocument(ctx, documentID) response.WriteEmpty(w) } diff --git a/domain/page/endpoint.go b/domain/page/endpoint.go index c17c88dd..5925e2a7 100644 --- a/domain/page/endpoint.go +++ b/domain/page/endpoint.go @@ -143,8 +143,7 @@ func (h *Handler) Add(w http.ResponseWriter, r *http.Request) { ctx.Transaction.Commit() np, _ := h.Store.Page.Get(ctx, pageID) - - h.Indexer.Add(ctx, np, pageID) + go h.Indexer.IndexContent(ctx, np) response.WriteJSON(w, np) } @@ -338,7 +337,7 @@ func (h *Handler) Delete(w http.ResponseWriter, r *http.Request) { h.Store.Audit.Record(ctx, audit.EventTypeSectionDelete) - h.Indexer.Delete(ctx, documentID, pageID) + go h.Indexer.DeleteContent(ctx, pageID) h.Store.Link.DeleteSourcePageLinks(ctx, pageID) @@ -421,7 +420,7 @@ func (h *Handler) DeletePages(w http.ResponseWriter, r *http.Request) { return } - h.Indexer.Delete(ctx, documentID, page.PageID) + go h.Indexer.DeleteContent(ctx, page.PageID) h.Store.Link.DeleteSourcePageLinks(ctx, page.PageID) @@ -590,7 +589,7 @@ func (h *Handler) Update(w http.ResponseWriter, r *http.Request) { ctx.Transaction.Commit() - h.Indexer.Update(ctx, model.Page) + go h.Indexer.IndexContent(ctx, model.Page) updatedPage, err := h.Store.Page.Get(ctx, pageID) @@ -649,8 +648,6 @@ func (h *Handler) ChangePageSequence(w http.ResponseWriter, r *http.Request) { h.Runtime.Log.Error(method, err) return } - - h.Indexer.UpdateSequence(ctx, documentID, p.PageID, p.Sequence) } h.Store.Audit.Record(ctx, audit.EventTypeSectionResequence) @@ -712,8 +709,6 @@ func (h *Handler) ChangePageLevel(w http.ResponseWriter, r *http.Request) { h.Runtime.Log.Error(method, err) return } - - h.Indexer.UpdateLevel(ctx, documentID, p.PageID, p.Level) } h.Store.Audit.Record(ctx, audit.EventTypeSectionResequence) diff --git a/domain/search/mysql/store.go b/domain/search/mysql/store.go index eb1d2ce3..3f6ae5ae 100644 --- a/domain/search/mysql/store.go +++ b/domain/search/mysql/store.go @@ -15,14 +15,13 @@ import ( "database/sql" "fmt" "strings" - "time" "github.com/documize/community/core/env" "github.com/documize/community/core/streamutil" "github.com/documize/community/core/stringutil" "github.com/documize/community/domain" - "github.com/documize/community/domain/store/mysql" - "github.com/documize/community/model" + "github.com/documize/community/model/attachment" + "github.com/documize/community/model/doc" "github.com/documize/community/model/page" "github.com/documize/community/model/search" "github.com/jmoiron/sqlx" @@ -34,210 +33,160 @@ type Scope struct { Runtime *env.Runtime } -// Add search entry (legacy name: searchAdd). -func (s Scope) Add(ctx domain.RequestContext, page page.Page) (err error) { - id := page.RefID - - // translate the html into text for the search - nonHTML, err := stringutil.HTML(page.Body).Text(false) +// IndexDocument adds search index entries for document inserting title, tags and attachments as +// searchable items. Any existing document entries are removed. +func (s Scope) IndexDocument(ctx domain.RequestContext, doc doc.Document, a []attachment.Attachment) (err error) { + // remove previous search entries + var stmt1 *sqlx.Stmt + stmt1, err = ctx.Transaction.Preparex("DELETE FROM search WHERE orgid=? AND documentid=? AND (itemtype='doc' OR itemtype='file' OR itemtype='tag')") + defer streamutil.Close(stmt1) if err != nil { - errors.Wrap(err, "search decode body") + err = errors.Wrap(err, "prepare delete document index entries") return } - // insert into the search table, getting the document title along the way - var stmt *sqlx.Stmt - stmt, err = ctx.Transaction.Preparex( - "INSERT INTO search (id, orgid, documentid, level, sequence, documenttitle, slug, pagetitle, body, created, revised) " + - " SELECT page.refid,page.orgid,document.refid,page.level,page.sequence,document.title,document.slug,page.title,?,page.created,page.revised " + - " FROM document,page WHERE page.refid=? AND document.refid=page.documentid") - - defer streamutil.Close(stmt) - + _, err = stmt1.Exec(ctx.OrgID, doc.RefID) if err != nil { - err = errors.Wrap(err, "prepare search insert") + err = errors.Wrap(err, "execute delete document index entries") return } - _, err = stmt.Exec(nonHTML, id) - - if err != nil { - err = errors.Wrap(err, "execute search insert") - return - } - - return nil -} - -// Update search entry (legacy name: searchUpdate). -func (s Scope) Update(ctx domain.RequestContext, page page.Page) (err error) { - // translate the html into text for the search - nonHTML, err := stringutil.HTML(page.Body).Text(false) - if err != nil { - err = errors.Wrap(err, "search decode body") - return - } - - su, err := ctx.Transaction.Preparex("UPDATE search SET pagetitle=?,body=?,sequence=?,level=?,revised=? WHERE id=?") - defer streamutil.Close(su) - - if err != nil { - err = errors.Wrap(err, "prepare search update") - return err - } - - _, err = su.Exec(page.Title, nonHTML, page.Sequence, page.Level, page.Revised, page.RefID) - - if err != nil { - err = errors.Wrap(err, "execute search update") - return - } - - return nil -} - -// UpdateDocument search entries for document (legacy name: searchUpdateDocument). -func (s Scope) UpdateDocument(ctx domain.RequestContext, page page.Page) (err error) { - stmt, err := ctx.Transaction.Preparex("UPDATE search SET documenttitle=?, slug=?, revised=? WHERE documentid=?") - defer streamutil.Close(stmt) - - if err != nil { - err = errors.Wrap(err, "prepare search document update") - return err - } - - _, err = stmt.Exec(page.Title, page.Body, time.Now().UTC(), page.DocumentID) - - if err != nil { - err = errors.Wrap(err, "execute search document update") - return err - } - - return nil -} - -// DeleteDocument removes document search entries (legacy name: searchDeleteDocument) -func (s Scope) DeleteDocument(ctx domain.RequestContext, page page.Page) (err error) { - var bm = mysql.BaseQuery{} - - _, err = bm.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE from search WHERE documentid='%s'", page.DocumentID)) - - if err != nil { - err = errors.Wrap(err, "delete document search entries") - } - - return nil -} - -// Rebuild ... (legacy name: searchRebuild) -func (s Scope) Rebuild(ctx domain.RequestContext, p page.Page) (err error) { - var bm = mysql.BaseQuery{} - - _, err = bm.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE from search WHERE documentid='%s'", p.DocumentID)) - if err != nil { - err = errors.Wrap(err, err.Error()) - return err - } - - var pages []struct{ ID string } - - stmt2, err := ctx.Transaction.Preparex("SELECT refid as id FROM page WHERE documentid=? ") + // insert doc title + var stmt2 *sqlx.Stmt + stmt2, err = ctx.Transaction.Preparex("INSERT INTO search (orgid, documentid, itemid, itemtype, content) VALUES (?, ?, ?, ?, ?)") defer streamutil.Close(stmt2) - if err != nil { - err = errors.Wrap(err, err.Error()) - return err + err = errors.Wrap(err, "prepare insert document title entry") + return } - err = stmt2.Select(&pages, p.DocumentID) + _, err = stmt2.Exec(ctx.OrgID, doc.RefID, "", "doc", doc.Title) if err != nil { - err = errors.Wrap(err, err.Error()) - return err + err = errors.Wrap(err, "execute insert document title entry") + return } - if len(pages) > 0 { - for _, pg := range pages { - err = s.Add(ctx, page.Page{BaseEntity: model.BaseEntity{RefID: pg.ID}}) - if err != nil { - err = errors.Wrap(err, err.Error()) - return err - } + // insert doc tags + tags := strings.Split(doc.Tags, "#") + for _, t := range tags { + if len(t) == 0 { + continue } - // rebuild doc-level tags & excerpts - // get the 0'th page data and rewrite it - - target := page.Page{} - - stmt1, err := ctx.Transaction.Preparex("SELECT * FROM page WHERE refid=?") - defer streamutil.Close(stmt1) - + var stmt3 *sqlx.Stmt + stmt3, err = ctx.Transaction.Preparex("INSERT INTO search (orgid, documentid, itemid, itemtype, content) VALUES (?, ?, ?, ?, ?)") + defer streamutil.Close(stmt3) if err != nil { - err = errors.Wrap(err, err.Error()) - return err + err = errors.Wrap(err, "prepare insert document tag entry") + return } - err = stmt1.Get(&target, pages[0].ID) + _, err = stmt3.Exec(ctx.OrgID, doc.RefID, "", "tag", t) if err != nil { - err = errors.Wrap(err, err.Error()) - return err - } - - err = s.Update(ctx, target) // to rebuild the document-level tags + excerpt - if err != nil { - err = errors.Wrap(err, err.Error()) - return err + err = errors.Wrap(err, "execute insert document tag entry") + return } } - return + for _, file := range a { + var stmt4 *sqlx.Stmt + stmt4, err = ctx.Transaction.Preparex("INSERT INTO search (orgid, documentid, itemid, itemtype, content) VALUES (?, ?, ?, ?, ?)") + defer streamutil.Close(stmt4) + if err != nil { + err = errors.Wrap(err, "prepare insert document file entry") + return + } + + _, err = stmt4.Exec(ctx.OrgID, doc.RefID, file.RefID, "file", file.Filename) + if err != nil { + err = errors.Wrap(err, "execute insert document file entry") + return + } + } + + return nil } -// UpdateSequence ... (legacy name: searchUpdateSequence) -func (s Scope) UpdateSequence(ctx domain.RequestContext, page page.Page) (err error) { - supdate, err := ctx.Transaction.Preparex("UPDATE search SET sequence=?,revised=? WHERE id=?") - defer streamutil.Close(supdate) - +// DeleteDocument removes all search entries for document. +func (s Scope) DeleteDocument(ctx domain.RequestContext, ID string) (err error) { + // remove all search entries + var stmt1 *sqlx.Stmt + stmt1, err = ctx.Transaction.Preparex("DELETE FROM search WHERE orgid=? AND documentid=?") + defer streamutil.Close(stmt1) if err != nil { - err = errors.Wrap(err, "prepare search update sequence") - return err + err = errors.Wrap(err, "prepare delete document entries") + return } - _, err = supdate.Exec(page.Sequence, time.Now().UTC(), page.RefID) + _, err = stmt1.Exec(ctx.OrgID, ID) if err != nil { - err = errors.Wrap(err, "execute search update sequence") + err = errors.Wrap(err, "execute delete document entries") return } return } -// UpdateLevel ... legacy name: searchUpdateLevel) -func (s Scope) UpdateLevel(ctx domain.RequestContext, page page.Page) (err error) { - pageID := page.RefID - level := page.Level - - supdate, err := ctx.Transaction.Preparex("UPDATE search SET level=?,revised=? WHERE id=?") - defer streamutil.Close(supdate) - +// IndexContent adds search index entry for document context. +// Any existing document entries are removed. +func (s Scope) IndexContent(ctx domain.RequestContext, p page.Page) (err error) { + // remove previous search entries + var stmt1 *sqlx.Stmt + stmt1, err = ctx.Transaction.Preparex("DELETE FROM search WHERE orgid=? AND documentid=? AND itemid=? AND itemtype='page' )") + defer streamutil.Close(stmt1) if err != nil { - err = errors.Wrap(err, "prepare search update level") - return err - } - - _, err = supdate.Exec(level, time.Now().UTC(), pageID) - if err != nil { - err = errors.Wrap(err, "execute search update level") + err = errors.Wrap(err, "prepare delete document content entry") return } - return + _, err = stmt1.Exec(ctx.OrgID, p.DocumentID) + if err != nil { + err = errors.Wrap(err, "execute delete document content entry") + return + } + + // insert doc title + var stmt2 *sqlx.Stmt + stmt2, err = ctx.Transaction.Preparex("INSERT INTO search (orgid, documentid, itemid, itemtype, content) VALUES (?, ?, ?, ?, ?)") + defer streamutil.Close(stmt2) + if err != nil { + err = errors.Wrap(err, "prepare insert document content entry") + return + } + + // prepare content + content, err := stringutil.HTML(p.Body).Text(false) + if err != nil { + err = errors.Wrap(err, "search strip HTMl failed") + return + } + content = strings.TrimSpace(content) + + _, err = stmt2.Exec(ctx.OrgID, p.DocumentID, p.RefID, "page", content) + if err != nil { + err = errors.Wrap(err, "execute insert document content entry") + return + } + + return nil } -// Delete ... (legacy name: searchDelete). -func (s Scope) Delete(ctx domain.RequestContext, page page.Page) (err error) { - var bm = mysql.BaseQuery{} - _, err = bm.DeleteConstrainedWithID(ctx.Transaction, "search", ctx.OrgID, page.RefID) +// DeleteContent removes all search entries for specific document content. +func (s Scope) DeleteContent(ctx domain.RequestContext, pageID string) (err error) { + // remove all search entries + var stmt1 *sqlx.Stmt + stmt1, err = ctx.Transaction.Preparex("DELETE FROM search WHERE orgid=? AND itemid=? AND itemtype='page'") + defer streamutil.Close(stmt1) + if err != nil { + err = errors.Wrap(err, "prepare delete document content entry") + return + } + + _, err = stmt1.Exec(ctx.OrgID, pageID) + if err != nil { + err = errors.Wrap(err, "execute delete document content entry") + return + } return } diff --git a/domain/search/queue.go b/domain/search/queue.go index 4a2c5aff..d442b348 100644 --- a/domain/search/queue.go +++ b/domain/search/queue.go @@ -12,131 +12,21 @@ package search import ( - "errors" - "fmt" - "sync" - "github.com/documize/community/core/env" "github.com/documize/community/domain" - "github.com/documize/community/model/page" ) -// Indexer type provides the datastructure for the queues of activity to be serialized through a single background goroutine. -// NOTE if the queue becomes full, the system will trigger the rebuilding entire files in order to clear the backlog. +// Indexer documents! type Indexer struct { - queue chan queueEntry - rebuild map[string]bool - rebuildLock sync.RWMutex - givenWarning bool - runtime *env.Runtime - store *domain.Store + runtime *env.Runtime + store *domain.Store } -type queueEntry struct { - action func(domain.RequestContext, page.Page) error - isRebuild bool - page.Page - ctx domain.RequestContext -} - -const searchQueueLength = 2048 // NOTE the largest 15Mb docx in the test set generates 2142 queue entries, but the queue is constantly emptied - // NewIndexer provides background search indexer func NewIndexer(rt *env.Runtime, s *domain.Store) (i Indexer) { i = Indexer{} - i.queue = make(chan queueEntry, searchQueueLength) // provide some decoupling - i.rebuild = make(map[string]bool) i.runtime = rt i.store = s - go i.processQueue() - return } - -// processQueue is run as a goroutine, it processes the queue of search index update requests. -func (m *Indexer) processQueue() { - for { - //fmt.Println("DEBUG queue length=", len(Searches.queue)) - if len(m.queue) <= searchQueueLength/20 { // on a busy server, the queue may never get to zero - so use 5% - m.rebuildLock.Lock() - for docid := range m.rebuild { - m.queue <- queueEntry{ - action: m.store.Search.Rebuild, - isRebuild: true, - Page: page.Page{DocumentID: docid}, - } - delete(m.rebuild, docid) - } - m.rebuildLock.Unlock() - } - - qe := <-m.queue - doit := true - - if len(qe.DocumentID) > 0 { - m.rebuildLock.RLock() - if m.rebuild[qe.DocumentID] { - doit = false // don't execute an action on a document queued to be rebuilt - } - m.rebuildLock.RUnlock() - } - - if doit { - tx, err := m.runtime.Db.Beginx() - if err != nil { - } else { - ctx := qe.ctx - ctx.Transaction = tx - err = qe.action(ctx, qe.Page) - if err != nil { - tx.Rollback() - // This action has failed, so re-build indexes for the entire document, - // provided it was not a re-build command that failed and we know the documentId. - if !qe.isRebuild && len(qe.DocumentID) > 0 { - m.rebuildLock.Lock() - m.rebuild[qe.DocumentID] = true - m.rebuildLock.Unlock() - } - } else { - tx.Commit() - } - } - } - } -} - -func (m *Indexer) addQueue(qe queueEntry) error { - lsq := len(m.queue) - - if lsq >= (searchQueueLength - 1) { - if qe.DocumentID != "" { - m.rebuildLock.Lock() - if !m.rebuild[qe.DocumentID] { - m.runtime.Log.Info(fmt.Sprintf("WARNING: Search Queue Has No Space! Marked rebuild index for document id %s", qe.DocumentID)) - } - m.rebuild[qe.DocumentID] = true - m.rebuildLock.Unlock() - } else { - m.runtime.Log.Error("addQueue", errors.New("WARNING: Search Queue Has No Space! But unable to index unknown document id")) - } - - return nil - } - - if lsq > ((8 * searchQueueLength) / 10) { - if !m.givenWarning { - m.runtime.Log.Info(fmt.Sprintf("WARNING: Searches.queue length %d exceeds 80%% of capacity", lsq)) - m.givenWarning = true - } - } else { - if m.givenWarning { - m.runtime.Log.Info(fmt.Sprintf("INFO: Searches.queue length %d now below 80%% of capacity", lsq)) - m.givenWarning = false - } - } - - m.queue <- qe - - return nil -} diff --git a/domain/search/search.go b/domain/search/search.go index ac17683a..c1684db6 100644 --- a/domain/search/search.go +++ b/domain/search/search.go @@ -13,102 +13,93 @@ package search import ( "github.com/documize/community/domain" - "github.com/documize/community/model" + "github.com/documize/community/model/attachment" "github.com/documize/community/model/doc" "github.com/documize/community/model/page" ) -// Add should be called when a new page is added to a document. -func (m *Indexer) Add(ctx domain.RequestContext, page page.Page, id string) (err error) { - page.RefID = id +// IndexDocument adds search indesd entries for document inserting title, tags and attachments as +// searchable items. Any existing document entries are removed. +func (m *Indexer) IndexDocument(ctx domain.RequestContext, d doc.Document, a []attachment.Attachment) { + method := "search.IndexDocument" + var err error - err = m.addQueue(queueEntry{ - action: m.store.Search.Add, - Page: page, - ctx: ctx, - }) - - return -} - -// Update should be called after a page record has been updated. -func (m *Indexer) Update(ctx domain.RequestContext, page page.Page) (err error) { - err = m.addQueue(queueEntry{ - action: m.store.Search.Update, - Page: page, - ctx: ctx, - }) - - return -} - -// UpdateDocument should be called after a document record has been updated. -func (m *Indexer) UpdateDocument(ctx domain.RequestContext, document doc.Document) (err error) { - err = m.addQueue(queueEntry{ - action: m.store.Search.UpdateDocument, - Page: page.Page{ - DocumentID: document.RefID, - Title: document.Title, - Body: document.Slug, // NOTE body==slug in this context - }, - ctx: ctx, - }) - - return -} - -// DeleteDocument should be called after a document record has been deleted. -func (m *Indexer) DeleteDocument(ctx domain.RequestContext, documentID string) (err error) { - if len(documentID) > 0 { - m.queue <- queueEntry{ - action: m.store.Search.DeleteDocument, - Page: page.Page{DocumentID: documentID}, - ctx: ctx, - } + ctx.Transaction, err = m.runtime.Db.Beginx() + if err != nil { + m.runtime.Log.Error(method, err) + return } - return + + err = m.store.Search.IndexDocument(ctx, d, a) + if err != nil { + ctx.Transaction.Rollback() + m.runtime.Log.Error(method, err) + return + } + + ctx.Transaction.Commit() } -// UpdateSequence should be called after a page record has been resequenced. -func (m *Indexer) UpdateSequence(ctx domain.RequestContext, documentID, pageID string, sequence float64) (err error) { - err = m.addQueue(queueEntry{ - action: m.store.Search.UpdateSequence, - Page: page.Page{ - BaseEntity: model.BaseEntity{RefID: pageID}, - Sequence: sequence, - DocumentID: documentID, - }, - ctx: ctx, - }) +// DeleteDocument removes all search entries for document. +func (m *Indexer) DeleteDocument(ctx domain.RequestContext, ID string) { + method := "search.DeleteDocument" + var err error - return + ctx.Transaction, err = m.runtime.Db.Beginx() + if err != nil { + m.runtime.Log.Error(method, err) + return + } + + err = m.store.Search.DeleteDocument(ctx, ID) + if err != nil { + ctx.Transaction.Rollback() + m.runtime.Log.Error(method, err) + return + } + + ctx.Transaction.Commit() } -// UpdateLevel should be called after the level of a page has been changed. -func (m *Indexer) UpdateLevel(ctx domain.RequestContext, documentID, pageID string, level int) (err error) { - err = m.addQueue(queueEntry{ - action: m.store.Search.UpdateLevel, - Page: page.Page{ - BaseEntity: model.BaseEntity{RefID: pageID}, - Level: uint64(level), - DocumentID: documentID, - }, - ctx: ctx, - }) +// IndexContent adds search index entry for document context. +// Any existing document entries are removed. +func (m *Indexer) IndexContent(ctx domain.RequestContext, p page.Page) { + method := "search.IndexContent" + var err error - return + ctx.Transaction, err = m.runtime.Db.Beginx() + if err != nil { + m.runtime.Log.Error(method, err) + return + } + + err = m.store.Search.IndexContent(ctx, p) + if err != nil { + ctx.Transaction.Rollback() + m.runtime.Log.Error(method, err) + return + } + + ctx.Transaction.Commit() } -// Delete should be called after a page has been deleted. -func (m *Indexer) Delete(ctx domain.RequestContext, documentID, pageID string) (rows int64, err error) { - err = m.addQueue(queueEntry{ - action: m.store.Search.Delete, - Page: page.Page{ - BaseEntity: model.BaseEntity{RefID: pageID}, - DocumentID: documentID, - }, - ctx: ctx, - }) +// DeleteContent removes all search entries for specific document content. +func (m *Indexer) DeleteContent(ctx domain.RequestContext, pageID string) { + method := "search.DeleteContent" + var err error - return -} + ctx.Transaction, err = m.runtime.Db.Beginx() + if err != nil { + m.runtime.Log.Error(method, err) + return + } + + err = m.store.Search.DeleteContent(ctx, pageID) + if err != nil { + ctx.Transaction.Rollback() + m.runtime.Log.Error(method, err) + return + } + + ctx.Transaction.Commit() +} \ No newline at end of file diff --git a/domain/storer.go b/domain/storer.go index 7c5bc8c6..fca30dc1 100644 --- a/domain/storer.go +++ b/domain/storer.go @@ -180,26 +180,19 @@ type ActivityStorer interface { // SearchStorer defines required methods for persisting search queries type SearchStorer interface { - Add(ctx RequestContext, page page.Page) (err error) - Update(ctx RequestContext, page page.Page) (err error) - UpdateDocument(ctx RequestContext, page page.Page) (err error) - DeleteDocument(ctx RequestContext, page page.Page) (err error) - Rebuild(ctx RequestContext, p page.Page) (err error) - UpdateSequence(ctx RequestContext, page page.Page) (err error) - UpdateLevel(ctx RequestContext, page page.Page) (err error) - Delete(ctx RequestContext, page page.Page) (err error) - Documents(ctx RequestContext, options search.QueryOptions) (results []search.QueryResult, err error) + IndexDocument(ctx RequestContext, doc doc.Document, a []attachment.Attachment) (err error) + DeleteDocument(ctx RequestContext, ID string) (err error) + IndexContent(ctx RequestContext, p page.Page) (err error) + DeleteContent(ctx RequestContext, pageID string) (err error) + Documents(ctx RequestContext, q search.QueryOptions) (results []search.QueryResult, err error) } // Indexer defines required methods for managing search indexing process type Indexer interface { - Add(ctx RequestContext, page page.Page, id string) (err error) - Update(ctx RequestContext, page page.Page) (err error) - UpdateDocument(ctx RequestContext, page page.Page) (err error) - DeleteDocument(ctx RequestContext, documentID string) (err error) - UpdateSequence(ctx RequestContext, documentID, pageID string, sequence float64) (err error) - UpdateLevel(ctx RequestContext, documentID, pageID string, level int) (err error) - Delete(ctx RequestContext, documentID, pageID string) (err error) + IndexDocument(ctx RequestContext, d doc.Document, a []attachment.Attachment) + DeleteDocument(ctx RequestContext, ID string) + IndexContent(ctx RequestContext, p page.Page) + DeleteContent(ctx RequestContext, pageID string) } // BlockStorer defines required methods for persisting reusable content blocks diff --git a/domain/template/endpoint.go b/domain/template/endpoint.go index 728a9483..8ee2209a 100644 --- a/domain/template/endpoint.go +++ b/domain/template/endpoint.go @@ -28,6 +28,7 @@ import ( "github.com/documize/community/core/uniqueid" "github.com/documize/community/domain" "github.com/documize/community/domain/document" + indexer "github.com/documize/community/domain/search" "github.com/documize/community/model/attachment" "github.com/documize/community/model/audit" "github.com/documize/community/model/doc" @@ -40,6 +41,7 @@ import ( type Handler struct { Runtime *env.Runtime Store *domain.Store + Indexer indexer.Indexer } // SavedList returns all templates saved by the user @@ -363,5 +365,8 @@ func (h *Handler) Use(w http.ResponseWriter, r *http.Request) { event.Handler().Publish(string(event.TypeAddDocument), nd.Title) + a, _ := h.Store.Attachment.GetAttachments(ctx, documentID) + go h.Indexer.IndexDocument(ctx, nd, a) + response.WriteJSON(w, nd) } diff --git a/gui/app/components/search/search-results.js b/gui/app/components/search/search-results.js index ef4d3bc4..812440be 100644 --- a/gui/app/components/search/search-results.js +++ b/gui/app/components/search/search-results.js @@ -17,15 +17,22 @@ export default Ember.Component.extend({ didReceiveAttrs() { let docs = this.get('results'); + let duped = []; let phrase = 'Nothing found'; if (docs.length > 0) { + duped = _.uniq(docs, function (item) { + return item.documentId; + }); + let references = docs.length === 1 ? "reference" : "references"; + let docLabel = duped.length === 1 ? "document" : "documents"; let i = docs.length; - phrase = `${i} ${references}`; + let j = duped.length; + phrase = `${i} ${references} across ${j} ${docLabel}`; } this.set('resultPhrase', phrase); - this.set('documents', docs); + this.set('documents', duped); } }); diff --git a/gui/app/pods/search/controller.js b/gui/app/pods/search/controller.js index 38b0a26a..8a2dbed5 100644 --- a/gui/app/pods/search/controller.js +++ b/gui/app/pods/search/controller.js @@ -17,8 +17,8 @@ export default Ember.Controller.extend({ results: [], matchDoc: true, matchContent: true, - matchFile: true, - matchTag: true, + matchFile: false, + matchTag: false, onKeywordChange: function () { Ember.run.debounce(this, this.fetch, 750); diff --git a/gui/app/pods/search/template.hbs b/gui/app/pods/search/template.hbs index afafd151..44c47f48 100644 --- a/gui/app/pods/search/template.hbs +++ b/gui/app/pods/search/template.hbs @@ -12,9 +12,9 @@
{{focus-input type="text" value=filter placeholder='type search phrase'}} {{#ui/ui-checkbox selected=matchDoc}}document name{{/ui/ui-checkbox}} - {{#ui/ui-checkbox selected=matchContent}}document content{{/ui/ui-checkbox}} + {{#ui/ui-checkbox selected=matchContent}}content{{/ui/ui-checkbox}} + {{#ui/ui-checkbox selected=matchTag}}tag{{/ui/ui-checkbox}} {{#ui/ui-checkbox selected=matchFile}}attachment name{{/ui/ui-checkbox}} - {{#ui/ui-checkbox selected=matchTag}}tag name{{/ui/ui-checkbox}}

a OR b

diff --git a/model/search/search.go b/model/search/search.go index f898564b..14e458ec 100644 --- a/model/search/search.go +++ b/model/search/search.go @@ -11,25 +11,6 @@ package search -import ( - "time" -) - -// Search holds raw search results. -type Search struct { - ID string `json:"id"` - Created time.Time `json:"created"` - Revised time.Time `json:"revised"` - OrgID string - DocumentID string - Level uint64 - Sequence float64 - DocumentTitle string - Slug string - PageTitle string - Body string -} - // QueryOptions defines how we search. type QueryOptions struct { Keywords string `json:"keywords"` diff --git a/server/routing/routes.go b/server/routing/routes.go index 74d92bf2..e182c8f1 100644 --- a/server/routing/routes.go +++ b/server/routing/routes.go @@ -54,10 +54,10 @@ func RegisterEndpoints(rt *env.Runtime, s *domain.Store) { section := section.Handler{Runtime: rt, Store: s} setting := setting.Handler{Runtime: rt, Store: s} keycloak := keycloak.Handler{Runtime: rt, Store: s} - template := template.Handler{Runtime: rt, Store: s} + template := template.Handler{Runtime: rt, Store: s, Indexer: indexer} document := document.Handler{Runtime: rt, Store: s, Indexer: indexer} - attachment := attachment.Handler{Runtime: rt, Store: s} - conversion := conversion.Handler{Runtime: rt, Store: s} + attachment := attachment.Handler{Runtime: rt, Store: s, Indexer: indexer} + conversion := conversion.Handler{Runtime: rt, Store: s, Indexer: indexer} organization := organization.Handler{Runtime: rt, Store: s} //**************************************************