Rebuild search index

README.md

@@ -52,9 +52,9 @@ Space view.
 
 ## Latest version
 
-[Community edition: v1.59.2](https://github.com/documize/community/releases)
+[Community edition: v1.60.0](https://github.com/documize/community/releases)
 
-[Enterprise edition: v1.61.2](https://documize.com/downloads)
+[Enterprise edition: v1.62.0](https://documize.com/downloads)
 
 ## OS support
 

domain/meta/endpoint.go

@@ -23,6 +23,7 @@ import (
 	"github.com/documize/community/domain"
 	"github.com/documize/community/domain/auth"
 	"github.com/documize/community/domain/organization"
+	indexer "github.com/documize/community/domain/search"
 	"github.com/documize/community/model/doc"
 	"github.com/documize/community/model/org"
 	"github.com/documize/community/model/space"
@@ -32,6 +33,7 @@ import (
 type Handler struct {
 	Runtime *env.Runtime
 	Store   *domain.Store
+	Indexer indexer.Indexer
 }
 
 // Meta provides org meta data based upon request domain (e.g. acme.documize.com).
@@ -176,7 +178,83 @@ func (h *Handler) Sitemap(w http.ResponseWriter, r *http.Request) {
 	response.WriteBytes(w, buffer.Bytes())
 }
 
+// Reindex indexes all documents and attachments.
+func (h *Handler) Reindex(w http.ResponseWriter, r *http.Request) {
+	ctx := domain.GetRequestContext(r)
+
+	if !ctx.Global {
+		response.WriteForbiddenError(w)
+		h.Runtime.Log.Info(fmt.Sprintf("%s attempted search reindex"))
+		return
+	}
+
+	go h.rebuildSearchIndex(ctx)
+
+	response.WriteEmpty(w)
+}
+
+// rebuildSearchIndex indexes all documents and attachments.
+func (h *Handler) rebuildSearchIndex(ctx domain.RequestContext) {
+	method := "meta.rebuildSearchIndex"
+
+	docs, err := h.Store.Meta.GetDocumentsID(ctx)
+	if err != nil {
+		h.Runtime.Log.Error(method, err)
+		return
+	}
+
+	h.Runtime.Log.Info(fmt.Sprintf("Search re-index started for %d documents", len(docs)))
+
+	for i := range docs {
+		d := docs[i]
+
+		pages, err := h.Store.Meta.GetDocumentPages(ctx, d)
+		if err != nil {
+			h.Runtime.Log.Error(method, err)
+			return
+		}
+
+		for j := range pages {
+			h.Indexer.IndexContent(ctx, pages[j])
+		}
+
+		// Log process every N documents.
+		if i%100 == 0 {
+			h.Runtime.Log.Info(fmt.Sprintf("Search re-indexed %d documents...", i))
+		}
+	}
+
+	h.Runtime.Log.Info(fmt.Sprintf("Search re-index finished for %d documents", len(docs)))
+}
+
+// SearchStatus returns state of search index
+func (h *Handler) SearchStatus(w http.ResponseWriter, r *http.Request) {
+	method := "meta.SearchStatus"
+	ctx := domain.GetRequestContext(r)
+
+	if !ctx.Global {
+		response.WriteForbiddenError(w)
+		h.Runtime.Log.Info(fmt.Sprintf("%s attempted get of search status"))
+		return
+	}
+
+	count, err := h.Store.Meta.SearchIndexCount(ctx)
+	if err != nil {
+		response.WriteServerError(w, method, err)
+		h.Runtime.Log.Error(method, err)
+		return
+	}
+
+	var ss = searchStatus{Entries: count}
+
+	response.WriteJSON(w, ss)
+}
+
 type sitemapItem struct {
 	URL  string
 	Date string
 }
+
+type searchStatus struct {
+	Entries int `json:"entries"`
+}

domain/meta/mysql/store.go

@@ -0,0 +1,73 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>.
+//
+// https://documize.com
+
+package mysql
+
+import (
+	"database/sql"
+
+	"github.com/documize/community/core/env"
+	"github.com/documize/community/domain"
+	"github.com/documize/community/model/page"
+	"github.com/pkg/errors"
+)
+
+// Scope provides data access to MySQL.
+type Scope struct {
+	Runtime *env.Runtime
+}
+
+// GetDocumentsID returns every document ID value stored.
+// The query runs at the instance level across all tenants.
+func (s Scope) GetDocumentsID(ctx domain.RequestContext) (documents []string, err error) {
+	err = s.Runtime.Db.Select(&documents, `SELECT refid FROM document WHERE lifecycle=1`)
+
+	if err == sql.ErrNoRows {
+		err = nil
+		documents = []string{}
+	}
+	if err != nil {
+		err = errors.Wrap(err, "failed to get instance document ID values")
+	}
+
+	return
+}
+
+// GetDocumentPages returns a slice containing all published page records for a given documentID, in presentation sequence.
+func (s Scope) GetDocumentPages(ctx domain.RequestContext, documentID string) (p []page.Page, err error) {
+	err = s.Runtime.Db.Select(&p,
+		`SELECT
+            a.id, a.refid, a.orgid, a.documentid, a.userid, a.contenttype,
+            a.pagetype, a.level, a.sequence, a.title, a.body, a.revisions,
+            a.blockid, a.status, a.relativeid, a.created, a.revised
+        FROM page a
+        WHERE a.documentid=? AND (a.status=0 OR ((a.status=4 OR a.status=2) AND a.relativeid=''))`,
+		documentID)
+
+	if err != nil {
+		err = errors.Wrap(err, "failed to get instance document pages")
+	}
+
+	return
+}
+
+// SearchIndexCount returns the numnber of index entries.
+func (s Scope) SearchIndexCount(ctx domain.RequestContext) (c int, err error) {
+	row := s.Runtime.Db.QueryRow("SELECT count(*) FROM search")
+
+	err = row.Scan(&c)
+	if err != nil {
+		err = errors.Wrap(err, "count search index entries")
+		c = 0
+	}
+
+	return
+}

domain/storer.go

@@ -42,6 +42,7 @@ type Store struct {
 	Document     DocumentStorer
 	Group        GroupStorer
 	Link         LinkStorer
+	Meta         MetaStorer
 	Organization OrganizationStorer
 	Page         PageStorer
 	Pin          PinStorer
@@ -282,3 +283,10 @@ type GroupStorer interface {
 	JoinGroup(ctx RequestContext, groupID, userID string) (err error)
 	LeaveGroup(ctx RequestContext, groupID, userID string) (err error)
 }
+
+// MetaStorer provide specialist methods for global administrators.
+type MetaStorer interface {
+	GetDocumentsID(ctx RequestContext) (documents []string, err error)
+	GetDocumentPages(ctx RequestContext, documentID string) (p []page.Page, err error)
+	SearchIndexCount(ctx RequestContext) (c int, err error)
+}

edition/boot/mysql.go

@@ -24,6 +24,7 @@ import (
 	doc "github.com/documize/community/domain/document/mysql"
 	group "github.com/documize/community/domain/group/mysql"
 	link "github.com/documize/community/domain/link/mysql"
+	meta "github.com/documize/community/domain/meta/mysql"
 	org "github.com/documize/community/domain/organization/mysql"
 	page "github.com/documize/community/domain/page/mysql"
 	permission "github.com/documize/community/domain/permission/mysql"
@@ -45,6 +46,7 @@ func StoreMySQL(r *env.Runtime, s *domain.Store) {
 	s.Document = doc.Scope{Runtime: r}
 	s.Group = group.Scope{Runtime: r}
 	s.Link = link.Scope{Runtime: r}
+	s.Meta = meta.Scope{Runtime: r}
 	s.Organization = org.Scope{Runtime: r}
 	s.Page = page.Scope{Runtime: r}
 	s.Pin = pin.Scope{Runtime: r}

edition/community.go

@@ -41,8 +41,8 @@ func main() {
 	// product details
 	rt.Product = env.ProdInfo{}
 	rt.Product.Major = "1"
-	rt.Product.Minor = "59"
-	rt.Product.Patch = "2"
+	rt.Product.Minor = "60"
+	rt.Product.Patch = "0"
 	rt.Product.Version = fmt.Sprintf("%s.%s.%s", rt.Product.Major, rt.Product.Minor, rt.Product.Patch)
 	rt.Product.Edition = "Community"
 	rt.Product.Title = fmt.Sprintf("%s Edition", rt.Product.Edition)

gui/app/components/customize/search-index.js

@@ -0,0 +1,25 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>.
+//
+// https://documize.com
+
+import { inject as service } from '@ember/service';
+import Component from '@ember/component';
+
+export default Component.extend({
+	appMeta: service(),
+	buttonLabel: 'Rebuild Search Index',
+
+	actions: {
+		reindex() {
+			this.set('buttonLabel', 'Rebuilding search index...')
+			this.get('reindex')();
+		}
+	}
+});

gui/app/components/toolbar/nav-bar.js

@@ -44,11 +44,12 @@ export default Component.extend(ModalMixin, {
 		});
 
 		let version = this.get('appMeta.version');
+		let edition = this.get('appMeta.edition').toLowerCase();
 
 		let self = this;
 		let cacheBuster = + new Date();
 		$.ajax({
-			url: `https://storage.googleapis.com/documize/news/${version}.html?cb=${cacheBuster}`,
+			url: `https://storage.googleapis.com/documize/news/${edition}/${version}.html?cb=${cacheBuster}`,
 			type: 'GET',
 			dataType: 'html',
 			success: function (response) {

gui/app/pods/customize/search/controller.js

@@ -0,0 +1,25 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>.
+//
+// https://documize.com
+
+import { inject as service } from '@ember/service';
+import Controller from '@ember/controller';
+
+export default Controller.extend({
+	global: service(),
+
+	actions: {
+		reindex() {
+			if(this.get('session.isGlobalAdmin')) {
+				return this.get('global').reindex();
+			}
+		}
+	}
+});

gui/app/pods/customize/search/route.js

@@ -0,0 +1,34 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>.
+//
+// https://documize.com
+
+import { inject as service } from '@ember/service';
+import AuthenticatedRouteMixin from 'ember-simple-auth/mixins/authenticated-route-mixin';
+import Route from '@ember/routing/route';
+
+export default Route.extend(AuthenticatedRouteMixin, {
+	appMeta: service(),
+	session: service(),
+	global: service(),
+
+	beforeModel() {
+		if (!this.get("session.isGlobalAdmin")) {
+			this.transitionTo('auth.login');
+		}
+	},
+
+	model() {
+		return this.get('global').searchStatus();
+	},
+
+	activate() {
+		this.get('browser').setTitle('Search');
+	}
+});

gui/app/pods/customize/search/template.hbs

@@ -0,0 +1 @@
+{{customize/search-index searchStatus=model reindex=(action 'reindex')}}

gui/app/pods/customize/template.hbs

@@ -20,6 +20,7 @@
 					{{#link-to 'customize.smtp' activeClass='selected' class="tab" tagName="li" }}SMTP{{/link-to}}
 					{{#link-to 'customize.license' activeClass='selected' class="tab" tagName="li" }}License{{/link-to}}
 					{{#link-to 'customize.auth' activeClass='selected' class="tab" tagName="li" }}Authentication{{/link-to}}
+					{{#link-to 'customize.search' activeClass='selected' class="tab" tagName="li" }}Search{{/link-to}}
 				{{/if}}
 				{{#link-to 'customize.archive' activeClass='selected' class="tab" tagName="li" }}Archive{{/link-to}}
 			</ul>

gui/app/router.js

@@ -85,6 +85,9 @@ export default Router.map(function () {
 			this.route('archive', {
 				path: 'archive'
 			});
+			this.route('search', {
+				path: 'search'
+			});
 		}
 	);
 

gui/app/services/global.js

@@ -70,7 +70,7 @@ export default Service.extend({
 				return response;
 			});
 		}
-	},	
+	},
 
 	// Saves auth config for Documize instance.
 	saveAuthConfig(config) {
@@ -93,4 +93,24 @@ export default Service.extend({
 			});
 		}
 	},
+
+	// Returns product license.
+	searchStatus() {
+		if (this.get('sessionService.isGlobalAdmin')) {
+			return this.get('ajax').request(`global/search/status`, {
+				method: 'GET',
+			}).then((response) => {
+				return response;
+			});
+		}
+	},
+
+	// Saves product license.
+	reindex() {
+		if (this.get('sessionService.isGlobalAdmin')) {
+			return this.get('ajax').request(`global/search/reindex`, {
+				method: 'POST',
+			});
+		}
+	}
 });

gui/app/templates/components/customize/search-index.hbs

@@ -0,0 +1,17 @@
+<div class="row">
+	<div class="col">
+		<div class="view-customize">
+			<h1 class="admin-heading">Search</h1>
+			<h2 class="sub-heading">There are currently {{searchStatus.entries}} search entries.</h2>
+		</div>
+	</div>
+</div>
+
+<div class="view-customize">
+	<form class="mt-5 ">
+		<div class="form-group">
+			<p>It can take up to 30 minutes to rebuild the search index across all documents and associated content.</p>
+			<div class="btn btn-danger mt-3" {{action 'reindex'}}>{{buttonLabel}}</div>
+		</div>
+	</form>
+</div>

gui/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "documize",
-	"version": "1.59.0",
+	"version": "1.60.0",
 	"description": "The Document IDE",
 	"private": true,
 	"repository": "",

server/routing/routes.go

@@ -48,7 +48,7 @@ func RegisterEndpoints(rt *env.Runtime, s *domain.Store) {
 	// DO NOT pass in per request context (that is done by auth middleware per request)
 	pin := pin.Handler{Runtime: rt, Store: s}
 	auth := auth.Handler{Runtime: rt, Store: s}
-	meta := meta.Handler{Runtime: rt, Store: s}
+	meta := meta.Handler{Runtime: rt, Store: s, Indexer: indexer}
 	user := user.Handler{Runtime: rt, Store: s}
 	link := link.Handler{Runtime: rt, Store: s}
 	page := page.Handler{Runtime: rt, Store: s, Indexer: indexer}
@@ -168,13 +168,6 @@ func RegisterEndpoints(rt *env.Runtime, s *domain.Store) {
 	AddPrivate(rt, "links", []string{"GET", "OPTIONS"}, nil, link.SearchLinkCandidates)
 	AddPrivate(rt, "documents/{documentID}/links", []string{"GET", "OPTIONS"}, nil, document.DocumentLinks)
 
-	AddPrivate(rt, "global/smtp", []string{"GET", "OPTIONS"}, nil, setting.SMTP)
-	AddPrivate(rt, "global/smtp", []string{"PUT", "OPTIONS"}, nil, setting.SetSMTP)
-	AddPrivate(rt, "global/license", []string{"GET", "OPTIONS"}, nil, setting.License)
-	AddPrivate(rt, "global/license", []string{"PUT", "OPTIONS"}, nil, setting.SetLicense)
-	AddPrivate(rt, "global/auth", []string{"GET", "OPTIONS"}, nil, setting.AuthConfig)
-	AddPrivate(rt, "global/auth", []string{"PUT", "OPTIONS"}, nil, setting.SetAuthConfig)
-
 	AddPrivate(rt, "pin/{userID}", []string{"POST", "OPTIONS"}, nil, pin.Add)
 	AddPrivate(rt, "pin/{userID}", []string{"GET", "OPTIONS"}, nil, pin.GetUserPins)
 	AddPrivate(rt, "pin/{userID}/sequence", []string{"POST", "OPTIONS"}, nil, pin.UpdatePinSequence)
@@ -203,6 +196,16 @@ func RegisterEndpoints(rt *env.Runtime, s *domain.Store) {
 	AddPrivate(rt, "fetch/document/{documentID}", []string{"GET", "OPTIONS"}, nil, document.FetchDocumentData)
 	AddPrivate(rt, "fetch/page/{documentID}", []string{"GET", "OPTIONS"}, nil, page.FetchPages)
 
+	// global admin routes
+	AddPrivate(rt, "global/smtp", []string{"GET", "OPTIONS"}, nil, setting.SMTP)
+	AddPrivate(rt, "global/smtp", []string{"PUT", "OPTIONS"}, nil, setting.SetSMTP)
+	AddPrivate(rt, "global/license", []string{"GET", "OPTIONS"}, nil, setting.License)
+	AddPrivate(rt, "global/license", []string{"PUT", "OPTIONS"}, nil, setting.SetLicense)
+	AddPrivate(rt, "global/auth", []string{"GET", "OPTIONS"}, nil, setting.AuthConfig)
+	AddPrivate(rt, "global/auth", []string{"PUT", "OPTIONS"}, nil, setting.SetAuthConfig)
+	AddPrivate(rt, "global/search/status", []string{"GET", "OPTIONS"}, nil, meta.SearchStatus)
+	AddPrivate(rt, "global/search/reindex", []string{"POST", "OPTIONS"}, nil, meta.Reindex)
+
 	Add(rt, RoutePrefixRoot, "robots.txt", []string{"GET", "OPTIONS"}, nil, meta.RobotsTxt)
 	Add(rt, RoutePrefixRoot, "sitemap.xml", []string{"GET", "OPTIONS"}, nil, meta.Sitemap)