From 2c5f73a486d415de19a4eb5c4c59abc67b062a5d Mon Sep 17 00:00:00 2001 From: Harvey Kandola Date: Tue, 15 Aug 2017 14:15:31 +0100 Subject: [PATCH] new search options and schema --- core/database/readme.md | 7 +- core/database/scripts/autobuild/db_00015.sql | 39 ++++ domain/document/endpoint.go | 22 ++- domain/search/mysql/store.go | 172 +++++++++++++----- domain/storer.go | 2 +- gui/app/components/search/search-results.js | 33 +--- gui/app/pods/search/controller.js | 44 ++++- gui/app/pods/search/template.hbs | 15 +- gui/app/services/search.js | 12 +- gui/app/styles/view/page-search.scss | 4 + .../components/search/search-results.hbs | 12 +- jsconfig.json | 2 +- model/search/search.go | 35 ++-- server/routing/routes.go | 2 +- 14 files changed, 283 insertions(+), 118 deletions(-) create mode 100644 core/database/scripts/autobuild/db_00015.sql diff --git a/core/database/readme.md b/core/database/readme.md index b27054c5..5c5737fe 100644 --- a/core/database/readme.md +++ b/core/database/readme.md @@ -4,7 +4,6 @@ 1. Remove audit table 2. Remove document.layout field ? - ## MYSQL ENCODING https://stackoverflow.com/questions/37307146/difference-between-utf8mb4-unicode-ci-and-utf8mb4-unicode-520-ci-collations-in-m @@ -13,7 +12,7 @@ https://mathiasbynens.be/notes/mysql-utf8mb4 https://medium.com/@adamhooper/in-mysql-never-use-utf8-use-utf8mb4-11761243e434 - +## MIGRATE ENCODING ALTER DATABASE documize CHARACTER SET utf8mb4 COLLATE utf8mb4_bin; ALTER TABLE account CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin; @@ -38,7 +37,3 @@ ALTER TABLE useraction CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin; ALTER TABLE useractivity CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin; ALTER TABLE userconfig CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin; ALTER TABLE userevent CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin; - - -documenttitle, pagetitle, body -CHARACTER SET utf8mb4 COLLATE utf8mb4 \ No newline at end of file diff --git a/core/database/scripts/autobuild/db_00015.sql b/core/database/scripts/autobuild/db_00015.sql new file mode 100644 index 00000000..9bc9b53e --- /dev/null +++ b/core/database/scripts/autobuild/db_00015.sql @@ -0,0 +1,39 @@ +/* community edition */ + +RENAME TABLE search TO search_old; + +DROP TABLE IF EXISTS `search`; + +CREATE TABLE IF NOT EXISTS `search` ( + `id` INT UNSIGNED NOT NULL AUTO_INCREMENT, + `orgid` CHAR(16) NOT NULL COLLATE utf8_bin, + `documentid` CHAR(16) NOT NULL COLLATE utf8_bin, + `itemid` CHAR(16) NOT NULL DEFAULT '' COLLATE utf8_bin, + `itemtype` VARCHAR(10) NOT NULL, + `content` LONGTEXT, + `created` TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE INDEX `idx_search_id` (`id` ASC), + INDEX `idx_search_orgid` (`orgid` ASC), + INDEX `idx_search_documentid` (`documentid` ASC), + FULLTEXT INDEX `idx_search_content` (`content`)) +DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci +ENGINE = MyISAM; + +-- migrate page content +INSERT INTO search (orgid, documentid, itemid, itemtype, content) SELECT orgid, documentid, id AS itemid, 'page' AS itemtype, TRIM(body) AS content FROM search_old; + +-- index document title +INSERT INTO search (orgid, documentid, itemid, itemtype, content) SELECT orgid, refid AS documentid, '' AS itemid, 'doc' AS itemtype, TRIM(title) AS content FROM document; + +-- index attachment name +INSERT INTO search (orgid, documentid, itemid, itemtype, content) SELECT orgid, documentid, refid AS itemid, 'file' AS itemtype, TRIM(filename) AS content FROM attachment; + +-- insert tag 1 +insert into search (orgid, documentid, itemid, itemtype, content) SELECT orgid, refid as documentid, '' as itemid, 'tag' as itemtype, TRIM(REPLACE(SUBSTRING_INDEX(tags, '#', 2), '#', '')) AS content FROM document WHERE tags != ''; + +-- insert tag 2 +insert into search (orgid, documentid, itemid, itemtype, content) SELECT orgid, refid as documentid, '' as itemid, 'tag' as itemtype, IF((LENGTH(tags) - LENGTH(REPLACE(tags, '#', '')) - 1) > 1, SUBSTRING_INDEX(SUBSTRING_INDEX(tags, '#', 3), '#', -1), '') AS content FROM document WHERE LENGTH(tags) - LENGTH(REPLACE(tags, "#", "")) > 2; + +-- insert tag 3 +insert into search (orgid, documentid, itemid, itemtype, content) SELECT orgid, refid as documentid, '' as itemid, 'tag' as itemtype, IF((LENGTH(tags) - LENGTH(REPLACE(tags, '#', '')) - 1) > 2, SUBSTRING_INDEX(SUBSTRING_INDEX(tags, '#', 4), '#', -1), '') AS content FROM document WHERE LENGTH(tags) - LENGTH(REPLACE(tags, "#", "")) > 3; + diff --git a/domain/document/endpoint.go b/domain/document/endpoint.go index 7adfffd0..579d6643 100644 --- a/domain/document/endpoint.go +++ b/domain/document/endpoint.go @@ -16,7 +16,6 @@ import ( "encoding/json" "io/ioutil" "net/http" - "net/url" "github.com/documize/community/core/env" "github.com/documize/community/core/request" @@ -327,27 +326,36 @@ func (h *Handler) SearchDocuments(w http.ResponseWriter, r *http.Request) { method := "document.search" ctx := domain.GetRequestContext(r) - keywords := request.Query(r, "keywords") - decoded, err := url.QueryUnescape(keywords) + defer streamutil.Close(r.Body) + body, err := ioutil.ReadAll(r.Body) if err != nil { response.WriteBadRequestError(w, method, err.Error()) + h.Runtime.Log.Error(method, err) return } - results, err := h.Store.Search.Documents(ctx, decoded) + options := search.QueryOptions{} + err = json.Unmarshal(body, &options) + if err != nil { + response.WriteBadRequestError(w, method, err.Error()) + h.Runtime.Log.Error(method, err) + return + } + + results, err := h.Store.Search.Documents(ctx, options) if err != nil { h.Runtime.Log.Error(method, err) } // Put in slugs for easy UI display of search URL for key, result := range results { - result.DocumentSlug = stringutil.MakeSlug(result.DocumentTitle) - result.FolderSlug = stringutil.MakeSlug(result.LabelName) + result.DocumentSlug = stringutil.MakeSlug(result.Document) + result.SpaceSlug = stringutil.MakeSlug(result.Space) results[key] = result } if len(results) == 0 { - results = []search.DocumentSearch{} + results = []search.QueryResult{} } h.Store.Audit.Record(ctx, audit.EventTypeSearch) diff --git a/domain/search/mysql/store.go b/domain/search/mysql/store.go index 83f14d99..eb1d2ce3 100644 --- a/domain/search/mysql/store.go +++ b/domain/search/mysql/store.go @@ -12,8 +12,8 @@ package mysql import ( + "database/sql" "fmt" - "regexp" "strings" "time" @@ -244,68 +244,156 @@ func (s Scope) Delete(ctx domain.RequestContext, page page.Page) (err error) { // Documents searches the documents that the client is allowed to see, using the keywords search string, then audits that search. // Visible documents include both those in the client's own organisation and those that are public, or whose visibility includes the client. -func (s Scope) Documents(ctx domain.RequestContext, keywords string) (results []search.DocumentSearch, err error) { - if len(keywords) == 0 { +func (s Scope) Documents(ctx domain.RequestContext, q search.QueryOptions) (results []search.QueryResult, err error) { + q.Keywords = strings.TrimSpace(q.Keywords) + + if len(q.Keywords) == 0 { return } - var tagQuery, keywordQuery string + results = []search.QueryResult{} - r, _ := regexp.Compile(`(#[a-z0-9][a-z0-9\-_]*)`) - res := r.FindAllString(keywords, -1) - - if len(res) == 0 { - tagQuery = " " - } else { - if len(res) == 1 { - tagQuery = " AND document.tags LIKE '%" + res[0] + "#%' " - } else { - fmt.Println("lots of tags!") - - tagQuery = " AND (" - - for i := 0; i < len(res); i++ { - tagQuery += "document.tags LIKE '%" + res[i] + "#%'" - if i < len(res)-1 { - tagQuery += " OR " - } - } - - tagQuery += ") " + // Match doc names + if q.Doc { + r1, err1 := s.matchFullText(ctx, q.Keywords, "doc") + if err1 != nil { + err = errors.Wrap(err1, "search document names") + return } - keywords = r.ReplaceAllString(keywords, "") - keywords = strings.Replace(keywords, " ", "", -1) + results = append(results, r1...) } - keywords = strings.TrimSpace(keywords) + // Match doc content + if q.Content { + r2, err2 := s.matchFullText(ctx, q.Keywords, "page") + if err2 != nil { + err = errors.Wrap(err2, "search document content") + return + } - if len(keywords) > 0 { - keywordQuery = "AND MATCH(documenttitle,pagetitle,body) AGAINST('" + keywords + "' in boolean mode)" + results = append(results, r2...) } - sql := `SELECT search.id, documentid, pagetitle, document.labelid, document.title as documenttitle, document.tags, - COALESCE(label.label,'Unknown') AS labelname, document.excerpt as documentexcerpt - FROM search, document LEFT JOIN label ON label.orgid=document.orgid AND label.refid = document.labelid - WHERE search.documentid = document.refid AND search.orgid=? AND document.template=0 ` + tagQuery + - `AND document.labelid IN - (SELECT refid from label WHERE orgid=? AND type=2 AND userid=? - UNION ALL SELECT refid FROM label a where orgid=? AND type=1 AND refid IN (SELECT labelid from labelrole WHERE orgid=? AND userid='' AND (canedit=1 OR canview=1)) - UNION ALL SELECT refid FROM label a where orgid=? AND type=3 AND refid IN (SELECT labelid from labelrole WHERE orgid=? AND userid=? AND (canedit=1 OR canview=1))) ` + keywordQuery + // Match doc tags + if q.Tag { + r3, err3 := s.matchFullText(ctx, q.Keywords, "tag") + if err3 != nil { + err = errors.Wrap(err3, "search document tag") + return + } - err = s.Runtime.Db.Select(&results, - sql, + results = append(results, r3...) + } + + // Match doc attachments + if q.Attachment { + r4, err4 := s.matchLike(ctx, q.Keywords, "file") + if err4 != nil { + err = errors.Wrap(err4, "search document attachments") + return + } + + results = append(results, r4...) + } + + return +} + +func (s Scope) matchFullText(ctx domain.RequestContext, keywords, itemType string) (r []search.QueryResult, err error) { + sql1 := ` + SELECT + s.id, s.orgid, s.documentid, s.itemid, s.itemtype, + d.labelid as spaceid, COALESCE(d.title,'Unknown') AS document, d.tags, d.excerpt, + COALESCE(l.label,'Unknown') AS space + FROM + search s, + document d + LEFT JOIN + label l ON l.orgid=d.orgid AND l.refid = d.labelid + WHERE + s.orgid = ? + AND s.itemtype = ? + AND s.documentid = d.refid + -- AND d.template = 0 + AND d.labelid IN (SELECT refid from label WHERE orgid=? AND type=2 AND userid=? + UNION ALL SELECT refid FROM label a where orgid=? AND type=1 AND refid IN (SELECT labelid from labelrole WHERE orgid=? AND userid='' AND (canedit=1 OR canview=1)) + UNION ALL SELECT refid FROM label a where orgid=? AND type=3 AND refid IN (SELECT labelid from labelrole WHERE orgid=? AND userid=? AND (canedit=1 OR canview=1))) + AND MATCH(s.content) AGAINST(? IN BOOLEAN MODE)` + + err = s.Runtime.Db.Select(&r, + sql1, ctx.OrgID, + itemType, ctx.OrgID, ctx.UserID, ctx.OrgID, ctx.OrgID, ctx.OrgID, ctx.OrgID, - ctx.UserID) + ctx.UserID, + keywords) + + if err == sql.ErrNoRows { + err = nil + r = []search.QueryResult{} + } if err != nil { - err = errors.Wrap(err, "search documents") + err = errors.Wrap(err, "search document "+itemType) + return + } + + return +} + +func (s Scope) matchLike(ctx domain.RequestContext, keywords, itemType string) (r []search.QueryResult, err error) { + // LIKE clause does not like quotes! + keywords = strings.Replace(keywords, "'", "", -1) + keywords = strings.Replace(keywords, "\"", "", -1) + keywords = strings.Replace(keywords, "%", "", -1) + keywords = fmt.Sprintf("%%%s%%", keywords) + + sql1 := ` + SELECT + s.id, s.orgid, s.documentid, s.itemid, s.itemtype, + d.labelid as spaceid, COALESCE(d.title,'Unknown') AS document, d.tags, d.excerpt, + COALESCE(l.label,'Unknown') AS space + FROM + search s, + document d + LEFT JOIN + label l ON l.orgid=d.orgid AND l.refid = d.labelid + WHERE + s.orgid = ? + AND s.itemtype = ? + AND s.documentid = d.refid + -- AND d.template = 0 + AND d.labelid IN (SELECT refid from label WHERE orgid=? AND type=2 AND userid=? + UNION ALL SELECT refid FROM label a where orgid=? AND type=1 AND refid IN (SELECT labelid from labelrole WHERE orgid=? AND userid='' AND (canedit=1 OR canview=1)) + UNION ALL SELECT refid FROM label a where orgid=? AND type=3 AND refid IN (SELECT labelid from labelrole WHERE orgid=? AND userid=? AND (canedit=1 OR canview=1))) + AND s.content LIKE ?` + + err = s.Runtime.Db.Select(&r, + sql1, + ctx.OrgID, + itemType, + ctx.OrgID, + ctx.UserID, + ctx.OrgID, + ctx.OrgID, + ctx.OrgID, + ctx.OrgID, + ctx.UserID, + keywords) + + if err == sql.ErrNoRows { + err = nil + r = []search.QueryResult{} + } + + if err != nil { + err = errors.Wrap(err, "search document "+itemType) return } diff --git a/domain/storer.go b/domain/storer.go index e697b6a6..7c5bc8c6 100644 --- a/domain/storer.go +++ b/domain/storer.go @@ -188,7 +188,7 @@ type SearchStorer interface { UpdateSequence(ctx RequestContext, page page.Page) (err error) UpdateLevel(ctx RequestContext, page page.Page) (err error) Delete(ctx RequestContext, page page.Page) (err error) - Documents(ctx RequestContext, keywords string) (results []search.DocumentSearch, err error) + Documents(ctx RequestContext, options search.QueryOptions) (results []search.QueryResult, err error) } // Indexer defines required methods for managing search indexing process diff --git a/gui/app/components/search/search-results.js b/gui/app/components/search/search-results.js index b59f2bcf..ef4d3bc4 100644 --- a/gui/app/components/search/search-results.js +++ b/gui/app/components/search/search-results.js @@ -16,39 +16,16 @@ export default Ember.Component.extend({ resultPhrase: "", didReceiveAttrs() { - let results = this.get('results'); - let temp = _.groupBy(results, 'documentId'); - let documents = []; - - _.each(temp, function (document) { - let refs = []; - - if (document.length > 1) { - refs = document.slice(1); - } - - _.each(refs, function (ref, index) { - ref.comma = index === refs.length - 1 ? "" : ", "; - }); - - let hasRefs = refs.length > 0; - - documents.pushObject({ - doc: document[0], - ref: refs, - hasReferences: hasRefs - }); - }); - + let docs = this.get('results'); let phrase = 'Nothing found'; - if (results.length > 0) { - let references = results.length === 1 ? "reference" : "references"; - let i = results.length; + if (docs.length > 0) { + let references = docs.length === 1 ? "reference" : "references"; + let i = docs.length; phrase = `${i} ${references}`; } this.set('resultPhrase', phrase); - this.set('documents', documents); + this.set('documents', docs); } }); diff --git a/gui/app/pods/search/controller.js b/gui/app/pods/search/controller.js index eb2bf411..38b0a26a 100644 --- a/gui/app/pods/search/controller.js +++ b/gui/app/pods/search/controller.js @@ -1,11 +1,11 @@ // Copyright 2016 Documize Inc. . All rights reserved. // -// This software (Documize Community Edition) is licensed under +// This software (Documize Community Edition) is licensed under // GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html // // You can operate outside the AGPL restrictions by purchasing // Documize Enterprise Edition and obtaining a commercial license -// by contacting . +// by contacting . // // https://documize.com @@ -13,19 +13,51 @@ import Ember from 'ember'; export default Ember.Controller.extend({ searchService: Ember.inject.service('search'), - queryParams: ['filter'], filter: "", results: [], + matchDoc: true, + matchContent: true, + matchFile: true, + matchTag: true, onKeywordChange: function () { Ember.run.debounce(this, this.fetch, 750); }.observes('filter'), + onMatchDoc: function () { + Ember.run.debounce(this, this.fetch, 750); + }.observes('matchDoc'), + onMatchContent: function () { + Ember.run.debounce(this, this.fetch, 750); + }.observes('matchContent'), + onMatchTag: function () { + Ember.run.debounce(this, this.fetch, 750); + }.observes('matchTag'), + onMatchFile: function () { + Ember.run.debounce(this, this.fetch, 750); + }.observes('matchFile'), + fetch() { let self = this; + let payload = { + keywords: this.get('filter'), + doc: this.get('matchDoc'), + attachment: this.get('matchFile'), + tag: this.get('matchTag'), + content: this.get('matchContent') + }; - this.get('searchService').find(this.get('filter')).then(function (response) { + payload.keywords = payload.keywords.trim(); + + if (payload.keywords.length == 0) { + return; + } + if (!payload.doc && !payload.tag && !payload.content && !payload.attachment) { + return; + } + + this.get('searchService').find(payload).then(function(response) { self.set('results', response); }); - } -}); \ No newline at end of file + }, +}); diff --git a/gui/app/pods/search/template.hbs b/gui/app/pods/search/template.hbs index ee7ac9dd..afafd151 100644 --- a/gui/app/pods/search/template.hbs +++ b/gui/app/pods/search/template.hbs @@ -1,15 +1,26 @@ {{layout/zone-navigation}} + {{#layout/zone-container}} {{#layout/zone-sidebar}} diff --git a/gui/app/services/search.js b/gui/app/services/search.js index e962b45a..6766de3d 100644 --- a/gui/app/services/search.js +++ b/gui/app/services/search.js @@ -19,12 +19,12 @@ export default Ember.Service.extend({ sessionService: service('session'), ajax: service(), - // getUsers returns all users for organization. - find(keywords) { - let url = "search?keywords=" + encodeURIComponent(keywords); - - return this.get('ajax').request(url, { - method: "GET" + // find all matching documents. + find(payload) { + return this.get('ajax').request("search", { + method: "POST", + data: JSON.stringify(payload), + contentType: 'json' }); }, }); diff --git a/gui/app/styles/view/page-search.scss b/gui/app/styles/view/page-search.scss index b9226ff2..2af34e56 100644 --- a/gui/app/styles/view/page-search.scss +++ b/gui/app/styles/view/page-search.scss @@ -5,6 +5,10 @@ } } + .examples { + color: $color-gray; + } + .search-results { > .heading { font-size: 2rem; diff --git a/gui/app/templates/components/search/search-results.hbs b/gui/app/templates/components/search/search-results.hbs index b9146df2..07fde071 100644 --- a/gui/app/templates/components/search/search-results.hbs +++ b/gui/app/templates/components/search/search-results.hbs @@ -1,13 +1,13 @@
{{resultPhrase}}
    - {{#each documents key="doc.id" as |result index|}} + {{#each documents key="id" as |result index|}}
  • - -
    {{ result.doc.documentTitle }}
    -
    {{ result.doc.folderName }}
    -
    {{ result.doc.documentExcerpt }}
    -
    {{search/tag-list documentTags=result.doc.documentTags}}
    +
    +
    {{ result.document }}
    +
    {{ result.space }}
    +
    {{ result.excerpt }}
    +
    {{search/tag-list documentTags=result.tags}}
  • {{/each}} diff --git a/jsconfig.json b/jsconfig.json index de7d667c..134ed86a 100644 --- a/jsconfig.json +++ b/jsconfig.json @@ -3,5 +3,5 @@ "target": "es6", "experimentalDecorators": true }, - "exclude": ["node_modules", "bower_components", "tmp", "vendor", ".git", "dist", "dist-prod"] + "exclude": ["node_modules", "bower_components", "tmp", "vendor", ".git", "dist", "dist-prod", "gui/node_modules", "gui/dist", "gui/dist-prod", "gui/tmp"] } diff --git a/model/search/search.go b/model/search/search.go index 6e726c57..f898564b 100644 --- a/model/search/search.go +++ b/model/search/search.go @@ -30,16 +30,27 @@ type Search struct { Body string } -// DocumentSearch represents 'presentable' search results. -type DocumentSearch struct { - ID string `json:"id"` - DocumentID string `json:"documentId"` - DocumentTitle string `json:"documentTitle"` - DocumentSlug string `json:"documentSlug"` - DocumentExcerpt string `json:"documentExcerpt"` - Tags string `json:"documentTags"` - PageTitle string `json:"pageTitle"` - LabelID string `json:"folderId"` - LabelName string `json:"folderName"` - FolderSlug string `json:"folderSlug"` +// QueryOptions defines how we search. +type QueryOptions struct { + Keywords string `json:"keywords"` + Doc bool `json:"doc"` + Tag bool `json:"tag"` + Attachment bool `json:"attachment"` + Content bool `json:"content"` +} + +// QueryResult represents 'presentable' search results. +type QueryResult struct { + ID string `json:"id"` + OrgID string `json:"orgId"` + ItemID string `json:"itemId"` + ItemType string `json:"itemType"` + DocumentID string `json:"documentId"` + DocumentSlug string `json:"documentSlug"` + Document string `json:"document"` + Excerpt string `json:"excerpt"` + Tags string `json:"tags"` + SpaceID string `json:"spaceId"` + Space string `json:"space"` + SpaceSlug string `json:"spaceSlug"` } diff --git a/server/routing/routes.go b/server/routing/routes.go index 8ab9cf6b..74d92bf2 100644 --- a/server/routing/routes.go +++ b/server/routing/routes.go @@ -132,7 +132,7 @@ func RegisterEndpoints(rt *env.Runtime, s *domain.Store) { Add(rt, RoutePrefixPrivate, "users/{userID}", []string{"DELETE", "OPTIONS"}, nil, user.Delete) Add(rt, RoutePrefixPrivate, "users/sync", []string{"GET", "OPTIONS"}, nil, keycloak.Sync) - Add(rt, RoutePrefixPrivate, "search", []string{"GET", "OPTIONS"}, nil, document.SearchDocuments) + Add(rt, RoutePrefixPrivate, "search", []string{"POST", "OPTIONS"}, nil, document.SearchDocuments) Add(rt, RoutePrefixPrivate, "templates", []string{"POST", "OPTIONS"}, nil, template.SaveAs) Add(rt, RoutePrefixPrivate, "templates", []string{"GET", "OPTIONS"}, nil, template.SavedList)