From 8a655671694f2363cdb218f2bdb26b641cdf703c Mon Sep 17 00:00:00 2001 From: HarveyKandola Date: Fri, 28 Sep 2018 16:33:15 +0100 Subject: [PATCH] Implement PostgreSQL Full Text Search++ 1. Full text search supports MySQL, MariaDB, Percona and now PostgreSQL. 2. Changed SQL Variant to typed enum. 3. Changed doc.Versioned from INT to BOOL. 4. Search Reindexer now parses all documents and attachments. 5. Site meta API call returns storage provider type. 6. README prep'ed for PostgreSQL support. 7. DELETE SQL statements ignore zero rows affected. Closes #100 !!! Co-Authored-By: Saul S Co-Authored-By: McMatts --- README.md | 33 +++-- core/database/installer.go | 4 +- core/database/scripts/postgresql/db_00001.sql | 3 +- core/env/provider.go | 2 +- domain/attachment/store.go | 11 ++ domain/category/store.go | 2 +- domain/document/store.go | 20 +-- domain/group/store.go | 8 +- domain/link/store.go | 4 +- domain/meta/endpoint.go | 14 ++ domain/meta/store.go | 4 + domain/permission/store.go | 5 +- domain/pin/store.go | 4 +- domain/search/search.go | 7 +- domain/search/store.go | 120 ++++++++++++++---- domain/space/endpoint.go | 8 +- domain/store/context.go | 13 +- edition/storage/mysql.go | 23 +++- edition/storage/postgresql.go | 12 +- gui/app/constants/constants.js | 6 + gui/app/pods/customize/search/route.js | 2 +- gui/app/pods/search/controller.js | 3 + gui/app/pods/search/route.js | 2 +- gui/app/pods/search/template.hbs | 46 ++++--- gui/app/services/app-meta.js | 1 + model/org/meta.go | 30 +++-- 26 files changed, 274 insertions(+), 113 deletions(-) diff --git a/README.md b/README.md index e66a56f2..22a3f886 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -> We're committed to providing frequent product releases to ensure self-host customers enjoy the same product as our cloud/SaaS customers. +> We provide frequent product releases ensuring self-host customers enjoy the same features as our cloud/SaaS customers. > > Harvey Kandola, CEO & Founder, Documize Inc. @@ -58,19 +58,32 @@ Space view. ## Latest version -[Community edition: v1.70.0](https://github.com/documize/community/releases) +[Community edition: v1.71.0](https://github.com/documize/community/releases) -[Enterprise edition: v1.72.0](https://documize.com/downloads) +[Enterprise edition: v1.73.0](https://documize.com/downloads) ## OS support -Documize runs on the following: +Documize can be installed and run on: - Linux - Windows - macOS -# Browser support +Heck, Documize will probably run just fine on a Raspberry Pi 3. + +## Database support + +Documize supports the following database systems: + +- PostgreSQL (v9.6+) +- MySQL (v5.7.10+ and v8.0.0+) +- Percona (v5.7.16-10+) +- MariaDB (10.3.0+) + +Coming soon: Microsoft SQL Server 2017 (Linux/Windows). + +## Browser support Documize supports the following (evergreen) browsers: @@ -78,6 +91,8 @@ Documize supports the following (evergreen) browsers: - Firefox - Safari - Brave +- Vivaldi +- Opera - MS Edge (16+) ## Technology stack @@ -87,14 +102,6 @@ Documize is built with the following technologies: - EmberJS (v3.1.2) - Go (v1.10.3) -...and supports the following databases: - -- MySQL (v5.7.10+) -- Percona (v5.7.16-10+) -- MariaDB (10.3.0+) - -Coming soon, PostgreSQL and Microsoft SQL Server database support. - ## Authentication options Besides email/password login, you can also leverage the following options. diff --git a/core/database/installer.go b/core/database/installer.go index 1757ee9e..f4b26d75 100644 --- a/core/database/installer.go +++ b/core/database/installer.go @@ -174,13 +174,13 @@ func runScripts(runtime *env.Runtime, tx *sqlx.Tx, scripts []Script) (err error) } // executeSQL runs specified SQL commands. -func executeSQL(tx *sqlx.Tx, st env.StoreType, variant string, SQLfile []byte) error { +func executeSQL(tx *sqlx.Tx, st env.StoreType, variant env.StoreType, SQLfile []byte) error { // Turn SQL file contents into runnable SQL statements. stmts := getStatements(SQLfile) for _, stmt := range stmts { // MariaDB has no specific JSON column type (but has JSON queries) - if st == env.StoreTypeMySQL && variant == "mariadb" { + if st == env.StoreTypeMySQL && variant == env.StoreTypeMariaDB { stmt = strings.Replace(stmt, "` JSON", "` TEXT", -1) } diff --git a/core/database/scripts/postgresql/db_00001.sql b/core/database/scripts/postgresql/db_00001.sql index 467b3acd..64255f7f 100644 --- a/core/database/scripts/postgresql/db_00001.sql +++ b/core/database/scripts/postgresql/db_00001.sql @@ -287,12 +287,13 @@ CREATE TABLE dmz_search ( c_itemid varchar(16) COLLATE ucs_basic NOT NULL DEFAULT '', c_itemtype varchar(10) COLLATE ucs_basic NOT NULL, c_content text COLLATE ucs_basic, + c_token TSVECTOR, c_created timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, UNIQUE (id) ); CREATE INDEX idx_search_1 ON dmz_search (c_orgid); CREATE INDEX idx_search_2 ON dmz_search (c_docid); -CREATE INDEX idx_search_3 ON dmz_search USING GIN (to_tsvector('english', c_content)); +CREATE INDEX idx_search_3 ON dmz_search USING GIN(c_token); DROP TABLE IF EXISTS dmz_section; CREATE TABLE dmz_section ( diff --git a/core/env/provider.go b/core/env/provider.go index db9097df..969771b4 100644 --- a/core/env/provider.go +++ b/core/env/provider.go @@ -38,7 +38,7 @@ type StoreProvider interface { Type() StoreType // TypeVariant returns flavor of database provider. - TypeVariant() string + TypeVariant() StoreType // SQL driver name used to open DB connection. DriverName() string diff --git a/domain/attachment/store.go b/domain/attachment/store.go index 0cecf68b..2f22611b 100644 --- a/domain/attachment/store.go +++ b/domain/attachment/store.go @@ -12,6 +12,7 @@ package attachment import ( + "database/sql" "strings" "time" @@ -75,8 +76,13 @@ func (s Store) GetAttachments(ctx domain.RequestContext, docID string) (a []atta ORDER BY c_filename`), ctx.OrgID, docID) + if err == sql.ErrNoRows { + err = nil + a = []attachment.Attachment{} + } if err != nil { err = errors.Wrap(err, "execute select attachments") + return } return @@ -94,6 +100,11 @@ func (s Store) GetAttachmentsWithData(ctx domain.RequestContext, docID string) ( ORDER BY c_filename`), ctx.OrgID, docID) + if err == sql.ErrNoRows { + err = nil + a = []attachment.Attachment{} + } + if err != nil { err = errors.Wrap(err, "execute select attachments with data") } diff --git a/domain/category/store.go b/domain/category/store.go index 32b9313e..84be3150 100644 --- a/domain/category/store.go +++ b/domain/category/store.go @@ -202,7 +202,7 @@ func (s Store) RemoveDocumentCategories(ctx domain.RequestContext, documentID st // DeleteBySpace removes all category and category associations for given space. func (s Store) DeleteBySpace(ctx domain.RequestContext, spaceID string) (rows int64, err error) { - s1 := fmt.Sprintf("DELETE FROM categorymember WHERE c_orgid='%s' AND c_groupid='%s'", ctx.OrgID, spaceID) + s1 := fmt.Sprintf("DELETE FROM dmz_category_member WHERE c_orgid='%s' AND c_spaceid='%s'", ctx.OrgID, spaceID) s.DeleteWhere(ctx.Transaction, s1) s2 := fmt.Sprintf("DELETE FROM dmz_category WHERE c_orgid='%s' AND c_spaceid='%s'", ctx.OrgID, spaceID) diff --git a/domain/document/store.go b/domain/document/store.go index 00aa5ccb..e1068114 100644 --- a/domain/document/store.go +++ b/domain/document/store.go @@ -245,28 +245,28 @@ func (s Store) MoveActivity(ctx domain.RequestContext, documentID, oldSpaceID, n // Delete removes the specified document. // Remove document pages, revisions, attachments, updates the search subsystem. func (s Store) Delete(ctx domain.RequestContext, documentID string) (rows int64, err error) { - rows, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_section WHERE c_docid=\"%s\" AND c_orgid=\"%s\"", documentID, ctx.OrgID)) + rows, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_section WHERE c_docid='%s' AND c_orgid='%s'", documentID, ctx.OrgID)) if err != nil { return } - _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_section_revision WHERE c_docid=\"%s\" AND c_orgid=\"%s\"", documentID, ctx.OrgID)) + _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_section_revision WHERE c_docid='%s' AND c_orgid='%s'", documentID, ctx.OrgID)) if err != nil { return } - _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_attachment WHERE c_docid=\"%s\" AND c_orgid=\"%s\"", documentID, ctx.OrgID)) + _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_attachment WHERE c_docid='%s' AND c_orgid='%s'", documentID, ctx.OrgID)) if err != nil { return } - _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_category_member WHERE c_docid=\"%s\" AND c_orgid=\"%s\"", documentID, ctx.OrgID)) + _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_category_member WHERE c_docid='%s' AND c_orgid='%s'", documentID, ctx.OrgID)) if err != nil { return } - _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_vote WHERE c_docid=\"%s\" AND c_orgid=\"%s\"", documentID, ctx.OrgID)) + _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_vote WHERE c_docid='%s' AND c_orgid='%s'", documentID, ctx.OrgID)) if err != nil { return } @@ -277,23 +277,23 @@ func (s Store) Delete(ctx domain.RequestContext, documentID string) (rows int64, // DeleteBySpace removes all documents for given space. // Remove document pages, revisions, attachments, updates the search subsystem. func (s Store) DeleteBySpace(ctx domain.RequestContext, spaceID string) (rows int64, err error) { - rows, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_section WHERE c_docid IN (SELECT c_refid FROM dmz_doc WHERE c_spaceid=\"%s\" AND c_orgid=\"%s\")", spaceID, ctx.OrgID)) + rows, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_section WHERE c_docid IN (SELECT c_refid FROM dmz_doc WHERE c_spaceid='%s' AND c_orgid='%s')", spaceID, ctx.OrgID)) if err != nil { return } - _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_section_revision WHERE c_docid IN (SELECT c_refid FROM dmz_doc WHERE c_spaceid=\"%s\" AND c_orgid=\"%s\")", spaceID, ctx.OrgID)) + _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_section_revision WHERE c_docid IN (SELECT c_refid FROM dmz_doc WHERE c_spaceid='%s' AND c_orgid='%s')", spaceID, ctx.OrgID)) if err != nil { return } - _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_attachment WHERE c_docid IN (SELECT c_refid FROM dmz_doc WHERE c_spaceid=\"%s\" AND c_orgid=\"%s\")", spaceID, ctx.OrgID)) + _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_attachment WHERE c_docid IN (SELECT c_refid FROM dmz_doc WHERE c_spaceid='%s' AND c_orgid='%s')", spaceID, ctx.OrgID)) if err != nil { return } - _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_vote WHERE c_docid IN (SELECT c_refid FROM dmz_doc WHERE c_spaceid=\"%s\" AND c_orgid=\"%s\")", spaceID, ctx.OrgID)) + _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_vote WHERE c_docid IN (SELECT c_refid FROM dmz_doc WHERE c_spaceid='%s' AND c_orgid='%s')", spaceID, ctx.OrgID)) if err != nil { return } @@ -332,7 +332,7 @@ func (s Store) GetVersions(ctx domain.RequestContext, groupID string) (v []doc.V // Any existing vote by the user is replaced. func (s Store) Vote(ctx domain.RequestContext, refID, orgID, documentID, userID string, vote int) (err error) { _, err = s.DeleteWhere(ctx.Transaction, - fmt.Sprintf("DELETE FROM dmz_doc_vote WHERE c_orgid=\"%s\" AND c_docid=\"%s\" AND c_voter=\"%s\"", + fmt.Sprintf("DELETE FROM dmz_doc_vote WHERE c_orgid='%s' AND c_docid='%s' AND c_voter='%s'", orgID, documentID, userID)) if err != nil { s.Runtime.Log.Error("store.Vote", err) diff --git a/domain/group/store.go b/domain/group/store.go index b3bacd7c..ff93e9bb 100644 --- a/domain/group/store.go +++ b/domain/group/store.go @@ -104,7 +104,7 @@ func (s Store) Delete(ctx domain.RequestContext, refID string) (rows int64, err if err != nil { return } - return s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_group_member WHERE c_orgid=\"%s\" AND c_groupid=\"%s\"", ctx.OrgID, refID)) + return s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_group_member WHERE c_orgid='%s' AND c_groupid='%s'", ctx.OrgID, refID)) } // GetGroupMembers returns all user associated with given group. @@ -143,8 +143,12 @@ func (s Store) JoinGroup(ctx domain.RequestContext, groupID, userID string) (err // LeaveGroup removes user from group. func (s Store) LeaveGroup(ctx domain.RequestContext, groupID, userID string) (err error) { - _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_group_member WHERE c_orgid=\"%s\" AND c_groupid=\"%s\" AND c_userid=\"%s\"", + _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_group_member WHERE c_orgid='%s' AND c_groupid='%s' AND c_userid='%s'", ctx.OrgID, groupID, userID)) + + if err == sql.ErrNoRows { + err = nil + } if err != nil { err = errors.Wrap(err, "clear group member") } diff --git a/domain/link/store.go b/domain/link/store.go index cf0137a7..94159e67 100644 --- a/domain/link/store.go +++ b/domain/link/store.go @@ -137,12 +137,12 @@ func (s Store) MarkOrphanAttachmentLink(ctx domain.RequestContext, attachmentID // DeleteSourcePageLinks removes saved links for given source. func (s Store) DeleteSourcePageLinks(ctx domain.RequestContext, pageID string) (rows int64, err error) { - return s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_link WHERE c_orgid=\"%s\" AND c_sourcesectionid=\"%s\"", ctx.OrgID, pageID)) + return s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_link WHERE c_orgid='%s' AND c_sourcesectionid='%s'", ctx.OrgID, pageID)) } // DeleteSourceDocumentLinks removes saved links for given document. func (s Store) DeleteSourceDocumentLinks(ctx domain.RequestContext, documentID string) (rows int64, err error) { - return s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_link WHERE c_orgid=\"%s\" AND c_sourcedocid=\"%s\"", ctx.OrgID, documentID)) + return s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_link WHERE c_orgid='%s' AND c_sourcedocid='%s'", ctx.OrgID, documentID)) } // DeleteLink removes saved link from the store. diff --git a/domain/meta/endpoint.go b/domain/meta/endpoint.go index ecd926d2..c6f4534f 100644 --- a/domain/meta/endpoint.go +++ b/domain/meta/endpoint.go @@ -62,6 +62,7 @@ func (h *Handler) Meta(w http.ResponseWriter, r *http.Request) { data.Valid = h.Runtime.Product.License.Valid data.ConversionEndpoint = org.ConversionEndpoint data.License = h.Runtime.Product.License + data.Storage = h.Runtime.StoreProvider.Type() // Strip secrets data.AuthConfig = auth.StripAuthSecrets(h.Runtime, org.AuthProvider, org.AuthConfig) @@ -213,6 +214,19 @@ func (h *Handler) rebuildSearchIndex(ctx domain.RequestContext) { for i := range docs { d := docs[i] + doc, err := h.Store.Document.Get(ctx, d) + if err != nil { + h.Runtime.Log.Error(method, err) + return + } + at, err := h.Store.Attachment.GetAttachments(ctx, d) + if err != nil { + h.Runtime.Log.Error(method, err) + return + } + + h.Indexer.IndexDocument(ctx, doc, at) + pages, err := h.Store.Meta.GetDocumentPages(ctx, d) if err != nil { h.Runtime.Log.Error(method, err) diff --git a/domain/meta/store.go b/domain/meta/store.go index 879dad3e..d7d6df87 100644 --- a/domain/meta/store.go +++ b/domain/meta/store.go @@ -54,6 +54,10 @@ func (s Store) GetDocumentPages(ctx domain.RequestContext, documentID string) (p WHERE c_docid=? AND (c_status=0 OR ((c_status=4 OR c_status=2) AND c_relativeid=''))`), documentID) + if err == sql.ErrNoRows { + err = nil + p = []page.Page{} + } if err != nil { err = errors.Wrap(err, "failed to get instance document pages") } diff --git a/domain/permission/store.go b/domain/permission/store.go index 4a256bb1..11d9b9d1 100644 --- a/domain/permission/store.go +++ b/domain/permission/store.go @@ -33,7 +33,8 @@ type Store struct { func (s Store) AddPermission(ctx domain.RequestContext, r permission.Permission) (err error) { r.Created = time.Now().UTC() - _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_permission (c_orgid, c_who, c_whoid, c_action, c_scope, c_location, c_refid, c_created) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"), + _, err = ctx.Transaction.Exec(s.Bind(`INSERT INTO dmz_permission + (c_orgid, c_who, c_whoid, c_action, c_scope, c_location, c_refid, c_created) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`), r.OrgID, string(r.Who), r.WhoID, string(r.Action), string(r.Scope), string(r.Location), r.RefID, r.Created) if err != nil { @@ -278,7 +279,7 @@ func (s Store) DeleteSpacePermissions(ctx domain.RequestContext, spaceID string) // DeleteUserSpacePermissions removes all roles for the specified user, for the specified space. func (s Store) DeleteUserSpacePermissions(ctx domain.RequestContext, spaceID, userID string) (rows int64, err error) { - sql := fmt.Sprintf("DELETE FROM dmz_permission WHERE c_orgid='%s' AND c_location='space' AND c_refid='%s' c_who='user' AND c_whoid='%s'", + sql := fmt.Sprintf("DELETE FROM dmz_permission WHERE c_orgid='%s' AND c_location='space' AND c_refid='%s' AND c_who='user' AND c_whoid='%s'", ctx.OrgID, spaceID, userID) return s.DeleteWhere(ctx.Transaction, sql) diff --git a/domain/pin/store.go b/domain/pin/store.go index d33063e5..23a4b665 100644 --- a/domain/pin/store.go +++ b/domain/pin/store.go @@ -120,10 +120,10 @@ func (s Store) DeletePin(ctx domain.RequestContext, id string) (rows int64, err // DeletePinnedSpace removes any pins for specified space. func (s Store) DeletePinnedSpace(ctx domain.RequestContext, spaceID string) (rows int64, err error) { - return s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_pin WHERE c_orgid=\"%s\" AND c_spaceid=\"%s\"", ctx.OrgID, spaceID)) + return s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_pin WHERE c_orgid='%s' AND c_spaceid='%s'", ctx.OrgID, spaceID)) } // DeletePinnedDocument removes any pins for specified document. func (s Store) DeletePinnedDocument(ctx domain.RequestContext, documentID string) (rows int64, err error) { - return s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_pin WHERE c_orgid=\"%s\" AND c_docid=\"%s\"", ctx.OrgID, documentID)) + return s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_pin WHERE c_orgid='%s' AND c_docid='%s'", ctx.OrgID, documentID)) } diff --git a/domain/search/search.go b/domain/search/search.go index 90cf572f..56088550 100644 --- a/domain/search/search.go +++ b/domain/search/search.go @@ -82,7 +82,12 @@ func (m *Indexer) IndexContent(ctx domain.RequestContext, p page.Page) { return } - ctx.Transaction.Commit() + err = ctx.Transaction.Commit() + if err != nil { + ctx.Transaction.Rollback() + m.runtime.Log.Error(method, err) + return + } } // DeleteContent removes all search entries for specific document content. diff --git a/domain/search/store.go b/domain/search/store.go index c108d75b..549c6589 100644 --- a/domain/search/store.go +++ b/domain/search/store.go @@ -39,43 +39,68 @@ type Store struct { // IndexDocument adds search index entries for document inserting title, tags and attachments as // searchable items. Any existing document entries are removed. func (s Store) IndexDocument(ctx domain.RequestContext, doc doc.Document, a []attachment.Attachment) (err error) { + method := "search.IndexDocument" + // remove previous search entries _, err = ctx.Transaction.Exec(s.Bind("DELETE FROM dmz_search WHERE c_orgid=? AND c_docid=? AND (c_itemtype='doc' OR c_itemtype='file' OR c_itemtype='tag')"), ctx.OrgID, doc.RefID) - - if err != nil { + if err != nil && err != sql.ErrNoRows { err = errors.Wrap(err, "execute delete document index entries") + s.Runtime.Log.Error(method, err) + return } // insert doc title - _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content) VALUES (?, ?, ?, ?, ?)"), - ctx.OrgID, doc.RefID, "", "doc", doc.Name) - if err != nil { + if s.Runtime.StoreProvider.Type() == env.StoreTypePostgreSQL { + _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content, c_token) VALUES (?, ?, ?, ?, ?, to_tsvector(?))"), + ctx.OrgID, doc.RefID, "", "doc", doc.Name, doc.Name) + + } else { + _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content) VALUES (?, ?, ?, ?, ?)"), + ctx.OrgID, doc.RefID, "", "doc", doc.Name) + } + if err != nil && err != sql.ErrNoRows { err = errors.Wrap(err, "execute insert document title entry") + s.Runtime.Log.Error(method, err) + return } // insert doc tags tags := strings.Split(doc.Tags, "#") for _, t := range tags { + t = strings.TrimSpace(t) if len(t) == 0 { continue } - _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content) VALUES (?, ?, ?, ?, ?)"), - ctx.OrgID, doc.RefID, "", "tag", t) + if s.Runtime.StoreProvider.Type() == env.StoreTypePostgreSQL { + _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content, c_token) VALUES (?, ?, ?, ?, ?, to_tsvector(?))"), + ctx.OrgID, doc.RefID, "", "tag", t, t) - if err != nil { + } else { + _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content) VALUES (?, ?, ?, ?, ?)"), + ctx.OrgID, doc.RefID, "", "tag", t) + } + if err != nil && err != sql.ErrNoRows { err = errors.Wrap(err, "execute insert document tag entry") + s.Runtime.Log.Error(method, err) return } } for _, file := range a { - _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content) VALUES (?, ?, ?, ?, ?)"), - ctx.OrgID, doc.RefID, file.RefID, "file", file.Filename) + if s.Runtime.StoreProvider.Type() == env.StoreTypePostgreSQL { + _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content, c_token) VALUES (?, ?, ?, ?, ?, to_tsvector(?))"), + ctx.OrgID, doc.RefID, file.RefID, "file", file.Filename, file.Filename) - if err != nil { + } else { + _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content) VALUES (?, ?, ?, ?, ?)"), + ctx.OrgID, doc.RefID, file.RefID, "file", file.Filename) + } + if err != nil && err != sql.ErrNoRows { err = errors.Wrap(err, "execute insert document file entry") + s.Runtime.Log.Error(method, err) + return } } @@ -84,11 +109,14 @@ func (s Store) IndexDocument(ctx domain.RequestContext, doc doc.Document, a []at // DeleteDocument removes all search entries for document. func (s Store) DeleteDocument(ctx domain.RequestContext, ID string) (err error) { + method := "search.DeleteDocument" + _, err = ctx.Transaction.Exec(s.Bind("DELETE FROM dmz_search WHERE c_orgid=? AND c_docid=?"), ctx.OrgID, ID) - if err != nil { + if err != nil && err != sql.ErrNoRows { err = errors.Wrap(err, "execute delete document entries") + s.Runtime.Log.Error(method, err) } return @@ -97,6 +125,8 @@ func (s Store) DeleteDocument(ctx domain.RequestContext, ID string) (err error) // IndexContent adds search index entry for document context. // Any existing document entries are removed. func (s Store) IndexContent(ctx domain.RequestContext, p page.Page) (err error) { + method := "search.IndexContent" + // we do not index pending pages if p.Status == workflow.ChangePending || p.Status == workflow.ChangePendingNew { return @@ -106,28 +136,49 @@ func (s Store) IndexContent(ctx domain.RequestContext, p page.Page) (err error) _, err = ctx.Transaction.Exec(s.Bind("DELETE FROM dmz_search WHERE c_orgid=? AND c_docid=? AND c_itemid=? AND c_itemtype='page'"), ctx.OrgID, p.DocumentID, p.RefID) - if err != nil { + if err != nil && err != sql.ErrNoRows { err = errors.Wrap(err, "execute delete document content entry") + s.Runtime.Log.Error(method, err) + return } + err = nil // prepare content content, err := stringutil.HTML(p.Body).Text(false) if err != nil { err = errors.Wrap(err, "search strip HTML failed") + s.Runtime.Log.Error(method, err) return } content = strings.TrimSpace(content) - _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content) VALUES (?, ?, ?, ?, ?)"), - ctx.OrgID, p.DocumentID, p.RefID, "page", content) - if err != nil { - err = errors.Wrap(err, "execute insert document content entry") - } + if s.Runtime.StoreProvider.Type() == env.StoreTypePostgreSQL { + _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content, c_token) VALUES (?, ?, ?, ?, ?, to_tsvector(?))"), + ctx.OrgID, p.DocumentID, p.RefID, "page", content, content) - _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content) VALUES (?, ?, ?, ?, ?)"), - ctx.OrgID, p.DocumentID, p.RefID, "page", p.Name) - if err != nil { - err = errors.Wrap(err, "execute insert document page title entry") + } else { + _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content) VALUES (?, ?, ?, ?, ?)"), + ctx.OrgID, p.DocumentID, p.RefID, "page", content) + } + if err != nil && err != sql.ErrNoRows { + err = errors.Wrap(err, "execute insert section content entry") + s.Runtime.Log.Error(method, err) + return + } + err = nil + + if s.Runtime.StoreProvider.Type() == env.StoreTypePostgreSQL { + _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content, c_token) VALUES (?, ?, ?, ?, ?, to_tsvector(?))"), + ctx.OrgID, p.DocumentID, p.RefID, "page", p.Name, p.Name) + + } else { + _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content) VALUES (?, ?, ?, ?, ?)"), + ctx.OrgID, p.DocumentID, p.RefID, "page", p.Name) + } + if err != nil && err != sql.ErrNoRows { + err = errors.Wrap(err, "execute insert section title entry") + s.Runtime.Log.Error(method, err) + return } return nil @@ -135,18 +186,23 @@ func (s Store) IndexContent(ctx domain.RequestContext, p page.Page) (err error) // DeleteContent removes all search entries for specific document content. func (s Store) DeleteContent(ctx domain.RequestContext, pageID string) (err error) { + method := "search.DeleteContent" + // remove all search entries var stmt1 *sqlx.Stmt stmt1, err = ctx.Transaction.Preparex(s.Bind("DELETE FROM dmz_search WHERE c_orgid=? AND c_itemid=? AND c_itemtype=?")) defer streamutil.Close(stmt1) - if err != nil { + + if err != nil && err != sql.ErrNoRows { err = errors.Wrap(err, "prepare delete document content entry") + s.Runtime.Log.Error(method, err) return } _, err = stmt1.Exec(ctx.OrgID, pageID, "page") - if err != nil { + if err != nil && err != sql.ErrNoRows { err = errors.Wrap(err, "execute delete document content entry") + s.Runtime.Log.Error(method, err) return } @@ -220,9 +276,17 @@ func (s Store) matchFullText(ctx domain.RequestContext, keywords, itemType strin switch s.Runtime.StoreProvider.Type() { case env.StoreTypeMySQL: - fts = " AND MATCH(s.c_content) AGAINST(? IN BOOLEAN MODE)" + fts = " AND MATCH(s.c_content) AGAINST(? IN BOOLEAN MODE) " case env.StoreTypePostgreSQL: - fts = "" + // By default, we expect no Postgres full text search operators. + parser := "plainto_tsquery" + // If we find operators then we have to use correct query processor. + operator := strings.ContainsAny(keywords, "!()&|*'`\":<->") + if operator { + parser = "to_tsquery" + } + + fts = fmt.Sprintf(" AND s.c_token @@ %s(?) ", parser) } sql1 := s.Bind(` @@ -279,7 +343,7 @@ func (s Store) matchLike(ctx domain.RequestContext, keywords, itemType string) ( keywords = strings.Replace(keywords, "'", "", -1) keywords = strings.Replace(keywords, "\"", "", -1) keywords = strings.Replace(keywords, "%", "", -1) - keywords = fmt.Sprintf("%%%s%%", keywords) + keywords = fmt.Sprintf("%%%s%%", strings.ToLower(keywords)) sql1 := s.Bind(`SELECT s.id, s.c_orgid AS orgid, s.c_docid AS documentid, s.c_itemid AS itemid, s.c_itemtype AS itemtype, @@ -304,7 +368,7 @@ func (s Store) matchLike(ctx domain.RequestContext, keywords, itemType string) ( AND p.c_location='space' AND (r.c_userid=? OR r.c_userid='0') ) ) - AND s.c_content LIKE ?`) + AND LOWER(s.c_content) LIKE ?`) err = s.Runtime.Db.Select(&r, sql1, diff --git a/domain/space/endpoint.go b/domain/space/endpoint.go index ba106d92..8ccf141e 100644 --- a/domain/space/endpoint.go +++ b/domain/space/endpoint.go @@ -470,7 +470,13 @@ func (h *Handler) Update(w http.ResponseWriter, r *http.Request) { // If newly marked Everyone space, ensure everyone has permission if prev.Type != space.ScopePublic && sp.Type == space.ScopePublic { - h.Store.Permission.DeleteUserSpacePermissions(ctx, sp.RefID, user.EveryoneUserID) + _, err = h.Store.Permission.DeleteUserSpacePermissions(ctx, sp.RefID, user.EveryoneUserID) + if err != nil { + ctx.Transaction.Rollback() + response.WriteServerError(w, method, err) + h.Runtime.Log.Error(method, err) + return + } perm := permission.Permission{} perm.OrgID = sp.OrgID diff --git a/domain/store/context.go b/domain/store/context.go index 58631832..75a6e993 100644 --- a/domain/store/context.go +++ b/domain/store/context.go @@ -12,6 +12,7 @@ package store import ( + "database/sql" "fmt" "github.com/documize/community/core/env" "github.com/jmoiron/sqlx" @@ -44,12 +45,13 @@ func (c *Context) Bind(sql string) string { func (c *Context) Delete(tx *sqlx.Tx, table string, id string) (rows int64, err error) { result, err := tx.Exec(c.Bind("DELETE FROM "+table+" WHERE c_refid=?"), id) - if err != nil { + if err != nil && err != sql.ErrNoRows { err = errors.Wrap(err, fmt.Sprintf("unable to delete row in table %s", table)) return } rows, err = result.RowsAffected() + err = nil return } @@ -58,12 +60,13 @@ func (c *Context) Delete(tx *sqlx.Tx, table string, id string) (rows int64, err func (c *Context) DeleteConstrained(tx *sqlx.Tx, table string, orgID, id string) (rows int64, err error) { result, err := tx.Exec(c.Bind("DELETE FROM "+table+" WHERE c_orgid=? AND c_refid=?"), orgID, id) - if err != nil { + if err != nil && err != sql.ErrNoRows { err = errors.Wrap(err, fmt.Sprintf("unable to delete row in table %s", table)) return } rows, err = result.RowsAffected() + err = nil return } @@ -72,12 +75,13 @@ func (c *Context) DeleteConstrained(tx *sqlx.Tx, table string, orgID, id string) func (c *Context) DeleteConstrainedWithID(tx *sqlx.Tx, table string, orgID, id string) (rows int64, err error) { result, err := tx.Exec(c.Bind("DELETE FROM "+table+" WHERE c_orgid=? AND id=?"), orgID, id) - if err != nil { + if err != nil && err != sql.ErrNoRows { err = errors.Wrap(err, fmt.Sprintf("unable to delete row in table %s", table)) return } rows, err = result.RowsAffected() + err = nil return } @@ -86,12 +90,13 @@ func (c *Context) DeleteConstrainedWithID(tx *sqlx.Tx, table string, orgID, id s func (c *Context) DeleteWhere(tx *sqlx.Tx, statement string) (rows int64, err error) { result, err := tx.Exec(statement) - if err != nil { + if err != nil && err != sql.ErrNoRows { err = errors.Wrap(err, fmt.Sprintf("unable to delete rows: %s", statement)) return } rows, err = result.RowsAffected() + err = nil return } diff --git a/edition/storage/mysql.go b/edition/storage/mysql.go index 114c22fd..d82e58db 100644 --- a/edition/storage/mysql.go +++ b/edition/storage/mysql.go @@ -43,13 +43,24 @@ import ( // SetMySQLProvider creates MySQL provider func SetMySQLProvider(r *env.Runtime, s *store.Store) { - // Set up provider specific details and wire up data prividers. - r.StoreProvider = MySQLProvider{ + // Set up provider specific details. + p := MySQLProvider{ ConnectionString: r.Flags.DBConn, - Variant: r.Flags.DBType, + } + switch r.Flags.DBType { + case "mysql": + p.Variant = env.StoreTypeMySQL + case "mariadb": + p.Variant = env.StoreTypeMariaDB + case "percona": + p.Variant = env.StoreTypePercona } - // Wire up data providers! + r.StoreProvider = p + + // Wire up data providers. + + // Account accountStore := account.Store{} accountStore.Runtime = r s.Account = accountStore @@ -146,7 +157,7 @@ type MySQLProvider struct { ConnectionString string // User specified db type (mysql, percona or mariadb). - Variant string + Variant env.StoreType } // Type returns name of provider @@ -155,7 +166,7 @@ func (p MySQLProvider) Type() env.StoreType { } // TypeVariant returns databse flavor -func (p MySQLProvider) TypeVariant() string { +func (p MySQLProvider) TypeVariant() env.StoreType { return p.Variant } diff --git a/edition/storage/postgresql.go b/edition/storage/postgresql.go index 9ca1527a..14f318f6 100644 --- a/edition/storage/postgresql.go +++ b/edition/storage/postgresql.go @@ -45,18 +45,20 @@ type PostgreSQLProvider struct { ConnectionString string // Unused for this provider. - Variant string + Variant env.StoreType } // SetPostgreSQLProvider creates PostgreSQL provider func SetPostgreSQLProvider(r *env.Runtime, s *store.Store) { - // Set up provider specific details and wire up data prividers. + // Set up provider specific details. r.StoreProvider = PostgreSQLProvider{ ConnectionString: r.Flags.DBConn, - Variant: "", + Variant: env.StoreTypePostgreSQL, } - // Wire up data providers! + // Wire up data providers. + + // Account accountStore := account.Store{} accountStore.Runtime = r s.Account = accountStore @@ -153,7 +155,7 @@ func (p PostgreSQLProvider) Type() env.StoreType { } // TypeVariant returns databse flavor -func (p PostgreSQLProvider) TypeVariant() string { +func (p PostgreSQLProvider) TypeVariant() env.StoreType { return p.Variant } diff --git a/gui/app/constants/constants.js b/gui/app/constants/constants.js index cd63062e..4c915c6e 100644 --- a/gui/app/constants/constants.js +++ b/gui/app/constants/constants.js @@ -135,6 +135,12 @@ let constants = EmberObject.extend({ Rejected: 6, Publish: 7, }, + + // Meta + StoreProvider: { // eslint-disable-line ember/avoid-leaking-state-in-ember-objects + MySQL: 'MySQL', + PostgreSQL: 'PostgreSQL', + }, }); export default { constants } diff --git a/gui/app/pods/customize/search/route.js b/gui/app/pods/customize/search/route.js index a2ee81d3..12e37183 100644 --- a/gui/app/pods/customize/search/route.js +++ b/gui/app/pods/customize/search/route.js @@ -29,6 +29,6 @@ export default Route.extend(AuthenticatedRouteMixin, { }, activate() { - this.get('browser').setTitle('Search'); + this.get('browser').setTitle('Search Engine'); } }); diff --git a/gui/app/pods/search/controller.js b/gui/app/pods/search/controller.js index ea37e3a6..6e2a26a7 100644 --- a/gui/app/pods/search/controller.js +++ b/gui/app/pods/search/controller.js @@ -9,9 +9,12 @@ // // https://documize.com +import { inject as service } from '@ember/service'; import Controller from '@ember/controller'; export default Controller.extend({ + appMeta: service(), + queryParams: ['filter', 'matchDoc', 'matchContent', 'matchTag', 'matchFile', 'slog'], filter: '', matchDoc: true, diff --git a/gui/app/pods/search/route.js b/gui/app/pods/search/route.js index c4b39a8b..5efb5ab3 100644 --- a/gui/app/pods/search/route.js +++ b/gui/app/pods/search/route.js @@ -9,8 +9,8 @@ // // https://documize.com -import Route from '@ember/routing/route'; import AuthenticatedRouteMixin from 'ember-simple-auth/mixins/authenticated-route-mixin'; +import Route from '@ember/routing/route'; export default Route.extend(AuthenticatedRouteMixin, { activate() { diff --git a/gui/app/pods/search/template.hbs b/gui/app/pods/search/template.hbs index 2eac0359..f0c96321 100644 --- a/gui/app/pods/search/template.hbs +++ b/gui/app/pods/search/template.hbs @@ -10,22 +10,36 @@ {{/layout/middle-zone-sidebar}} diff --git a/gui/app/services/app-meta.js b/gui/app/services/app-meta.js index f40fdb10..8b6f9998 100644 --- a/gui/app/services/app-meta.js +++ b/gui/app/services/app-meta.js @@ -36,6 +36,7 @@ export default Service.extend({ setupMode: false, secureMode: false, maxTags: 3, + storageProvider: '', // for major.minor semver release detection // for bugfix releases, only admin is made aware of new release and end users see no What's New messaging diff --git a/model/org/meta.go b/model/org/meta.go index bd26cb7c..51b1f37b 100644 --- a/model/org/meta.go +++ b/model/org/meta.go @@ -12,8 +12,9 @@ package org import ( - "github.com/documize/community/core/env" "time" + + "github.com/documize/community/core/env" ) // SitemapDocument details a document that can be exposed via Sitemap. @@ -27,17 +28,18 @@ type SitemapDocument struct { // SiteMeta holds information associated with an Organization. type SiteMeta struct { - OrgID string `json:"orgId"` - Title string `json:"title"` - Message string `json:"message"` - URL string `json:"url"` - AllowAnonymousAccess bool `json:"allowAnonymousAccess"` - AuthProvider string `json:"authProvider"` - AuthConfig string `json:"authConfig"` - Version string `json:"version"` - MaxTags int `json:"maxTags"` - Edition string `json:"edition"` - Valid bool `json:"valid"` - ConversionEndpoint string `json:"conversionEndpoint"` - License env.License `json:"license"` + OrgID string `json:"orgId"` + Title string `json:"title"` + Message string `json:"message"` + URL string `json:"url"` + AllowAnonymousAccess bool `json:"allowAnonymousAccess"` + AuthProvider string `json:"authProvider"` + AuthConfig string `json:"authConfig"` + Version string `json:"version"` + MaxTags int `json:"maxTags"` + Edition string `json:"edition"` + Valid bool `json:"valid"` + ConversionEndpoint string `json:"conversionEndpoint"` + License env.License `json:"license"` + Storage env.StoreType `json:"storageProvider"` }