diff --git a/README.md b/README.md index e66a56f2..22a3f886 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -> We're committed to providing frequent product releases to ensure self-host customers enjoy the same product as our cloud/SaaS customers. +> We provide frequent product releases ensuring self-host customers enjoy the same features as our cloud/SaaS customers. > > Harvey Kandola, CEO & Founder, Documize Inc. @@ -58,19 +58,32 @@ Space view. ## Latest version -[Community edition: v1.70.0](https://github.com/documize/community/releases) +[Community edition: v1.71.0](https://github.com/documize/community/releases) -[Enterprise edition: v1.72.0](https://documize.com/downloads) +[Enterprise edition: v1.73.0](https://documize.com/downloads) ## OS support -Documize runs on the following: +Documize can be installed and run on: - Linux - Windows - macOS -# Browser support +Heck, Documize will probably run just fine on a Raspberry Pi 3. + +## Database support + +Documize supports the following database systems: + +- PostgreSQL (v9.6+) +- MySQL (v5.7.10+ and v8.0.0+) +- Percona (v5.7.16-10+) +- MariaDB (10.3.0+) + +Coming soon: Microsoft SQL Server 2017 (Linux/Windows). + +## Browser support Documize supports the following (evergreen) browsers: @@ -78,6 +91,8 @@ Documize supports the following (evergreen) browsers: - Firefox - Safari - Brave +- Vivaldi +- Opera - MS Edge (16+) ## Technology stack @@ -87,14 +102,6 @@ Documize is built with the following technologies: - EmberJS (v3.1.2) - Go (v1.10.3) -...and supports the following databases: - -- MySQL (v5.7.10+) -- Percona (v5.7.16-10+) -- MariaDB (10.3.0+) - -Coming soon, PostgreSQL and Microsoft SQL Server database support. - ## Authentication options Besides email/password login, you can also leverage the following options. diff --git a/core/database/installer.go b/core/database/installer.go index 1757ee9e..f4b26d75 100644 --- a/core/database/installer.go +++ b/core/database/installer.go @@ -174,13 +174,13 @@ func runScripts(runtime *env.Runtime, tx *sqlx.Tx, scripts []Script) (err error) } // executeSQL runs specified SQL commands. -func executeSQL(tx *sqlx.Tx, st env.StoreType, variant string, SQLfile []byte) error { +func executeSQL(tx *sqlx.Tx, st env.StoreType, variant env.StoreType, SQLfile []byte) error { // Turn SQL file contents into runnable SQL statements. stmts := getStatements(SQLfile) for _, stmt := range stmts { // MariaDB has no specific JSON column type (but has JSON queries) - if st == env.StoreTypeMySQL && variant == "mariadb" { + if st == env.StoreTypeMySQL && variant == env.StoreTypeMariaDB { stmt = strings.Replace(stmt, "` JSON", "` TEXT", -1) } diff --git a/core/database/scripts/postgresql/db_00001.sql b/core/database/scripts/postgresql/db_00001.sql index 467b3acd..64255f7f 100644 --- a/core/database/scripts/postgresql/db_00001.sql +++ b/core/database/scripts/postgresql/db_00001.sql @@ -287,12 +287,13 @@ CREATE TABLE dmz_search ( c_itemid varchar(16) COLLATE ucs_basic NOT NULL DEFAULT '', c_itemtype varchar(10) COLLATE ucs_basic NOT NULL, c_content text COLLATE ucs_basic, + c_token TSVECTOR, c_created timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, UNIQUE (id) ); CREATE INDEX idx_search_1 ON dmz_search (c_orgid); CREATE INDEX idx_search_2 ON dmz_search (c_docid); -CREATE INDEX idx_search_3 ON dmz_search USING GIN (to_tsvector('english', c_content)); +CREATE INDEX idx_search_3 ON dmz_search USING GIN(c_token); DROP TABLE IF EXISTS dmz_section; CREATE TABLE dmz_section ( diff --git a/core/env/provider.go b/core/env/provider.go index db9097df..969771b4 100644 --- a/core/env/provider.go +++ b/core/env/provider.go @@ -38,7 +38,7 @@ type StoreProvider interface { Type() StoreType // TypeVariant returns flavor of database provider. - TypeVariant() string + TypeVariant() StoreType // SQL driver name used to open DB connection. DriverName() string diff --git a/domain/attachment/store.go b/domain/attachment/store.go index 0cecf68b..2f22611b 100644 --- a/domain/attachment/store.go +++ b/domain/attachment/store.go @@ -12,6 +12,7 @@ package attachment import ( + "database/sql" "strings" "time" @@ -75,8 +76,13 @@ func (s Store) GetAttachments(ctx domain.RequestContext, docID string) (a []atta ORDER BY c_filename`), ctx.OrgID, docID) + if err == sql.ErrNoRows { + err = nil + a = []attachment.Attachment{} + } if err != nil { err = errors.Wrap(err, "execute select attachments") + return } return @@ -94,6 +100,11 @@ func (s Store) GetAttachmentsWithData(ctx domain.RequestContext, docID string) ( ORDER BY c_filename`), ctx.OrgID, docID) + if err == sql.ErrNoRows { + err = nil + a = []attachment.Attachment{} + } + if err != nil { err = errors.Wrap(err, "execute select attachments with data") } diff --git a/domain/category/store.go b/domain/category/store.go index 32b9313e..84be3150 100644 --- a/domain/category/store.go +++ b/domain/category/store.go @@ -202,7 +202,7 @@ func (s Store) RemoveDocumentCategories(ctx domain.RequestContext, documentID st // DeleteBySpace removes all category and category associations for given space. func (s Store) DeleteBySpace(ctx domain.RequestContext, spaceID string) (rows int64, err error) { - s1 := fmt.Sprintf("DELETE FROM categorymember WHERE c_orgid='%s' AND c_groupid='%s'", ctx.OrgID, spaceID) + s1 := fmt.Sprintf("DELETE FROM dmz_category_member WHERE c_orgid='%s' AND c_spaceid='%s'", ctx.OrgID, spaceID) s.DeleteWhere(ctx.Transaction, s1) s2 := fmt.Sprintf("DELETE FROM dmz_category WHERE c_orgid='%s' AND c_spaceid='%s'", ctx.OrgID, spaceID) diff --git a/domain/document/store.go b/domain/document/store.go index 00aa5ccb..e1068114 100644 --- a/domain/document/store.go +++ b/domain/document/store.go @@ -245,28 +245,28 @@ func (s Store) MoveActivity(ctx domain.RequestContext, documentID, oldSpaceID, n // Delete removes the specified document. // Remove document pages, revisions, attachments, updates the search subsystem. func (s Store) Delete(ctx domain.RequestContext, documentID string) (rows int64, err error) { - rows, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_section WHERE c_docid=\"%s\" AND c_orgid=\"%s\"", documentID, ctx.OrgID)) + rows, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_section WHERE c_docid='%s' AND c_orgid='%s'", documentID, ctx.OrgID)) if err != nil { return } - _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_section_revision WHERE c_docid=\"%s\" AND c_orgid=\"%s\"", documentID, ctx.OrgID)) + _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_section_revision WHERE c_docid='%s' AND c_orgid='%s'", documentID, ctx.OrgID)) if err != nil { return } - _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_attachment WHERE c_docid=\"%s\" AND c_orgid=\"%s\"", documentID, ctx.OrgID)) + _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_attachment WHERE c_docid='%s' AND c_orgid='%s'", documentID, ctx.OrgID)) if err != nil { return } - _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_category_member WHERE c_docid=\"%s\" AND c_orgid=\"%s\"", documentID, ctx.OrgID)) + _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_category_member WHERE c_docid='%s' AND c_orgid='%s'", documentID, ctx.OrgID)) if err != nil { return } - _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_vote WHERE c_docid=\"%s\" AND c_orgid=\"%s\"", documentID, ctx.OrgID)) + _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_vote WHERE c_docid='%s' AND c_orgid='%s'", documentID, ctx.OrgID)) if err != nil { return } @@ -277,23 +277,23 @@ func (s Store) Delete(ctx domain.RequestContext, documentID string) (rows int64, // DeleteBySpace removes all documents for given space. // Remove document pages, revisions, attachments, updates the search subsystem. func (s Store) DeleteBySpace(ctx domain.RequestContext, spaceID string) (rows int64, err error) { - rows, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_section WHERE c_docid IN (SELECT c_refid FROM dmz_doc WHERE c_spaceid=\"%s\" AND c_orgid=\"%s\")", spaceID, ctx.OrgID)) + rows, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_section WHERE c_docid IN (SELECT c_refid FROM dmz_doc WHERE c_spaceid='%s' AND c_orgid='%s')", spaceID, ctx.OrgID)) if err != nil { return } - _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_section_revision WHERE c_docid IN (SELECT c_refid FROM dmz_doc WHERE c_spaceid=\"%s\" AND c_orgid=\"%s\")", spaceID, ctx.OrgID)) + _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_section_revision WHERE c_docid IN (SELECT c_refid FROM dmz_doc WHERE c_spaceid='%s' AND c_orgid='%s')", spaceID, ctx.OrgID)) if err != nil { return } - _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_attachment WHERE c_docid IN (SELECT c_refid FROM dmz_doc WHERE c_spaceid=\"%s\" AND c_orgid=\"%s\")", spaceID, ctx.OrgID)) + _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_attachment WHERE c_docid IN (SELECT c_refid FROM dmz_doc WHERE c_spaceid='%s' AND c_orgid='%s')", spaceID, ctx.OrgID)) if err != nil { return } - _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_vote WHERE c_docid IN (SELECT c_refid FROM dmz_doc WHERE c_spaceid=\"%s\" AND c_orgid=\"%s\")", spaceID, ctx.OrgID)) + _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_vote WHERE c_docid IN (SELECT c_refid FROM dmz_doc WHERE c_spaceid='%s' AND c_orgid='%s')", spaceID, ctx.OrgID)) if err != nil { return } @@ -332,7 +332,7 @@ func (s Store) GetVersions(ctx domain.RequestContext, groupID string) (v []doc.V // Any existing vote by the user is replaced. func (s Store) Vote(ctx domain.RequestContext, refID, orgID, documentID, userID string, vote int) (err error) { _, err = s.DeleteWhere(ctx.Transaction, - fmt.Sprintf("DELETE FROM dmz_doc_vote WHERE c_orgid=\"%s\" AND c_docid=\"%s\" AND c_voter=\"%s\"", + fmt.Sprintf("DELETE FROM dmz_doc_vote WHERE c_orgid='%s' AND c_docid='%s' AND c_voter='%s'", orgID, documentID, userID)) if err != nil { s.Runtime.Log.Error("store.Vote", err) diff --git a/domain/group/store.go b/domain/group/store.go index b3bacd7c..ff93e9bb 100644 --- a/domain/group/store.go +++ b/domain/group/store.go @@ -104,7 +104,7 @@ func (s Store) Delete(ctx domain.RequestContext, refID string) (rows int64, err if err != nil { return } - return s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_group_member WHERE c_orgid=\"%s\" AND c_groupid=\"%s\"", ctx.OrgID, refID)) + return s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_group_member WHERE c_orgid='%s' AND c_groupid='%s'", ctx.OrgID, refID)) } // GetGroupMembers returns all user associated with given group. @@ -143,8 +143,12 @@ func (s Store) JoinGroup(ctx domain.RequestContext, groupID, userID string) (err // LeaveGroup removes user from group. func (s Store) LeaveGroup(ctx domain.RequestContext, groupID, userID string) (err error) { - _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_group_member WHERE c_orgid=\"%s\" AND c_groupid=\"%s\" AND c_userid=\"%s\"", + _, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_group_member WHERE c_orgid='%s' AND c_groupid='%s' AND c_userid='%s'", ctx.OrgID, groupID, userID)) + + if err == sql.ErrNoRows { + err = nil + } if err != nil { err = errors.Wrap(err, "clear group member") } diff --git a/domain/link/store.go b/domain/link/store.go index cf0137a7..94159e67 100644 --- a/domain/link/store.go +++ b/domain/link/store.go @@ -137,12 +137,12 @@ func (s Store) MarkOrphanAttachmentLink(ctx domain.RequestContext, attachmentID // DeleteSourcePageLinks removes saved links for given source. func (s Store) DeleteSourcePageLinks(ctx domain.RequestContext, pageID string) (rows int64, err error) { - return s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_link WHERE c_orgid=\"%s\" AND c_sourcesectionid=\"%s\"", ctx.OrgID, pageID)) + return s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_link WHERE c_orgid='%s' AND c_sourcesectionid='%s'", ctx.OrgID, pageID)) } // DeleteSourceDocumentLinks removes saved links for given document. func (s Store) DeleteSourceDocumentLinks(ctx domain.RequestContext, documentID string) (rows int64, err error) { - return s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_link WHERE c_orgid=\"%s\" AND c_sourcedocid=\"%s\"", ctx.OrgID, documentID)) + return s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_link WHERE c_orgid='%s' AND c_sourcedocid='%s'", ctx.OrgID, documentID)) } // DeleteLink removes saved link from the store. diff --git a/domain/meta/endpoint.go b/domain/meta/endpoint.go index ecd926d2..c6f4534f 100644 --- a/domain/meta/endpoint.go +++ b/domain/meta/endpoint.go @@ -62,6 +62,7 @@ func (h *Handler) Meta(w http.ResponseWriter, r *http.Request) { data.Valid = h.Runtime.Product.License.Valid data.ConversionEndpoint = org.ConversionEndpoint data.License = h.Runtime.Product.License + data.Storage = h.Runtime.StoreProvider.Type() // Strip secrets data.AuthConfig = auth.StripAuthSecrets(h.Runtime, org.AuthProvider, org.AuthConfig) @@ -213,6 +214,19 @@ func (h *Handler) rebuildSearchIndex(ctx domain.RequestContext) { for i := range docs { d := docs[i] + doc, err := h.Store.Document.Get(ctx, d) + if err != nil { + h.Runtime.Log.Error(method, err) + return + } + at, err := h.Store.Attachment.GetAttachments(ctx, d) + if err != nil { + h.Runtime.Log.Error(method, err) + return + } + + h.Indexer.IndexDocument(ctx, doc, at) + pages, err := h.Store.Meta.GetDocumentPages(ctx, d) if err != nil { h.Runtime.Log.Error(method, err) diff --git a/domain/meta/store.go b/domain/meta/store.go index 879dad3e..d7d6df87 100644 --- a/domain/meta/store.go +++ b/domain/meta/store.go @@ -54,6 +54,10 @@ func (s Store) GetDocumentPages(ctx domain.RequestContext, documentID string) (p WHERE c_docid=? AND (c_status=0 OR ((c_status=4 OR c_status=2) AND c_relativeid=''))`), documentID) + if err == sql.ErrNoRows { + err = nil + p = []page.Page{} + } if err != nil { err = errors.Wrap(err, "failed to get instance document pages") } diff --git a/domain/permission/store.go b/domain/permission/store.go index 4a256bb1..11d9b9d1 100644 --- a/domain/permission/store.go +++ b/domain/permission/store.go @@ -33,7 +33,8 @@ type Store struct { func (s Store) AddPermission(ctx domain.RequestContext, r permission.Permission) (err error) { r.Created = time.Now().UTC() - _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_permission (c_orgid, c_who, c_whoid, c_action, c_scope, c_location, c_refid, c_created) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"), + _, err = ctx.Transaction.Exec(s.Bind(`INSERT INTO dmz_permission + (c_orgid, c_who, c_whoid, c_action, c_scope, c_location, c_refid, c_created) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`), r.OrgID, string(r.Who), r.WhoID, string(r.Action), string(r.Scope), string(r.Location), r.RefID, r.Created) if err != nil { @@ -278,7 +279,7 @@ func (s Store) DeleteSpacePermissions(ctx domain.RequestContext, spaceID string) // DeleteUserSpacePermissions removes all roles for the specified user, for the specified space. func (s Store) DeleteUserSpacePermissions(ctx domain.RequestContext, spaceID, userID string) (rows int64, err error) { - sql := fmt.Sprintf("DELETE FROM dmz_permission WHERE c_orgid='%s' AND c_location='space' AND c_refid='%s' c_who='user' AND c_whoid='%s'", + sql := fmt.Sprintf("DELETE FROM dmz_permission WHERE c_orgid='%s' AND c_location='space' AND c_refid='%s' AND c_who='user' AND c_whoid='%s'", ctx.OrgID, spaceID, userID) return s.DeleteWhere(ctx.Transaction, sql) diff --git a/domain/pin/store.go b/domain/pin/store.go index d33063e5..23a4b665 100644 --- a/domain/pin/store.go +++ b/domain/pin/store.go @@ -120,10 +120,10 @@ func (s Store) DeletePin(ctx domain.RequestContext, id string) (rows int64, err // DeletePinnedSpace removes any pins for specified space. func (s Store) DeletePinnedSpace(ctx domain.RequestContext, spaceID string) (rows int64, err error) { - return s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_pin WHERE c_orgid=\"%s\" AND c_spaceid=\"%s\"", ctx.OrgID, spaceID)) + return s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_pin WHERE c_orgid='%s' AND c_spaceid='%s'", ctx.OrgID, spaceID)) } // DeletePinnedDocument removes any pins for specified document. func (s Store) DeletePinnedDocument(ctx domain.RequestContext, documentID string) (rows int64, err error) { - return s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_pin WHERE c_orgid=\"%s\" AND c_docid=\"%s\"", ctx.OrgID, documentID)) + return s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_pin WHERE c_orgid='%s' AND c_docid='%s'", ctx.OrgID, documentID)) } diff --git a/domain/search/search.go b/domain/search/search.go index 90cf572f..56088550 100644 --- a/domain/search/search.go +++ b/domain/search/search.go @@ -82,7 +82,12 @@ func (m *Indexer) IndexContent(ctx domain.RequestContext, p page.Page) { return } - ctx.Transaction.Commit() + err = ctx.Transaction.Commit() + if err != nil { + ctx.Transaction.Rollback() + m.runtime.Log.Error(method, err) + return + } } // DeleteContent removes all search entries for specific document content. diff --git a/domain/search/store.go b/domain/search/store.go index c108d75b..549c6589 100644 --- a/domain/search/store.go +++ b/domain/search/store.go @@ -39,43 +39,68 @@ type Store struct { // IndexDocument adds search index entries for document inserting title, tags and attachments as // searchable items. Any existing document entries are removed. func (s Store) IndexDocument(ctx domain.RequestContext, doc doc.Document, a []attachment.Attachment) (err error) { + method := "search.IndexDocument" + // remove previous search entries _, err = ctx.Transaction.Exec(s.Bind("DELETE FROM dmz_search WHERE c_orgid=? AND c_docid=? AND (c_itemtype='doc' OR c_itemtype='file' OR c_itemtype='tag')"), ctx.OrgID, doc.RefID) - - if err != nil { + if err != nil && err != sql.ErrNoRows { err = errors.Wrap(err, "execute delete document index entries") + s.Runtime.Log.Error(method, err) + return } // insert doc title - _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content) VALUES (?, ?, ?, ?, ?)"), - ctx.OrgID, doc.RefID, "", "doc", doc.Name) - if err != nil { + if s.Runtime.StoreProvider.Type() == env.StoreTypePostgreSQL { + _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content, c_token) VALUES (?, ?, ?, ?, ?, to_tsvector(?))"), + ctx.OrgID, doc.RefID, "", "doc", doc.Name, doc.Name) + + } else { + _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content) VALUES (?, ?, ?, ?, ?)"), + ctx.OrgID, doc.RefID, "", "doc", doc.Name) + } + if err != nil && err != sql.ErrNoRows { err = errors.Wrap(err, "execute insert document title entry") + s.Runtime.Log.Error(method, err) + return } // insert doc tags tags := strings.Split(doc.Tags, "#") for _, t := range tags { + t = strings.TrimSpace(t) if len(t) == 0 { continue } - _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content) VALUES (?, ?, ?, ?, ?)"), - ctx.OrgID, doc.RefID, "", "tag", t) + if s.Runtime.StoreProvider.Type() == env.StoreTypePostgreSQL { + _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content, c_token) VALUES (?, ?, ?, ?, ?, to_tsvector(?))"), + ctx.OrgID, doc.RefID, "", "tag", t, t) - if err != nil { + } else { + _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content) VALUES (?, ?, ?, ?, ?)"), + ctx.OrgID, doc.RefID, "", "tag", t) + } + if err != nil && err != sql.ErrNoRows { err = errors.Wrap(err, "execute insert document tag entry") + s.Runtime.Log.Error(method, err) return } } for _, file := range a { - _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content) VALUES (?, ?, ?, ?, ?)"), - ctx.OrgID, doc.RefID, file.RefID, "file", file.Filename) + if s.Runtime.StoreProvider.Type() == env.StoreTypePostgreSQL { + _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content, c_token) VALUES (?, ?, ?, ?, ?, to_tsvector(?))"), + ctx.OrgID, doc.RefID, file.RefID, "file", file.Filename, file.Filename) - if err != nil { + } else { + _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content) VALUES (?, ?, ?, ?, ?)"), + ctx.OrgID, doc.RefID, file.RefID, "file", file.Filename) + } + if err != nil && err != sql.ErrNoRows { err = errors.Wrap(err, "execute insert document file entry") + s.Runtime.Log.Error(method, err) + return } } @@ -84,11 +109,14 @@ func (s Store) IndexDocument(ctx domain.RequestContext, doc doc.Document, a []at // DeleteDocument removes all search entries for document. func (s Store) DeleteDocument(ctx domain.RequestContext, ID string) (err error) { + method := "search.DeleteDocument" + _, err = ctx.Transaction.Exec(s.Bind("DELETE FROM dmz_search WHERE c_orgid=? AND c_docid=?"), ctx.OrgID, ID) - if err != nil { + if err != nil && err != sql.ErrNoRows { err = errors.Wrap(err, "execute delete document entries") + s.Runtime.Log.Error(method, err) } return @@ -97,6 +125,8 @@ func (s Store) DeleteDocument(ctx domain.RequestContext, ID string) (err error) // IndexContent adds search index entry for document context. // Any existing document entries are removed. func (s Store) IndexContent(ctx domain.RequestContext, p page.Page) (err error) { + method := "search.IndexContent" + // we do not index pending pages if p.Status == workflow.ChangePending || p.Status == workflow.ChangePendingNew { return @@ -106,28 +136,49 @@ func (s Store) IndexContent(ctx domain.RequestContext, p page.Page) (err error) _, err = ctx.Transaction.Exec(s.Bind("DELETE FROM dmz_search WHERE c_orgid=? AND c_docid=? AND c_itemid=? AND c_itemtype='page'"), ctx.OrgID, p.DocumentID, p.RefID) - if err != nil { + if err != nil && err != sql.ErrNoRows { err = errors.Wrap(err, "execute delete document content entry") + s.Runtime.Log.Error(method, err) + return } + err = nil // prepare content content, err := stringutil.HTML(p.Body).Text(false) if err != nil { err = errors.Wrap(err, "search strip HTML failed") + s.Runtime.Log.Error(method, err) return } content = strings.TrimSpace(content) - _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content) VALUES (?, ?, ?, ?, ?)"), - ctx.OrgID, p.DocumentID, p.RefID, "page", content) - if err != nil { - err = errors.Wrap(err, "execute insert document content entry") - } + if s.Runtime.StoreProvider.Type() == env.StoreTypePostgreSQL { + _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content, c_token) VALUES (?, ?, ?, ?, ?, to_tsvector(?))"), + ctx.OrgID, p.DocumentID, p.RefID, "page", content, content) - _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content) VALUES (?, ?, ?, ?, ?)"), - ctx.OrgID, p.DocumentID, p.RefID, "page", p.Name) - if err != nil { - err = errors.Wrap(err, "execute insert document page title entry") + } else { + _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content) VALUES (?, ?, ?, ?, ?)"), + ctx.OrgID, p.DocumentID, p.RefID, "page", content) + } + if err != nil && err != sql.ErrNoRows { + err = errors.Wrap(err, "execute insert section content entry") + s.Runtime.Log.Error(method, err) + return + } + err = nil + + if s.Runtime.StoreProvider.Type() == env.StoreTypePostgreSQL { + _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content, c_token) VALUES (?, ?, ?, ?, ?, to_tsvector(?))"), + ctx.OrgID, p.DocumentID, p.RefID, "page", p.Name, p.Name) + + } else { + _, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content) VALUES (?, ?, ?, ?, ?)"), + ctx.OrgID, p.DocumentID, p.RefID, "page", p.Name) + } + if err != nil && err != sql.ErrNoRows { + err = errors.Wrap(err, "execute insert section title entry") + s.Runtime.Log.Error(method, err) + return } return nil @@ -135,18 +186,23 @@ func (s Store) IndexContent(ctx domain.RequestContext, p page.Page) (err error) // DeleteContent removes all search entries for specific document content. func (s Store) DeleteContent(ctx domain.RequestContext, pageID string) (err error) { + method := "search.DeleteContent" + // remove all search entries var stmt1 *sqlx.Stmt stmt1, err = ctx.Transaction.Preparex(s.Bind("DELETE FROM dmz_search WHERE c_orgid=? AND c_itemid=? AND c_itemtype=?")) defer streamutil.Close(stmt1) - if err != nil { + + if err != nil && err != sql.ErrNoRows { err = errors.Wrap(err, "prepare delete document content entry") + s.Runtime.Log.Error(method, err) return } _, err = stmt1.Exec(ctx.OrgID, pageID, "page") - if err != nil { + if err != nil && err != sql.ErrNoRows { err = errors.Wrap(err, "execute delete document content entry") + s.Runtime.Log.Error(method, err) return } @@ -220,9 +276,17 @@ func (s Store) matchFullText(ctx domain.RequestContext, keywords, itemType strin switch s.Runtime.StoreProvider.Type() { case env.StoreTypeMySQL: - fts = " AND MATCH(s.c_content) AGAINST(? IN BOOLEAN MODE)" + fts = " AND MATCH(s.c_content) AGAINST(? IN BOOLEAN MODE) " case env.StoreTypePostgreSQL: - fts = "" + // By default, we expect no Postgres full text search operators. + parser := "plainto_tsquery" + // If we find operators then we have to use correct query processor. + operator := strings.ContainsAny(keywords, "!()&|*'`\":<->") + if operator { + parser = "to_tsquery" + } + + fts = fmt.Sprintf(" AND s.c_token @@ %s(?) ", parser) } sql1 := s.Bind(` @@ -279,7 +343,7 @@ func (s Store) matchLike(ctx domain.RequestContext, keywords, itemType string) ( keywords = strings.Replace(keywords, "'", "", -1) keywords = strings.Replace(keywords, "\"", "", -1) keywords = strings.Replace(keywords, "%", "", -1) - keywords = fmt.Sprintf("%%%s%%", keywords) + keywords = fmt.Sprintf("%%%s%%", strings.ToLower(keywords)) sql1 := s.Bind(`SELECT s.id, s.c_orgid AS orgid, s.c_docid AS documentid, s.c_itemid AS itemid, s.c_itemtype AS itemtype, @@ -304,7 +368,7 @@ func (s Store) matchLike(ctx domain.RequestContext, keywords, itemType string) ( AND p.c_location='space' AND (r.c_userid=? OR r.c_userid='0') ) ) - AND s.c_content LIKE ?`) + AND LOWER(s.c_content) LIKE ?`) err = s.Runtime.Db.Select(&r, sql1, diff --git a/domain/space/endpoint.go b/domain/space/endpoint.go index ba106d92..8ccf141e 100644 --- a/domain/space/endpoint.go +++ b/domain/space/endpoint.go @@ -470,7 +470,13 @@ func (h *Handler) Update(w http.ResponseWriter, r *http.Request) { // If newly marked Everyone space, ensure everyone has permission if prev.Type != space.ScopePublic && sp.Type == space.ScopePublic { - h.Store.Permission.DeleteUserSpacePermissions(ctx, sp.RefID, user.EveryoneUserID) + _, err = h.Store.Permission.DeleteUserSpacePermissions(ctx, sp.RefID, user.EveryoneUserID) + if err != nil { + ctx.Transaction.Rollback() + response.WriteServerError(w, method, err) + h.Runtime.Log.Error(method, err) + return + } perm := permission.Permission{} perm.OrgID = sp.OrgID diff --git a/domain/store/context.go b/domain/store/context.go index 58631832..75a6e993 100644 --- a/domain/store/context.go +++ b/domain/store/context.go @@ -12,6 +12,7 @@ package store import ( + "database/sql" "fmt" "github.com/documize/community/core/env" "github.com/jmoiron/sqlx" @@ -44,12 +45,13 @@ func (c *Context) Bind(sql string) string { func (c *Context) Delete(tx *sqlx.Tx, table string, id string) (rows int64, err error) { result, err := tx.Exec(c.Bind("DELETE FROM "+table+" WHERE c_refid=?"), id) - if err != nil { + if err != nil && err != sql.ErrNoRows { err = errors.Wrap(err, fmt.Sprintf("unable to delete row in table %s", table)) return } rows, err = result.RowsAffected() + err = nil return } @@ -58,12 +60,13 @@ func (c *Context) Delete(tx *sqlx.Tx, table string, id string) (rows int64, err func (c *Context) DeleteConstrained(tx *sqlx.Tx, table string, orgID, id string) (rows int64, err error) { result, err := tx.Exec(c.Bind("DELETE FROM "+table+" WHERE c_orgid=? AND c_refid=?"), orgID, id) - if err != nil { + if err != nil && err != sql.ErrNoRows { err = errors.Wrap(err, fmt.Sprintf("unable to delete row in table %s", table)) return } rows, err = result.RowsAffected() + err = nil return } @@ -72,12 +75,13 @@ func (c *Context) DeleteConstrained(tx *sqlx.Tx, table string, orgID, id string) func (c *Context) DeleteConstrainedWithID(tx *sqlx.Tx, table string, orgID, id string) (rows int64, err error) { result, err := tx.Exec(c.Bind("DELETE FROM "+table+" WHERE c_orgid=? AND id=?"), orgID, id) - if err != nil { + if err != nil && err != sql.ErrNoRows { err = errors.Wrap(err, fmt.Sprintf("unable to delete row in table %s", table)) return } rows, err = result.RowsAffected() + err = nil return } @@ -86,12 +90,13 @@ func (c *Context) DeleteConstrainedWithID(tx *sqlx.Tx, table string, orgID, id s func (c *Context) DeleteWhere(tx *sqlx.Tx, statement string) (rows int64, err error) { result, err := tx.Exec(statement) - if err != nil { + if err != nil && err != sql.ErrNoRows { err = errors.Wrap(err, fmt.Sprintf("unable to delete rows: %s", statement)) return } rows, err = result.RowsAffected() + err = nil return } diff --git a/edition/storage/mysql.go b/edition/storage/mysql.go index 114c22fd..d82e58db 100644 --- a/edition/storage/mysql.go +++ b/edition/storage/mysql.go @@ -43,13 +43,24 @@ import ( // SetMySQLProvider creates MySQL provider func SetMySQLProvider(r *env.Runtime, s *store.Store) { - // Set up provider specific details and wire up data prividers. - r.StoreProvider = MySQLProvider{ + // Set up provider specific details. + p := MySQLProvider{ ConnectionString: r.Flags.DBConn, - Variant: r.Flags.DBType, + } + switch r.Flags.DBType { + case "mysql": + p.Variant = env.StoreTypeMySQL + case "mariadb": + p.Variant = env.StoreTypeMariaDB + case "percona": + p.Variant = env.StoreTypePercona } - // Wire up data providers! + r.StoreProvider = p + + // Wire up data providers. + + // Account accountStore := account.Store{} accountStore.Runtime = r s.Account = accountStore @@ -146,7 +157,7 @@ type MySQLProvider struct { ConnectionString string // User specified db type (mysql, percona or mariadb). - Variant string + Variant env.StoreType } // Type returns name of provider @@ -155,7 +166,7 @@ func (p MySQLProvider) Type() env.StoreType { } // TypeVariant returns databse flavor -func (p MySQLProvider) TypeVariant() string { +func (p MySQLProvider) TypeVariant() env.StoreType { return p.Variant } diff --git a/edition/storage/postgresql.go b/edition/storage/postgresql.go index 9ca1527a..14f318f6 100644 --- a/edition/storage/postgresql.go +++ b/edition/storage/postgresql.go @@ -45,18 +45,20 @@ type PostgreSQLProvider struct { ConnectionString string // Unused for this provider. - Variant string + Variant env.StoreType } // SetPostgreSQLProvider creates PostgreSQL provider func SetPostgreSQLProvider(r *env.Runtime, s *store.Store) { - // Set up provider specific details and wire up data prividers. + // Set up provider specific details. r.StoreProvider = PostgreSQLProvider{ ConnectionString: r.Flags.DBConn, - Variant: "", + Variant: env.StoreTypePostgreSQL, } - // Wire up data providers! + // Wire up data providers. + + // Account accountStore := account.Store{} accountStore.Runtime = r s.Account = accountStore @@ -153,7 +155,7 @@ func (p PostgreSQLProvider) Type() env.StoreType { } // TypeVariant returns databse flavor -func (p PostgreSQLProvider) TypeVariant() string { +func (p PostgreSQLProvider) TypeVariant() env.StoreType { return p.Variant } diff --git a/gui/app/constants/constants.js b/gui/app/constants/constants.js index cd63062e..4c915c6e 100644 --- a/gui/app/constants/constants.js +++ b/gui/app/constants/constants.js @@ -135,6 +135,12 @@ let constants = EmberObject.extend({ Rejected: 6, Publish: 7, }, + + // Meta + StoreProvider: { // eslint-disable-line ember/avoid-leaking-state-in-ember-objects + MySQL: 'MySQL', + PostgreSQL: 'PostgreSQL', + }, }); export default { constants } diff --git a/gui/app/pods/customize/search/route.js b/gui/app/pods/customize/search/route.js index a2ee81d3..12e37183 100644 --- a/gui/app/pods/customize/search/route.js +++ b/gui/app/pods/customize/search/route.js @@ -29,6 +29,6 @@ export default Route.extend(AuthenticatedRouteMixin, { }, activate() { - this.get('browser').setTitle('Search'); + this.get('browser').setTitle('Search Engine'); } }); diff --git a/gui/app/pods/search/controller.js b/gui/app/pods/search/controller.js index ea37e3a6..6e2a26a7 100644 --- a/gui/app/pods/search/controller.js +++ b/gui/app/pods/search/controller.js @@ -9,9 +9,12 @@ // // https://documize.com +import { inject as service } from '@ember/service'; import Controller from '@ember/controller'; export default Controller.extend({ + appMeta: service(), + queryParams: ['filter', 'matchDoc', 'matchContent', 'matchTag', 'matchFile', 'slog'], filter: '', matchDoc: true, diff --git a/gui/app/pods/search/route.js b/gui/app/pods/search/route.js index c4b39a8b..5efb5ab3 100644 --- a/gui/app/pods/search/route.js +++ b/gui/app/pods/search/route.js @@ -9,8 +9,8 @@ // // https://documize.com -import Route from '@ember/routing/route'; import AuthenticatedRouteMixin from 'ember-simple-auth/mixins/authenticated-route-mixin'; +import Route from '@ember/routing/route'; export default Route.extend(AuthenticatedRouteMixin, { activate() { diff --git a/gui/app/pods/search/template.hbs b/gui/app/pods/search/template.hbs index 2eac0359..f0c96321 100644 --- a/gui/app/pods/search/template.hbs +++ b/gui/app/pods/search/template.hbs @@ -10,22 +10,36 @@ {{/layout/middle-zone-sidebar}} diff --git a/gui/app/services/app-meta.js b/gui/app/services/app-meta.js index f40fdb10..8b6f9998 100644 --- a/gui/app/services/app-meta.js +++ b/gui/app/services/app-meta.js @@ -36,6 +36,7 @@ export default Service.extend({ setupMode: false, secureMode: false, maxTags: 3, + storageProvider: '', // for major.minor semver release detection // for bugfix releases, only admin is made aware of new release and end users see no What's New messaging diff --git a/model/org/meta.go b/model/org/meta.go index bd26cb7c..51b1f37b 100644 --- a/model/org/meta.go +++ b/model/org/meta.go @@ -12,8 +12,9 @@ package org import ( - "github.com/documize/community/core/env" "time" + + "github.com/documize/community/core/env" ) // SitemapDocument details a document that can be exposed via Sitemap. @@ -27,17 +28,18 @@ type SitemapDocument struct { // SiteMeta holds information associated with an Organization. type SiteMeta struct { - OrgID string `json:"orgId"` - Title string `json:"title"` - Message string `json:"message"` - URL string `json:"url"` - AllowAnonymousAccess bool `json:"allowAnonymousAccess"` - AuthProvider string `json:"authProvider"` - AuthConfig string `json:"authConfig"` - Version string `json:"version"` - MaxTags int `json:"maxTags"` - Edition string `json:"edition"` - Valid bool `json:"valid"` - ConversionEndpoint string `json:"conversionEndpoint"` - License env.License `json:"license"` + OrgID string `json:"orgId"` + Title string `json:"title"` + Message string `json:"message"` + URL string `json:"url"` + AllowAnonymousAccess bool `json:"allowAnonymousAccess"` + AuthProvider string `json:"authProvider"` + AuthConfig string `json:"authConfig"` + Version string `json:"version"` + MaxTags int `json:"maxTags"` + Edition string `json:"edition"` + Valid bool `json:"valid"` + ConversionEndpoint string `json:"conversionEndpoint"` + License env.License `json:"license"` + Storage env.StoreType `json:"storageProvider"` }