Docker/documize
Docker/documize
1
0
Fork 0
mirror of https://github.com/documize/community.git synced 2025-07-19 05:09:42 +02:00
Code Releases Activity

Implement PostgreSQL Full Text Search++

Browse source 
1. Full text search supports MySQL, MariaDB, Percona and now PostgreSQL.
2. Changed SQL Variant to typed enum.
3. Changed doc.Versioned from INT to BOOL.
4. Search Reindexer now parses all documents and attachments.
5. Site meta API call returns storage provider type.
6. README prep'ed for PostgreSQL support.
7. DELETE SQL statements ignore zero rows affected.

Closes #100 !!!

Co-Authored-By: Saul S <sauls8t@users.noreply.github.com>
Co-Authored-By: McMatts <matt@documize.com>
This commit is contained in:
HarveyKandola 2018-09-28 16:33:15 +01:00
parent 97beb3f4d3
commit 8a65567169
26 changed files with 274 additions and 113 deletions
Download patch file Download diff file Expand all files Collapse all files

33
README.md
View file

@ -1,4 +1,4 @@
> We're committed to providing frequent product releases to ensure self-host customers enjoy the same product as our cloud/SaaS customers.
> We provide frequent product releases ensuring self-host customers enjoy the same features as our cloud/SaaS customers.
>
> Harvey Kandola, CEO & Founder, Documize Inc.
@ -58,19 +58,32 @@ Space view.
## Latest version
[Community edition: v1.70.0](https://github.com/documize/community/releases)
[Community edition: v1.71.0](https://github.com/documize/community/releases)
[Enterprise edition: v1.72.0](https://documize.com/downloads)
[Enterprise edition: v1.73.0](https://documize.com/downloads)
## OS support
Documize runs on the following:
Documize can be installed and run on:
- Linux
- Windows
- macOS
# Browser support
Heck, Documize will probably run just fine on a Raspberry Pi 3.
## Database support
Documize supports the following database systems:
- PostgreSQL (v9.6+)
- MySQL (v5.7.10+ and v8.0.0+)
- Percona (v5.7.16-10+)
- MariaDB (10.3.0+)
Coming soon: Microsoft SQL Server 2017 (Linux/Windows).
## Browser support
Documize supports the following (evergreen) browsers:
@ -78,6 +91,8 @@ Documize supports the following (evergreen) browsers:
- Firefox
- Safari
- Brave
- Vivaldi
- Opera
- MS Edge (16+)
## Technology stack
@ -87,14 +102,6 @@ Documize is built with the following technologies:
- EmberJS (v3.1.2)
- Go (v1.10.3)
...and supports the following databases:
- MySQL (v5.7.10+)
- Percona (v5.7.16-10+)
- MariaDB (10.3.0+)
Coming soon, PostgreSQL and Microsoft SQL Server database support.
## Authentication options
Besides email/password login, you can also leverage the following options.

4
core/database/installer.go
View file

@ -174,13 +174,13 @@ func runScripts(runtime *env.Runtime, tx *sqlx.Tx, scripts []Script) (err error)
}
// executeSQL runs specified SQL commands.
func executeSQL(tx *sqlx.Tx, st env.StoreType, variant string, SQLfile []byte) error {
func executeSQL(tx *sqlx.Tx, st env.StoreType, variant env.StoreType, SQLfile []byte) error {
// Turn SQL file contents into runnable SQL statements.
stmts := getStatements(SQLfile)
for _, stmt := range stmts {
// MariaDB has no specific JSON column type (but has JSON queries)
if st == env.StoreTypeMySQL && variant == "mariadb" {
if st == env.StoreTypeMySQL && variant == env.StoreTypeMariaDB {
stmt = strings.Replace(stmt, "` JSON", "` TEXT", -1)
}

3
core/database/scripts/postgresql/db_00001.sql
View file

@ -287,12 +287,13 @@ CREATE TABLE dmz_search (
c_itemid varchar(16) COLLATE ucs_basic NOT NULL DEFAULT '',
c_itemtype varchar(10) COLLATE ucs_basic NOT NULL,
c_content text COLLATE ucs_basic,
c_token TSVECTOR,
c_created timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
UNIQUE (id)
);
CREATE INDEX idx_search_1 ON dmz_search (c_orgid);
CREATE INDEX idx_search_2 ON dmz_search (c_docid);
CREATE INDEX idx_search_3 ON dmz_search USING GIN (to_tsvector('english', c_content));
CREATE INDEX idx_search_3 ON dmz_search USING GIN(c_token);
DROP TABLE IF EXISTS dmz_section;
CREATE TABLE dmz_section (

2
core/env/provider.go vendored
View file

@ -38,7 +38,7 @@ type StoreProvider interface {
Type() StoreType
// TypeVariant returns flavor of database provider.
TypeVariant() string
TypeVariant() StoreType
// SQL driver name used to open DB connection.
DriverName() string

11
domain/attachment/store.go
View file

@ -12,6 +12,7 @@
package attachment
import (
"database/sql"
"strings"
"time"
@ -75,8 +76,13 @@ func (s Store) GetAttachments(ctx domain.RequestContext, docID string) (a []atta
ORDER BY c_filename`),
ctx.OrgID, docID)
if err == sql.ErrNoRows {
err = nil
a = []attachment.Attachment{}
}
if err != nil {
err = errors.Wrap(err, "execute select attachments")
return
}
return
@ -94,6 +100,11 @@ func (s Store) GetAttachmentsWithData(ctx domain.RequestContext, docID string) (
ORDER BY c_filename`),
ctx.OrgID, docID)
if err == sql.ErrNoRows {
err = nil
a = []attachment.Attachment{}
}
if err != nil {
err = errors.Wrap(err, "execute select attachments with data")
}

2
domain/category/store.go
View file

@ -202,7 +202,7 @@ func (s Store) RemoveDocumentCategories(ctx domain.RequestContext, documentID st
// DeleteBySpace removes all category and category associations for given space.
func (s Store) DeleteBySpace(ctx domain.RequestContext, spaceID string) (rows int64, err error) {
s1 := fmt.Sprintf("DELETE FROM categorymember WHERE c_orgid='%s' AND c_groupid='%s'", ctx.OrgID, spaceID)
s1 := fmt.Sprintf("DELETE FROM dmz_category_member WHERE c_orgid='%s' AND c_spaceid='%s'", ctx.OrgID, spaceID)
s.DeleteWhere(ctx.Transaction, s1)
s2 := fmt.Sprintf("DELETE FROM dmz_category WHERE c_orgid='%s' AND c_spaceid='%s'", ctx.OrgID, spaceID)

20
domain/document/store.go
View file

@ -245,28 +245,28 @@ func (s Store) MoveActivity(ctx domain.RequestContext, documentID, oldSpaceID, n
// Delete removes the specified document.
// Remove document pages, revisions, attachments, updates the search subsystem.
func (s Store) Delete(ctx domain.RequestContext, documentID string) (rows int64, err error) {
rows, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_section WHERE c_docid=\"%s\" AND c_orgid=\"%s\"", documentID, ctx.OrgID))
rows, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_section WHERE c_docid='%s' AND c_orgid='%s'", documentID, ctx.OrgID))
if err != nil {
return
}
_, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_section_revision WHERE c_docid=\"%s\" AND c_orgid=\"%s\"", documentID, ctx.OrgID))
_, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_section_revision WHERE c_docid='%s' AND c_orgid='%s'", documentID, ctx.OrgID))
if err != nil {
return
}
_, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_attachment WHERE c_docid=\"%s\" AND c_orgid=\"%s\"", documentID, ctx.OrgID))
_, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_attachment WHERE c_docid='%s' AND c_orgid='%s'", documentID, ctx.OrgID))
if err != nil {
return
}
_, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_category_member WHERE c_docid=\"%s\" AND c_orgid=\"%s\"", documentID, ctx.OrgID))
_, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_category_member WHERE c_docid='%s' AND c_orgid='%s'", documentID, ctx.OrgID))
if err != nil {
return
}
_, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_vote WHERE c_docid=\"%s\" AND c_orgid=\"%s\"", documentID, ctx.OrgID))
_, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_vote WHERE c_docid='%s' AND c_orgid='%s'", documentID, ctx.OrgID))
if err != nil {
return
}
@ -277,23 +277,23 @@ func (s Store) Delete(ctx domain.RequestContext, documentID string) (rows int64,
// DeleteBySpace removes all documents for given space.
// Remove document pages, revisions, attachments, updates the search subsystem.
func (s Store) DeleteBySpace(ctx domain.RequestContext, spaceID string) (rows int64, err error) {
rows, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_section WHERE c_docid IN (SELECT c_refid FROM dmz_doc WHERE c_spaceid=\"%s\" AND c_orgid=\"%s\")", spaceID, ctx.OrgID))
rows, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_section WHERE c_docid IN (SELECT c_refid FROM dmz_doc WHERE c_spaceid='%s' AND c_orgid='%s')", spaceID, ctx.OrgID))
if err != nil {
return
}
_, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_section_revision WHERE c_docid IN (SELECT c_refid FROM dmz_doc WHERE c_spaceid=\"%s\" AND c_orgid=\"%s\")", spaceID, ctx.OrgID))
_, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_section_revision WHERE c_docid IN (SELECT c_refid FROM dmz_doc WHERE c_spaceid='%s' AND c_orgid='%s')", spaceID, ctx.OrgID))
if err != nil {
return
}
_, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_attachment WHERE c_docid IN (SELECT c_refid FROM dmz_doc WHERE c_spaceid=\"%s\" AND c_orgid=\"%s\")", spaceID, ctx.OrgID))
_, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_attachment WHERE c_docid IN (SELECT c_refid FROM dmz_doc WHERE c_spaceid='%s' AND c_orgid='%s')", spaceID, ctx.OrgID))
if err != nil {
return
}
_, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_vote WHERE c_docid IN (SELECT c_refid FROM dmz_doc WHERE c_spaceid=\"%s\" AND c_orgid=\"%s\")", spaceID, ctx.OrgID))
_, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_vote WHERE c_docid IN (SELECT c_refid FROM dmz_doc WHERE c_spaceid='%s' AND c_orgid='%s')", spaceID, ctx.OrgID))
if err != nil {
return
}
@ -332,7 +332,7 @@ func (s Store) GetVersions(ctx domain.RequestContext, groupID string) (v []doc.V
// Any existing vote by the user is replaced.
func (s Store) Vote(ctx domain.RequestContext, refID, orgID, documentID, userID string, vote int) (err error) {
_, err = s.DeleteWhere(ctx.Transaction,
fmt.Sprintf("DELETE FROM dmz_doc_vote WHERE c_orgid=\"%s\" AND c_docid=\"%s\" AND c_voter=\"%s\"",
fmt.Sprintf("DELETE FROM dmz_doc_vote WHERE c_orgid='%s' AND c_docid='%s' AND c_voter='%s'",
orgID, documentID, userID))
if err != nil {
s.Runtime.Log.Error("store.Vote", err)

8
domain/group/store.go
View file

@ -104,7 +104,7 @@ func (s Store) Delete(ctx domain.RequestContext, refID string) (rows int64, err
if err != nil {
return
}
return s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_group_member WHERE c_orgid=\"%s\" AND c_groupid=\"%s\"", ctx.OrgID, refID))
return s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_group_member WHERE c_orgid='%s' AND c_groupid='%s'", ctx.OrgID, refID))
}
// GetGroupMembers returns all user associated with given group.
@ -143,8 +143,12 @@ func (s Store) JoinGroup(ctx domain.RequestContext, groupID, userID string) (err
// LeaveGroup removes user from group.
func (s Store) LeaveGroup(ctx domain.RequestContext, groupID, userID string) (err error) {
_, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_group_member WHERE c_orgid=\"%s\" AND c_groupid=\"%s\" AND c_userid=\"%s\"",
_, err = s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_group_member WHERE c_orgid='%s' AND c_groupid='%s' AND c_userid='%s'",
ctx.OrgID, groupID, userID))
if err == sql.ErrNoRows {
err = nil
}
if err != nil {
err = errors.Wrap(err, "clear group member")
}

4
domain/link/store.go
View file

@ -137,12 +137,12 @@ func (s Store) MarkOrphanAttachmentLink(ctx domain.RequestContext, attachmentID
// DeleteSourcePageLinks removes saved links for given source.
func (s Store) DeleteSourcePageLinks(ctx domain.RequestContext, pageID string) (rows int64, err error) {
return s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_link WHERE c_orgid=\"%s\" AND c_sourcesectionid=\"%s\"", ctx.OrgID, pageID))
return s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_link WHERE c_orgid='%s' AND c_sourcesectionid='%s'", ctx.OrgID, pageID))
}
// DeleteSourceDocumentLinks removes saved links for given document.
func (s Store) DeleteSourceDocumentLinks(ctx domain.RequestContext, documentID string) (rows int64, err error) {
return s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_link WHERE c_orgid=\"%s\" AND c_sourcedocid=\"%s\"", ctx.OrgID, documentID))
return s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_doc_link WHERE c_orgid='%s' AND c_sourcedocid='%s'", ctx.OrgID, documentID))
}
// DeleteLink removes saved link from the store.

14
domain/meta/endpoint.go
View file

@ -62,6 +62,7 @@ func (h *Handler) Meta(w http.ResponseWriter, r *http.Request) {
data.Valid = h.Runtime.Product.License.Valid
data.ConversionEndpoint = org.ConversionEndpoint
data.License = h.Runtime.Product.License
data.Storage = h.Runtime.StoreProvider.Type()
// Strip secrets
data.AuthConfig = auth.StripAuthSecrets(h.Runtime, org.AuthProvider, org.AuthConfig)
@ -213,6 +214,19 @@ func (h *Handler) rebuildSearchIndex(ctx domain.RequestContext) {
for i := range docs {
d := docs[i]
doc, err := h.Store.Document.Get(ctx, d)
if err != nil {
h.Runtime.Log.Error(method, err)
return
}
at, err := h.Store.Attachment.GetAttachments(ctx, d)
if err != nil {
h.Runtime.Log.Error(method, err)
return
}
h.Indexer.IndexDocument(ctx, doc, at)
pages, err := h.Store.Meta.GetDocumentPages(ctx, d)
if err != nil {
h.Runtime.Log.Error(method, err)

4
domain/meta/store.go
View file

@ -54,6 +54,10 @@ func (s Store) GetDocumentPages(ctx domain.RequestContext, documentID string) (p
WHERE c_docid=? AND (c_status=0 OR ((c_status=4 OR c_status=2) AND c_relativeid=''))`),
documentID)
if err == sql.ErrNoRows {
err = nil
p = []page.Page{}
}
if err != nil {
err = errors.Wrap(err, "failed to get instance document pages")
}

5
domain/permission/store.go
View file

@ -33,7 +33,8 @@ type Store struct {
func (s Store) AddPermission(ctx domain.RequestContext, r permission.Permission) (err error) {
r.Created = time.Now().UTC()
_, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_permission (c_orgid, c_who, c_whoid, c_action, c_scope, c_location, c_refid, c_created) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"),
_, err = ctx.Transaction.Exec(s.Bind(`INSERT INTO dmz_permission
(c_orgid, c_who, c_whoid, c_action, c_scope, c_location, c_refid, c_created) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`),
r.OrgID, string(r.Who), r.WhoID, string(r.Action), string(r.Scope), string(r.Location), r.RefID, r.Created)
if err != nil {
@ -278,7 +279,7 @@ func (s Store) DeleteSpacePermissions(ctx domain.RequestContext, spaceID string)
// DeleteUserSpacePermissions removes all roles for the specified user, for the specified space.
func (s Store) DeleteUserSpacePermissions(ctx domain.RequestContext, spaceID, userID string) (rows int64, err error) {
sql := fmt.Sprintf("DELETE FROM dmz_permission WHERE c_orgid='%s' AND c_location='space' AND c_refid='%s' c_who='user' AND c_whoid='%s'",
sql := fmt.Sprintf("DELETE FROM dmz_permission WHERE c_orgid='%s' AND c_location='space' AND c_refid='%s' AND c_who='user' AND c_whoid='%s'",
ctx.OrgID, spaceID, userID)
return s.DeleteWhere(ctx.Transaction, sql)

4
domain/pin/store.go
View file

@ -120,10 +120,10 @@ func (s Store) DeletePin(ctx domain.RequestContext, id string) (rows int64, err
// DeletePinnedSpace removes any pins for specified space.
func (s Store) DeletePinnedSpace(ctx domain.RequestContext, spaceID string) (rows int64, err error) {
return s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_pin WHERE c_orgid=\"%s\" AND c_spaceid=\"%s\"", ctx.OrgID, spaceID))
return s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_pin WHERE c_orgid='%s' AND c_spaceid='%s'", ctx.OrgID, spaceID))
}
// DeletePinnedDocument removes any pins for specified document.
func (s Store) DeletePinnedDocument(ctx domain.RequestContext, documentID string) (rows int64, err error) {
return s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_pin WHERE c_orgid=\"%s\" AND c_docid=\"%s\"", ctx.OrgID, documentID))
return s.DeleteWhere(ctx.Transaction, fmt.Sprintf("DELETE FROM dmz_pin WHERE c_orgid='%s' AND c_docid='%s'", ctx.OrgID, documentID))
}

7
domain/search/search.go
View file

@ -82,7 +82,12 @@ func (m *Indexer) IndexContent(ctx domain.RequestContext, p page.Page) {
return
}
ctx.Transaction.Commit()
err = ctx.Transaction.Commit()
if err != nil {
ctx.Transaction.Rollback()
m.runtime.Log.Error(method, err)
return
}
}
// DeleteContent removes all search entries for specific document content.

120
domain/search/store.go
View file

@ -39,43 +39,68 @@ type Store struct {
// IndexDocument adds search index entries for document inserting title, tags and attachments as
// searchable items. Any existing document entries are removed.
func (s Store) IndexDocument(ctx domain.RequestContext, doc doc.Document, a []attachment.Attachment) (err error) {
method := "search.IndexDocument"
// remove previous search entries
_, err = ctx.Transaction.Exec(s.Bind("DELETE FROM dmz_search WHERE c_orgid=? AND c_docid=? AND (c_itemtype='doc' OR c_itemtype='file' OR c_itemtype='tag')"),
ctx.OrgID, doc.RefID)
if err != nil {
if err != nil && err != sql.ErrNoRows {
err = errors.Wrap(err, "execute delete document index entries")
s.Runtime.Log.Error(method, err)
return
}
// insert doc title
_, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content) VALUES (?, ?, ?, ?, ?)"),
ctx.OrgID, doc.RefID, "", "doc", doc.Name)
if err != nil {
if s.Runtime.StoreProvider.Type() == env.StoreTypePostgreSQL {
_, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content, c_token) VALUES (?, ?, ?, ?, ?, to_tsvector(?))"),
ctx.OrgID, doc.RefID, "", "doc", doc.Name, doc.Name)
} else {
_, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content) VALUES (?, ?, ?, ?, ?)"),
ctx.OrgID, doc.RefID, "", "doc", doc.Name)
}
if err != nil && err != sql.ErrNoRows {
err = errors.Wrap(err, "execute insert document title entry")
s.Runtime.Log.Error(method, err)
return
}
// insert doc tags
tags := strings.Split(doc.Tags, "#")
for _, t := range tags {
t = strings.TrimSpace(t)
if len(t) == 0 {
continue
}
_, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content) VALUES (?, ?, ?, ?, ?)"),
ctx.OrgID, doc.RefID, "", "tag", t)
if s.Runtime.StoreProvider.Type() == env.StoreTypePostgreSQL {
_, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content, c_token) VALUES (?, ?, ?, ?, ?, to_tsvector(?))"),
ctx.OrgID, doc.RefID, "", "tag", t, t)
if err != nil {
} else {
_, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content) VALUES (?, ?, ?, ?, ?)"),
ctx.OrgID, doc.RefID, "", "tag", t)
}
if err != nil && err != sql.ErrNoRows {
err = errors.Wrap(err, "execute insert document tag entry")
s.Runtime.Log.Error(method, err)
return
}
}
for _, file := range a {
_, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content) VALUES (?, ?, ?, ?, ?)"),
ctx.OrgID, doc.RefID, file.RefID, "file", file.Filename)
if s.Runtime.StoreProvider.Type() == env.StoreTypePostgreSQL {
_, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content, c_token) VALUES (?, ?, ?, ?, ?, to_tsvector(?))"),
ctx.OrgID, doc.RefID, file.RefID, "file", file.Filename, file.Filename)
if err != nil {
} else {
_, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content) VALUES (?, ?, ?, ?, ?)"),
ctx.OrgID, doc.RefID, file.RefID, "file", file.Filename)
}
if err != nil && err != sql.ErrNoRows {
err = errors.Wrap(err, "execute insert document file entry")
s.Runtime.Log.Error(method, err)
return
}
}
@ -84,11 +109,14 @@ func (s Store) IndexDocument(ctx domain.RequestContext, doc doc.Document, a []at
// DeleteDocument removes all search entries for document.
func (s Store) DeleteDocument(ctx domain.RequestContext, ID string) (err error) {
method := "search.DeleteDocument"
_, err = ctx.Transaction.Exec(s.Bind("DELETE FROM dmz_search WHERE c_orgid=? AND c_docid=?"),
ctx.OrgID, ID)
if err != nil {
if err != nil && err != sql.ErrNoRows {
err = errors.Wrap(err, "execute delete document entries")
s.Runtime.Log.Error(method, err)
}
return
@ -97,6 +125,8 @@ func (s Store) DeleteDocument(ctx domain.RequestContext, ID string) (err error)
// IndexContent adds search index entry for document context.
// Any existing document entries are removed.
func (s Store) IndexContent(ctx domain.RequestContext, p page.Page) (err error) {
method := "search.IndexContent"
// we do not index pending pages
if p.Status == workflow.ChangePending || p.Status == workflow.ChangePendingNew {
return
@ -106,28 +136,49 @@ func (s Store) IndexContent(ctx domain.RequestContext, p page.Page) (err error)
_, err = ctx.Transaction.Exec(s.Bind("DELETE FROM dmz_search WHERE c_orgid=? AND c_docid=? AND c_itemid=? AND c_itemtype='page'"),
ctx.OrgID, p.DocumentID, p.RefID)
if err != nil {
if err != nil && err != sql.ErrNoRows {
err = errors.Wrap(err, "execute delete document content entry")
s.Runtime.Log.Error(method, err)
return
}
err = nil
// prepare content
content, err := stringutil.HTML(p.Body).Text(false)
if err != nil {
err = errors.Wrap(err, "search strip HTML failed")
s.Runtime.Log.Error(method, err)
return
}
content = strings.TrimSpace(content)
_, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content) VALUES (?, ?, ?, ?, ?)"),
ctx.OrgID, p.DocumentID, p.RefID, "page", content)
if err != nil {
err = errors.Wrap(err, "execute insert document content entry")
}
if s.Runtime.StoreProvider.Type() == env.StoreTypePostgreSQL {
_, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content, c_token) VALUES (?, ?, ?, ?, ?, to_tsvector(?))"),
ctx.OrgID, p.DocumentID, p.RefID, "page", content, content)
_, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content) VALUES (?, ?, ?, ?, ?)"),
ctx.OrgID, p.DocumentID, p.RefID, "page", p.Name)
if err != nil {
err = errors.Wrap(err, "execute insert document page title entry")
} else {
_, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content) VALUES (?, ?, ?, ?, ?)"),
ctx.OrgID, p.DocumentID, p.RefID, "page", content)
}
if err != nil && err != sql.ErrNoRows {
err = errors.Wrap(err, "execute insert section content entry")
s.Runtime.Log.Error(method, err)
return
}
err = nil
if s.Runtime.StoreProvider.Type() == env.StoreTypePostgreSQL {
_, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content, c_token) VALUES (?, ?, ?, ?, ?, to_tsvector(?))"),
ctx.OrgID, p.DocumentID, p.RefID, "page", p.Name, p.Name)
} else {
_, err = ctx.Transaction.Exec(s.Bind("INSERT INTO dmz_search (c_orgid, c_docid, c_itemid, c_itemtype, c_content) VALUES (?, ?, ?, ?, ?)"),
ctx.OrgID, p.DocumentID, p.RefID, "page", p.Name)
}
if err != nil && err != sql.ErrNoRows {
err = errors.Wrap(err, "execute insert section title entry")
s.Runtime.Log.Error(method, err)
return
}
return nil
@ -135,18 +186,23 @@ func (s Store) IndexContent(ctx domain.RequestContext, p page.Page) (err error)
// DeleteContent removes all search entries for specific document content.
func (s Store) DeleteContent(ctx domain.RequestContext, pageID string) (err error) {
method := "search.DeleteContent"
// remove all search entries
var stmt1 *sqlx.Stmt
stmt1, err = ctx.Transaction.Preparex(s.Bind("DELETE FROM dmz_search WHERE c_orgid=? AND c_itemid=? AND c_itemtype=?"))
defer streamutil.Close(stmt1)
if err != nil {
if err != nil && err != sql.ErrNoRows {
err = errors.Wrap(err, "prepare delete document content entry")
s.Runtime.Log.Error(method, err)
return
}
_, err = stmt1.Exec(ctx.OrgID, pageID, "page")
if err != nil {
if err != nil && err != sql.ErrNoRows {
err = errors.Wrap(err, "execute delete document content entry")
s.Runtime.Log.Error(method, err)
return
}
@ -220,9 +276,17 @@ func (s Store) matchFullText(ctx domain.RequestContext, keywords, itemType strin
switch s.Runtime.StoreProvider.Type() {
case env.StoreTypeMySQL:
fts = " AND MATCH(s.c_content) AGAINST(? IN BOOLEAN MODE)"
fts = " AND MATCH(s.c_content) AGAINST(? IN BOOLEAN MODE) "
case env.StoreTypePostgreSQL:
fts = ""
// By default, we expect no Postgres full text search operators.
parser := "plainto_tsquery"
// If we find operators then we have to use correct query processor.
operator := strings.ContainsAny(keywords, "!()&|*'`\":<->")
if operator {
parser = "to_tsquery"
}
fts = fmt.Sprintf(" AND s.c_token @@ %s(?) ", parser)
}
sql1 := s.Bind(`
@ -279,7 +343,7 @@ func (s Store) matchLike(ctx domain.RequestContext, keywords, itemType string) (
keywords = strings.Replace(keywords, "'", "", -1)
keywords = strings.Replace(keywords, "\"", "", -1)
keywords = strings.Replace(keywords, "%", "", -1)
keywords = fmt.Sprintf("%%%s%%", keywords)
keywords = fmt.Sprintf("%%%s%%", strings.ToLower(keywords))
sql1 := s.Bind(`SELECT
s.id, s.c_orgid AS orgid, s.c_docid AS documentid, s.c_itemid AS itemid, s.c_itemtype AS itemtype,
@ -304,7 +368,7 @@ func (s Store) matchLike(ctx domain.RequestContext, keywords, itemType string) (
AND p.c_location='space' AND (r.c_userid=? OR r.c_userid='0')
)
)
AND s.c_content LIKE ?`)
AND LOWER(s.c_content) LIKE ?`)
err = s.Runtime.Db.Select(&r,
sql1,

8
domain/space/endpoint.go
View file

@ -470,7 +470,13 @@ func (h *Handler) Update(w http.ResponseWriter, r *http.Request) {
// If newly marked Everyone space, ensure everyone has permission
if prev.Type != space.ScopePublic && sp.Type == space.ScopePublic {
h.Store.Permission.DeleteUserSpacePermissions(ctx, sp.RefID, user.EveryoneUserID)
_, err = h.Store.Permission.DeleteUserSpacePermissions(ctx, sp.RefID, user.EveryoneUserID)
if err != nil {
ctx.Transaction.Rollback()
response.WriteServerError(w, method, err)
h.Runtime.Log.Error(method, err)
return
}
perm := permission.Permission{}
perm.OrgID = sp.OrgID

13
domain/store/context.go
View file

@ -12,6 +12,7 @@
package store
import (
"database/sql"
"fmt"
"github.com/documize/community/core/env"
"github.com/jmoiron/sqlx"
@ -44,12 +45,13 @@ func (c *Context) Bind(sql string) string {
func (c *Context) Delete(tx *sqlx.Tx, table string, id string) (rows int64, err error) {
result, err := tx.Exec(c.Bind("DELETE FROM "+table+" WHERE c_refid=?"), id)
if err != nil {
if err != nil && err != sql.ErrNoRows {
err = errors.Wrap(err, fmt.Sprintf("unable to delete row in table %s", table))
return
}
rows, err = result.RowsAffected()
err = nil
return
}
@ -58,12 +60,13 @@ func (c *Context) Delete(tx *sqlx.Tx, table string, id string) (rows int64, err
func (c *Context) DeleteConstrained(tx *sqlx.Tx, table string, orgID, id string) (rows int64, err error) {
result, err := tx.Exec(c.Bind("DELETE FROM "+table+" WHERE c_orgid=? AND c_refid=?"), orgID, id)
if err != nil {
if err != nil && err != sql.ErrNoRows {
err = errors.Wrap(err, fmt.Sprintf("unable to delete row in table %s", table))
return
}
rows, err = result.RowsAffected()
err = nil
return
}
@ -72,12 +75,13 @@ func (c *Context) DeleteConstrained(tx *sqlx.Tx, table string, orgID, id string)
func (c *Context) DeleteConstrainedWithID(tx *sqlx.Tx, table string, orgID, id string) (rows int64, err error) {
result, err := tx.Exec(c.Bind("DELETE FROM "+table+" WHERE c_orgid=? AND id=?"), orgID, id)
if err != nil {
if err != nil && err != sql.ErrNoRows {
err = errors.Wrap(err, fmt.Sprintf("unable to delete row in table %s", table))
return
}
rows, err = result.RowsAffected()
err = nil
return
}
@ -86,12 +90,13 @@ func (c *Context) DeleteConstrainedWithID(tx *sqlx.Tx, table string, orgID, id s
func (c *Context) DeleteWhere(tx *sqlx.Tx, statement string) (rows int64, err error) {
result, err := tx.Exec(statement)
if err != nil {
if err != nil && err != sql.ErrNoRows {
err = errors.Wrap(err, fmt.Sprintf("unable to delete rows: %s", statement))
return
}
rows, err = result.RowsAffected()
err = nil
return
}

23
edition/storage/mysql.go
View file

@ -43,13 +43,24 @@ import (
// SetMySQLProvider creates MySQL provider
func SetMySQLProvider(r *env.Runtime, s *store.Store) {
// Set up provider specific details and wire up data prividers.
r.StoreProvider = MySQLProvider{
// Set up provider specific details.
p := MySQLProvider{
ConnectionString: r.Flags.DBConn,
Variant: r.Flags.DBType,
}
switch r.Flags.DBType {
case "mysql":
p.Variant = env.StoreTypeMySQL
case "mariadb":
p.Variant = env.StoreTypeMariaDB
case "percona":
p.Variant = env.StoreTypePercona
}
// Wire up data providers!
r.StoreProvider = p
// Wire up data providers.
// Account
accountStore := account.Store{}
accountStore.Runtime = r
s.Account = accountStore
@ -146,7 +157,7 @@ type MySQLProvider struct {
ConnectionString string
// User specified db type (mysql, percona or mariadb).
Variant string
Variant env.StoreType
}
// Type returns name of provider
@ -155,7 +166,7 @@ func (p MySQLProvider) Type() env.StoreType {
}
// TypeVariant returns databse flavor
func (p MySQLProvider) TypeVariant() string {
func (p MySQLProvider) TypeVariant() env.StoreType {
return p.Variant
}

12
edition/storage/postgresql.go
View file

@ -45,18 +45,20 @@ type PostgreSQLProvider struct {
ConnectionString string
// Unused for this provider.
Variant string
Variant env.StoreType
}
// SetPostgreSQLProvider creates PostgreSQL provider
func SetPostgreSQLProvider(r *env.Runtime, s *store.Store) {
// Set up provider specific details and wire up data prividers.
// Set up provider specific details.
r.StoreProvider = PostgreSQLProvider{
ConnectionString: r.Flags.DBConn,
Variant: "",
Variant: env.StoreTypePostgreSQL,
}
// Wire up data providers!
// Wire up data providers.
// Account
accountStore := account.Store{}
accountStore.Runtime = r
s.Account = accountStore
@ -153,7 +155,7 @@ func (p PostgreSQLProvider) Type() env.StoreType {
}
// TypeVariant returns databse flavor
func (p PostgreSQLProvider) TypeVariant() string {
func (p PostgreSQLProvider) TypeVariant() env.StoreType {
return p.Variant
}

6
gui/app/constants/constants.js
View file

@ -135,6 +135,12 @@ let constants = EmberObject.extend({
Rejected: 6,
Publish: 7,
},
// Meta
StoreProvider: { // eslint-disable-line ember/avoid-leaking-state-in-ember-objects
MySQL: 'MySQL',
PostgreSQL: 'PostgreSQL',
},
});
export default { constants }

2
gui/app/pods/customize/search/route.js
View file

@ -29,6 +29,6 @@ export default Route.extend(AuthenticatedRouteMixin, {
},
activate() {
this.get('browser').setTitle('Search');
this.get('browser').setTitle('Search Engine');
}
});

3
gui/app/pods/search/controller.js
View file

@ -9,9 +9,12 @@
//
// https://documize.com
import { inject as service } from '@ember/service';
import Controller from '@ember/controller';
export default Controller.extend({
appMeta: service(),
queryParams: ['filter', 'matchDoc', 'matchContent', 'matchTag', 'matchFile', 'slog'],
filter: '',
matchDoc: true,

2
gui/app/pods/search/route.js
View file

@ -9,8 +9,8 @@
//
// https://documize.com
import Route from '@ember/routing/route';
import AuthenticatedRouteMixin from 'ember-simple-auth/mixins/authenticated-route-mixin';
import Route from '@ember/routing/route';
export default Route.extend(AuthenticatedRouteMixin, {
activate() {

46
gui/app/pods/search/template.hbs
View file

@ -10,22 +10,36 @@
<div id="sidebar" class="sidebar">
<h1>Search</h1>
<div class="view-search">
<div class="syntax">
<div class="example">apple banana</div>
<div class="explain">Find rows that contain at least one of the two words</div>
<div class="example">+apple +banana</div>
<div class="explain">Find rows that contain both words</div>
<div class="example">+apple macintosh</div>
<div class="explain">Find rows that contain the word "apple", but rank rows higher if they also contain "macintosh"</div>
<div class="example">+apple -macintosh</div>
<div class="explain">Find rows that contain the word "apple" but not "macintosh"</div>
<div class="example">+apple +(&gt;turnover &lt;strudel)</div>
<div class="explain">Find rows that contain the words "apple" and "turnover", or "apple" and "strudel" (in any order), but rank "apple turnover" higher than "apple strudel"</div>
<div class="example">apple*</div>
<div class="explain">Find rows that contain words such as "apple", "apples", "applesauce", or "applet"</div>
<div class="example">"some words"</div>
<div class="explain">Find rows that contain the exact phrase "some words" (for example, rows that contain "some words of wisdom" but not "some noise words")</div>
</div>
{{#if (eq appMeta.storageProvider constants.StoreProvider.MySQL)}}
<div class="syntax">
<div class="example">apple banana</div>
<div class="explain">Show results that contain at least one of the two words</div>
<div class="example">+apple +banana</div>
<div class="explain">Show results that contain both words</div>
<div class="example">+apple macintosh</div>
<div class="explain">Show results that contain the word "apple", but rank rows higher if they also contain "macintosh"</div>
<div class="example">+apple -macintosh</div>
<div class="explain">Show results that contain the word "apple" but not "macintosh"</div>
<div class="example">+apple +(&gt;turnover &lt;strudel)</div>
<div class="explain">Show results that contain the words "apple" and "turnover", or "apple" and "strudel" (in any order), but rank "apple turnover" higher than "apple strudel"</div>
<div class="example">apple*</div>
<div class="explain">Show results that contain words such as "apple", "apples", "applesauce", or "applet"</div>
<div class="example">"some words"</div>
<div class="explain">Show results that contain the exact phrase "some words" (for example, rows that contain "some words of wisdom" but not "some noise words")</div>
</div>
{{/if}}
{{#if (eq appMeta.storageProvider constants.StoreProvider.PostgreSQL)}}
<div class="syntax">
<div class="example">apple | banana</div>
<div class="explain">Show results that contain at either word</div>
<div class="example">apple & banana</div>
<div class="explain">Show results that contain both words</div>
<div class="example">apple !macintosh</div>
<div class="explain">Show results that contain the word "apple" but not "macintosh"</div>
<div class="example">google & (apple | microsoft) & !ibm</div>
<div class="explain">Show results that have "google", either "apple" or "microsoft" but not "ibm"</div>
</div>
{{/if}}
</div>
</div>
{{/layout/middle-zone-sidebar}}

1
gui/app/services/app-meta.js
View file

@ -36,6 +36,7 @@ export default Service.extend({
setupMode: false,
secureMode: false,
maxTags: 3,
storageProvider: '',
// for major.minor semver release detection
// for bugfix releases, only admin is made aware of new release and end users see no What's New messaging

30
model/org/meta.go
View file

@ -12,8 +12,9 @@
package org
import (
"github.com/documize/community/core/env"
"time"
"github.com/documize/community/core/env"
)
// SitemapDocument details a document that can be exposed via Sitemap.
@ -27,17 +28,18 @@ type SitemapDocument struct {
// SiteMeta holds information associated with an Organization.
type SiteMeta struct {
OrgID string `json:"orgId"`
Title string `json:"title"`
Message string `json:"message"`
URL string `json:"url"`
AllowAnonymousAccess bool `json:"allowAnonymousAccess"`
AuthProvider string `json:"authProvider"`
AuthConfig string `json:"authConfig"`
Version string `json:"version"`
MaxTags int `json:"maxTags"`
Edition string `json:"edition"`
Valid bool `json:"valid"`
ConversionEndpoint string `json:"conversionEndpoint"`
License env.License `json:"license"`
OrgID string `json:"orgId"`
Title string `json:"title"`
Message string `json:"message"`
URL string `json:"url"`
AllowAnonymousAccess bool `json:"allowAnonymousAccess"`
AuthProvider string `json:"authProvider"`
AuthConfig string `json:"authConfig"`
Version string `json:"version"`
MaxTags int `json:"maxTags"`
Edition string `json:"edition"`
Valid bool `json:"valid"`
ConversionEndpoint string `json:"conversionEndpoint"`
License env.License `json:"license"`
Storage env.StoreType `json:"storageProvider"`
}