From 66d5e73ed1bd8a8d101f53bd9df347e2ab2914e9 Mon Sep 17 00:00:00 2001 From: sauls8t Date: Thu, 11 Oct 2018 16:19:11 +0100 Subject: [PATCH] Generate global or tenant level backup file --- domain/backup/backup.go | 549 +++++++++++++++++- domain/backup/endpoint.go | 2 + .../components/customize/backup-restore.js | 6 +- .../components/customize/backup-restore.hbs | 2 +- model/attachment/attachment.go | 2 +- 5 files changed, 533 insertions(+), 28 deletions(-) diff --git a/domain/backup/backup.go b/domain/backup/backup.go index 4e62f7df..e936b573 100644 --- a/domain/backup/backup.go +++ b/domain/backup/backup.go @@ -34,12 +34,24 @@ import ( "github.com/documize/community/domain" "github.com/documize/community/domain/store" "github.com/documize/community/model/account" + "github.com/documize/community/model/action" + "github.com/documize/community/model/activity" + "github.com/documize/community/model/attachment" + "github.com/documize/community/model/audit" m "github.com/documize/community/model/backup" + "github.com/documize/community/model/block" + "github.com/documize/community/model/category" + "github.com/documize/community/model/doc" "github.com/documize/community/model/group" + "github.com/documize/community/model/link" "github.com/documize/community/model/org" + "github.com/documize/community/model/page" + "github.com/documize/community/model/permission" + "github.com/documize/community/model/pin" "github.com/documize/community/model/space" "github.com/documize/community/model/user" uuid "github.com/nu7hatch/gouuid" + "github.com/pkg/errors" ) // Handler contains the runtime information such as logging and database. @@ -113,30 +125,10 @@ func (b backerHandler) GenerateBackup() (filename string, err error) { return filename, nil } -// Manifest describes envrionement of backup source. -func (b backerHandler) getManifest(id string) (string, error) { - m := m.Manifest{ - ID: id, - Edition: b.Runtime.Product.Edition, - Version: b.Runtime.Product.Version, - Major: b.Runtime.Product.Major, - Minor: b.Runtime.Product.Minor, - Patch: b.Runtime.Product.Patch, - Revision: b.Runtime.Product.Revision, - StoreType: b.Runtime.StoreProvider.Type(), - Created: time.Now().UTC(), - OrgID: b.Spec.OrgID, - } - - s, err := toJSON(m) - - return s, err -} - // Produce collection of files to be included in backup file. func (b backerHandler) produce(id string) (files []backupItem, err error) { // Backup manifest - c, err := b.getManifest(id) + c, err := b.manifest(id) if err != nil { return } @@ -148,6 +140,12 @@ func (b backerHandler) produce(id string) (files []backupItem, err error) { return } + // Config, User Config + err = b.dmzConfig(&files) + if err != nil { + return + } + // User, Account err = b.dmzUserAccount(&files) if err != nil { @@ -160,15 +158,71 @@ func (b backerHandler) produce(id string) (files []backupItem, err error) { return } - // Space + // Activity, Audit + err = b.dmzActivity(&files) + if err != nil { + return + } + + // Pin + err = b.dmzPin(&files) + if err != nil { + return + } + + // Space, Permission. err = b.dmzSpace(&files) if err != nil { return } + // Category, Category Member. + err = b.dmzCategory(&files) + if err != nil { + return + } + + // Section, Section Meta, Section Revision, Section Template. + err = b.dmzSection(&files) + if err != nil { + return + } + + // Document, Link, Vote, Comment, Share, Attachment. + err = b.dmzDocument(&files) + if err != nil { + return + } + + // Action + err = b.dmzAction(&files) + if err != nil { + return + } + return } +// Manifest describes envrionement of backup source. +func (b backerHandler) manifest(id string) (string, error) { + m := m.Manifest{ + ID: id, + Edition: b.Runtime.Product.Edition, + Version: b.Runtime.Product.Version, + Major: b.Runtime.Product.Major, + Minor: b.Runtime.Product.Minor, + Patch: b.Runtime.Product.Patch, + Revision: b.Runtime.Product.Revision, + StoreType: b.Runtime.StoreProvider.Type(), + Created: time.Now().UTC(), + OrgID: b.Spec.OrgID, + } + + s, err := toJSON(m) + + return s, err +} + // Organization. func (b backerHandler) dmzOrg(files *[]backupItem) (err error) { w := "" @@ -197,6 +251,52 @@ func (b backerHandler) dmzOrg(files *[]backupItem) (err error) { return } +// Config, User Config. +func (b backerHandler) dmzConfig(files *[]backupItem) (err error) { + type config struct { + ConfigKey string `json:"key"` + ConfigValue string `json:"config"` + } + c := []config{} + err = b.Runtime.Db.Select(&c, `SELECT c_key AS configkey, c_config AS configvalue FROM dmz_config`) + if err != nil { + return + } + + content, err := toJSON(c) + if err != nil { + return + } + *files = append(*files, backupItem{Filename: "dmz_config.json", Content: content}) + + w := "" + if !b.Spec.SystemBackup() { + w = fmt.Sprintf(" where c_orgid='%s' ", b.Spec.OrgID) + } + + type userConfig struct { + OrgID string `json:"orgId"` + UserID string `json:"userId"` + ConfigKey string `json:"key"` + ConfigValue string `json:"config"` + } + uc := []userConfig{} + + err = b.Runtime.Db.Select(&uc, `select c_orgid AS orgid, c_userid AS userid, + c_key AS configkey, c_config AS configvalue FROM dmz_user_config`+w) + if err != nil { + return + } + + content, err = toJSON(uc) + if err != nil { + return + } + *files = append(*files, backupItem{Filename: "dmz_user_config.json", Content: content}) + + return +} + // User, Account. func (b backerHandler) dmzUserAccount(files *[]backupItem) (err error) { w := "" @@ -287,7 +387,79 @@ func (b backerHandler) dmzGroup(files *[]backupItem) (err error) { return } -// Space. +// Activity, Audit +func (b backerHandler) dmzActivity(files *[]backupItem) (err error) { + w := "" + if !b.Spec.SystemBackup() { + w = fmt.Sprintf(" WHERE c_orgid='%s' ", b.Spec.OrgID) + } + + ac := []activity.UserActivity{} + err = b.Runtime.Db.Select(&ac, ` + SELECT id, c_orgid AS orgid, c_userid AS userid, c_spaceid AS spaceid, + c_docid AS documentid, c_sectionid AS sectionid, c_sourcetype AS sourcetype, + c_activitytype AS activitytype, c_metadata AS metadata, c_created AS created + FROM dmz_user_activity`+w) + if err != nil { + return errors.Wrap(err, "select.activity") + } + + content, err := toJSON(ac) + if err != nil { + return errors.Wrap(err, "json.activity") + } + *files = append(*files, backupItem{Filename: "dmz_user_activity.json", Content: content}) + + w = "" + if !b.Spec.SystemBackup() { + w = fmt.Sprintf(" WHERE c_orgid='%s' ", b.Spec.OrgID) + } + + al := []audit.AppEvent{} + err = b.Runtime.Db.Select(&al, ` + SELECT c_orgid AS orgid, c_userid AS userid, c_eventtype AS type, + c_ip AS ip, c_created AS created + FROM dmz_audit_log`+w) + if err != nil { + return errors.Wrap(err, "select.audit") + } + + content, err = toJSON(al) + if err != nil { + return errors.Wrap(err, "json.audit") + } + *files = append(*files, backupItem{Filename: "dmz_audit_log.json", Content: content}) + + return +} + +// Pin +func (b backerHandler) dmzPin(files *[]backupItem) (err error) { + w := "" + if !b.Spec.SystemBackup() { + w = fmt.Sprintf(" WHERE c_orgid='%s' ", b.Spec.OrgID) + } + + p := []pin.Pin{} + err = b.Runtime.Db.Select(&p, ` + SELECT id, c_refid AS refid, + c_orgid AS orgid, c_userid AS userid, c_spaceid AS spaceid, c_docid AS documentid, + c_name AS name, c_sequence AS sequence, c_created AS created, c_revised AS revised + FROM dmz_pin`+w) + if err != nil { + return errors.Wrap(err, "select.pin") + } + + content, err := toJSON(p) + if err != nil { + return errors.Wrap(err, "json.pin") + } + *files = append(*files, backupItem{Filename: "dmz_pin.json", Content: content}) + + return +} + +// Space, Permission. func (b backerHandler) dmzSpace(files *[]backupItem) (err error) { w := "" if !b.Spec.SystemBackup() { @@ -310,5 +482,336 @@ func (b backerHandler) dmzSpace(files *[]backupItem) (err error) { } *files = append(*files, backupItem{Filename: "dmz_space.json", Content: content}) + w = "" + if !b.Spec.SystemBackup() { + w = fmt.Sprintf(" WHERE c_orgid='%s' ", b.Spec.OrgID) + } + + p := []permission.Permission{} + err = b.Runtime.Db.Select(&p, ` + SELECT id, c_orgid AS orgid, c_who AS who, c_whoid AS whoid, + c_action AS action, c_scope AS scope, c_location AS location, + c_refid AS refid, c_created AS created + FROM dmz_permission`+w) + if err != nil { + return errors.Wrap(err, "select.permission") + } + + content, err = toJSON(p) + if err != nil { + return errors.Wrap(err, "json.permission") + } + *files = append(*files, backupItem{Filename: "dmz_permission.json", Content: content}) + + return +} + +// Category, Category Member. +func (b backerHandler) dmzCategory(files *[]backupItem) (err error) { + w := "" + if !b.Spec.SystemBackup() { + w = fmt.Sprintf(" WHERE c_orgid='%s' ", b.Spec.OrgID) + } + + cat := []category.Category{} + err = b.Runtime.Db.Select(&cat, ` + SELECT id, c_refid AS refid, + c_orgid AS orgid, c_spaceid AS spaceid, + c_name AS name, c_created AS created, c_revised AS revised + FROM dmz_category`+w) + if err != nil { + return errors.Wrap(err, "select.category") + } + + content, err := toJSON(cat) + if err != nil { + return errors.Wrap(err, "json.category") + } + *files = append(*files, backupItem{Filename: "dmz_category.json", Content: content}) + + w = "" + if !b.Spec.SystemBackup() { + w = fmt.Sprintf(" WHERE c_orgid='%s' ", b.Spec.OrgID) + } + + cm := []category.Member{} + err = b.Runtime.Db.Select(&cm, ` + SELECT id, c_refid AS refid, c_orgid AS orgid, + c_spaceid AS spaceid, c_categoryid AS categoryid, + c_docid AS documentid, c_created AS created, c_revised AS revised + FROM dmz_category_member`+w) + if err != nil { + return errors.Wrap(err, "select.categorymember") + } + + content, err = toJSON(cm) + if err != nil { + return errors.Wrap(err, "json.categorymember") + } + *files = append(*files, backupItem{Filename: "dmz_category_member.json", Content: content}) + + return +} + +// Section, Section Meta, Section Revision, Section Template. +func (b backerHandler) dmzSection(files *[]backupItem) (err error) { + w := "" + if !b.Spec.SystemBackup() { + w = fmt.Sprintf(" WHERE c_orgid='%s' ", b.Spec.OrgID) + } + + // Section + sec := []page.Page{} + err = b.Runtime.Db.Select(&sec, ` + SELECT id, c_refid AS refid, c_orgid AS orgid, c_docid AS documentid, c_userid AS userid, c_contenttype AS contenttype, c_type AS type, + c_level AS level, c_sequence AS sequence, c_name AS name, c_body AS body, c_revisions AS revisions, c_templateid AS templateid, + c_status AS status, c_relativeid AS relativeid, c_created AS created, c_revised AS revised + FROM dmz_section`+w) + if err != nil { + return errors.Wrap(err, "select.section") + } + + content, err := toJSON(sec) + if err != nil { + return errors.Wrap(err, "json.section") + } + *files = append(*files, backupItem{Filename: "dmz_section.json", Content: content}) + + // Section Meta + sm := []page.Meta{} + err = b.Runtime.Db.Select(&sm, ` + SELECT id, c_sectionid AS sectionid, + c_orgid AS orgid, c_userid AS userid, c_docid AS documentid, + c_rawbody AS rawbody, coalesce(c_config,`+b.Runtime.StoreProvider.JSONEmpty()+`) as config, + c_external AS externalsource, c_created AS created, c_revised AS revised + FROM dmz_section_meta`+w) + if err != nil { + return errors.Wrap(err, "select.sectionmeta") + } + + content, err = toJSON(sm) + if err != nil { + return errors.Wrap(err, "json.sectionmeta") + } + *files = append(*files, backupItem{Filename: "dmz_section_meta.json", Content: content}) + + // Section Revision + sr := []page.Revision{} + err = b.Runtime.Db.Select(&sr, ` + SELECT id, c_refid AS refid, + c_orgid AS orgid, c_docid AS documentid, c_ownerid AS ownerid, + c_sectionid AS sectionid, + c_userid AS userid, c_contenttype AS contenttype, c_type AS type, + c_name AS name, c_body AS body, coalesce(c_rawbody, '') as rawbody, + coalesce(c_config,`+b.Runtime.StoreProvider.JSONEmpty()+`) as config, + c_created AS created, c_revised AS revised + FROM dmz_section_revision`+w) + if err != nil { + return errors.Wrap(err, "select.sectionrevision") + } + + content, err = toJSON(sr) + if err != nil { + return errors.Wrap(err, "json.sectionrevision") + } + *files = append(*files, backupItem{Filename: "dmz_section_revision.json", Content: content}) + + // Section Template + st := []block.Block{} + err = b.Runtime.Db.Select(&st, ` + SELECT id, c_refid as refid, + c_orgid as orgid, + c_spaceid AS spaceid, c_userid AS userid, c_contenttype AS contenttype, c_type AS type, + c_name AS name, c_body AS body, c_desc AS excerpt, c_rawbody AS rawbody, + c_config AS config, c_external AS externalsource, c_used AS used, + c_created AS created, c_revised AS revised + FROM dmz_section_template`+w) + if err != nil { + return errors.Wrap(err, "select.sectiontemplate") + } + + content, err = toJSON(st) + if err != nil { + return errors.Wrap(err, "json.sectiontemplate") + } + *files = append(*files, backupItem{Filename: "dmz_section_template.json", Content: content}) + + return +} + +// Document, Link, Vote, Comment, Share, Attachment. +func (b backerHandler) dmzDocument(files *[]backupItem) (err error) { + w := "" + if !b.Spec.SystemBackup() { + w = fmt.Sprintf(" WHERE c_orgid='%s' ", b.Spec.OrgID) + } + + // Document + d := []doc.Document{} + err = b.Runtime.Db.Select(&d, ` + SELECT id, c_refid AS refid, c_orgid AS orgid, c_spaceid AS spaceid, c_userid AS userid, + c_job AS job, c_location AS location, c_name AS name, c_desc AS excerpt, c_slug AS slug, + c_tags AS tags, c_template AS template, c_protection AS protection, c_approval AS approval, + c_lifecycle AS lifecycle, c_versioned AS versioned, c_versionid AS versionid, + c_versionorder AS versionorder, c_groupid AS groupid, c_created AS created, c_revised AS revised + FROM dmz_doc`+w) + if err != nil { + return errors.Wrap(err, "select.document") + } + + content, err := toJSON(d) + if err != nil { + return errors.Wrap(err, "json.document") + } + *files = append(*files, backupItem{Filename: "dmz_doc.json", Content: content}) + + // Vote + type vote struct { + RefID string `json:"refId"` + OrgID string `json:"orgId"` + DocumentID string `json:"documentId"` + VoterID string `json:"voterId"` + Vote int `json:"vote"` + Created time.Time `json:"created"` + Revised time.Time `json:"revised"` + } + + vt := []vote{} + err = b.Runtime.Db.Select(&vt, ` + SELECT c_refid AS refid, c_orgid AS orgid, + c_voter AS voterid, c_vote AS vote, + c_docid AS documentid, c_created AS created, c_revised AS revised + FROM dmz_doc_vote`+w) + if err != nil { + return errors.Wrap(err, "select.docvote") + } + + content, err = toJSON(vt) + if err != nil { + return errors.Wrap(err, "json.docvote") + } + *files = append(*files, backupItem{Filename: "dmz_doc_vote.json", Content: content}) + + // Link + ln := []link.Link{} + err = b.Runtime.Db.Select(&ln, ` + select c_refid AS refid, c_orgid AS orgid, c_spaceid AS spaceid, c_userid AS userid, + c_sourcedocid AS sourcedocumentid, c_sourcesectionid AS sourcesectionid, + c_targetdocid AS targetdocumentid, c_targetid AS targetid, c_externalid AS externalid, + c_type as linktype, c_orphan As orphan, c_created AS created, c_revised AS revised + FROM dmz_doc_link`+w) + if err != nil { + return errors.Wrap(err, "select.doclink") + } + + content, err = toJSON(ln) + if err != nil { + return errors.Wrap(err, "json.doclink") + } + *files = append(*files, backupItem{Filename: "dmz_doc_link.json", Content: content}) + + // Comment + type comment struct { + RefID string `json:"feedbackId"` + OrgID string `json:"orgId"` + DocumentID string `json:"documentId"` + UserID string `json:"userId"` + Email string `json:"email"` + Feedback string `json:"feedback"` + Created string `json:"created"` + } + + cm := []comment{} + err = b.Runtime.Db.Select(&cm, ` + SELECT c_refid AS refid, c_orgid AS orgid, c_docid AS documentid, + c_userid AS userid, c_email AS email, + c_feedback AS feedback, c_created AS created + FROM dmz_doc_comment`+w) + if err != nil { + return errors.Wrap(err, "select.doccomment") + } + + content, err = toJSON(cm) + if err != nil { + return errors.Wrap(err, "json.doccomment") + } + *files = append(*files, backupItem{Filename: "dmz_doc_comment.json", Content: content}) + + // Share + type share struct { + ID uint64 `json:"id"` + OrgID string `json:"-"` + UserID string `json:"userId"` + DocumentID string `json:"documentId"` + Email string `json:"email"` + Message string `json:"message"` + Viewed string `json:"viewed"` // recording each view as |date-viewed|date-viewed| + Secret string `json:"-"` // secure token used to access document + Expires string `json:"expires"` // number of days from creation, value of 0 means never + Active bool `json:"active"` + Created time.Time `json:"created"` + } + + sh := []share{} + err = b.Runtime.Db.Select(&sh, ` + SELECT id AS id, c_orgid AS orgid, c_docid AS documentid, + c_userid AS userid, c_email AS email, c_message AS message, c_viewed AS viewed, + c_expires AS expires, c_active AS active, c_created AS created + FROM dmz_doc_share`+w) + if err != nil { + return errors.Wrap(err, "select.docshare") + } + + content, err = toJSON(sh) + if err != nil { + return errors.Wrap(err, "json.docshare") + } + *files = append(*files, backupItem{Filename: "dmz_doc_share.json", Content: content}) + + // Attachment + at := []attachment.Attachment{} + err = b.Runtime.Db.Select(&at, ` + SELECT id, c_refid AS refid, + c_orgid AS orgid, c_docid AS documentid, c_job AS job, c_fileid AS fileid, + c_filename AS filename, c_data AS data, c_extension AS extension, + c_created AS created, c_revised AS revised + FROM dmz_doc_attachment`+w) + if err != nil { + return errors.Wrap(err, "select.docattachment") + } + + content, err = toJSON(at) + if err != nil { + return errors.Wrap(err, "json.docattachment") + } + *files = append(*files, backupItem{Filename: "dmz_doc_attachment.json", Content: content}) + + return +} + +// Action +func (b backerHandler) dmzAction(files *[]backupItem) (err error) { + w := "" + if !b.Spec.SystemBackup() { + w = fmt.Sprintf(" WHERE c_orgid='%s' ", b.Spec.OrgID) + } + + ac := []action.UserAction{} + err = b.Runtime.Db.Select(&ac, ` + SELECT c_refid AS refid, c_orgid AS orgid, c_docid AS documentid, c_userid AS userid, + c_actiontype AS actiontype, c_note AS note, c_requestorid AS requestorid, c_requested AS requested, c_due AS due, + c_completed AS completed, c_iscomplete AS iscomplete, c_reftype AS reftype, c_reftypeid AS reftypeid, + c_created AS created, c_revised AS revised + FROM dmz_action`+w) + if err != nil { + return errors.Wrap(err, "select.action") + } + + content, err := toJSON(ac) + if err != nil { + return errors.Wrap(err, "json.action") + } + *files = append(*files, backupItem{Filename: "dmz_action.json", Content: content}) + return } diff --git a/domain/backup/endpoint.go b/domain/backup/endpoint.go index 1aee24d3..a95ed94a 100644 --- a/domain/backup/endpoint.go +++ b/domain/backup/endpoint.go @@ -83,6 +83,8 @@ func (h *Handler) Backup(w http.ResponseWriter, r *http.Request) { return } + h.Runtime.Log.Info("Backup started") + bh := backerHandler{Runtime: h.Runtime, Store: h.Store, Context: ctx, Spec: spec} // Produce zip file on disk. diff --git a/gui/app/components/customize/backup-restore.js b/gui/app/components/customize/backup-restore.js index ff27c1c0..ad011a97 100644 --- a/gui/app/components/customize/backup-restore.js +++ b/gui/app/components/customize/backup-restore.js @@ -25,9 +25,9 @@ export default Component.extend(Notifier, { didReceiveAttrs() { this._super(...arguments); this.set('backupSpec', { - retain: true, - org: '*' - // org: this.get('appMeta.orgId') + retain: false, + // org: '*' + org: this.get('appMeta.orgId') }); }, diff --git a/gui/app/templates/components/customize/backup-restore.hbs b/gui/app/templates/components/customize/backup-restore.hbs index 288c0594..8eac3434 100644 --- a/gui/app/templates/components/customize/backup-restore.hbs +++ b/gui/app/templates/components/customize/backup-restore.hbs @@ -16,7 +16,7 @@

Backup failed — please check server logs

{{/if}} {{#if backupSuccess}} -

Backup successful ({{backupFilename}})

+

Backup successful ({{backupFilename}})

{{/if}} diff --git a/model/attachment/attachment.go b/model/attachment/attachment.go index a5f7b967..0896b24a 100644 --- a/model/attachment/attachment.go +++ b/model/attachment/attachment.go @@ -21,6 +21,6 @@ type Attachment struct { Job string `json:"job"` FileID string `json:"fileId"` Filename string `json:"filename"` - Data []byte `json:"-"` + Data []byte `json:"data"` Extension string `json:"extension"` }