From e284a46f86ddaf569cc1b790a0afd5d1f28538c2 Mon Sep 17 00:00:00 2001 From: Harvey Kandola Date: Wed, 2 Aug 2017 12:39:12 +0100 Subject: [PATCH] first pass of code move to new API --- domain/conversion/conversion.go | 263 ++++++++++++++++++++++++++ domain/conversion/endpoint.go | 37 ++-- domain/conversion/model.go | 22 +++ domain/conversion/store/local.go | 124 ++++++++++++ domain/conversion/store/local_test.go | 90 +++++++++ server/routing/routes.go | 7 +- 6 files changed, 513 insertions(+), 30 deletions(-) create mode 100644 domain/conversion/conversion.go create mode 100644 domain/conversion/model.go create mode 100644 domain/conversion/store/local.go create mode 100644 domain/conversion/store/local_test.go diff --git a/domain/conversion/conversion.go b/domain/conversion/conversion.go new file mode 100644 index 00000000..9850f91d --- /dev/null +++ b/domain/conversion/conversion.go @@ -0,0 +1,263 @@ +// Copyright 2016 Documize Inc. . All rights reserved. +// +// This software (Documize Community Edition) is licensed under +// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html +// +// You can operate outside the AGPL restrictions by purchasing +// Documize Enterprise Edition and obtaining a commercial license +// by contacting . +// +// https://documize.com + +package conversion + +import ( + "bytes" + "encoding/hex" + "fmt" + "io" + "net/http" + "strings" + + "github.com/documize/community/core/api/store" + api "github.com/documize/community/core/convapi" + "github.com/documize/community/core/request" + "github.com/documize/community/core/response" + "github.com/documize/community/core/stringutil" + "github.com/documize/community/core/uniqueid" + "github.com/documize/community/domain" + "github.com/documize/community/domain/document" + "github.com/documize/community/model/activity" + "github.com/documize/community/model/attachment" + "github.com/documize/community/model/audit" + "github.com/documize/community/model/doc" + "github.com/documize/community/model/page" + uuid "github.com/nu7hatch/gouuid" + "github.com/pkg/errors" +) + +var storageProvider store.StorageProvider + +func init() { + storageProvider = new(store.LocalStorageProvider) +} + +func (h *Handler) upload(w http.ResponseWriter, r *http.Request) (string, string, string) { + method := "conversion.upload" + ctx := domain.GetRequestContext(r) + + folderID := request.Param(r, "folderID") + + if !document.CanUploadDocument(ctx, *h.Store, folderID) { + response.WriteForbiddenError(w) + return "", "", "" + } + + // grab file + filedata, filename, err := r.FormFile("attachment") + if err != nil { + response.WriteMissingDataError(w, method, "attachment") + return "", "", "" + } + + b := new(bytes.Buffer) + _, err = io.Copy(b, filedata) + if err != nil { + response.WriteServerError(w, method, err) + return "", "", "" + } + + // generate job id + newUUID, err := uuid.NewV4() + if err != nil { + response.WriteServerError(w, method, err) + return "", "", "" + } + + job := newUUID.String() + + err = storageProvider.Upload(job, filename.Filename, b.Bytes()) + if err != nil { + response.WriteServerError(w, method, err) + return "", "", "" + } + + h.Runtime.Log.Info(fmt.Sprintf("Org %s (%s) [Uploaded] %s", ctx.OrgName, ctx.OrgID, filename.Filename)) + + return job, folderID, ctx.OrgID +} + +func (h *Handler) convert(w http.ResponseWriter, r *http.Request, job, folderID string, conversion api.ConversionJobRequest) { + method := "conversion.upload" + ctx := domain.GetRequestContext(r) + + licenseKey := h.Store.Setting.Get(ctx, "EDITION-LICENSE", "key") + licenseSignature := h.Store.Setting.Get(ctx, "EDITION-LICENSE", "signature") + k, _ := hex.DecodeString(licenseKey) + s, _ := hex.DecodeString(licenseSignature) + + conversion.LicenseKey = k + conversion.LicenseSignature = s + + org, err := h.Store.Organization.GetOrganization(ctx, ctx.OrgID) + if err != nil { + response.WriteServerError(w, method, err) + return + } + + conversion.ServiceEndpoint = org.ConversionEndpoint + + var fileResult *api.DocumentConversionResponse + var filename string + filename, fileResult, err = storageProvider.Convert(conversion) + if err != nil { + response.WriteServerError(w, method, err) + return + } + + if fileResult.Err != "" { + response.WriteServerError(w, method, errors.New(fileResult.Err)) + return + } + + // NOTE: empty .docx documents trigger this error + if len(fileResult.Pages) == 0 { + response.WriteMissingDataError(w, method, "no pages in document") + return + } + + ctx.Transaction, err = h.Runtime.Db.Beginx() + if err != nil { + response.WriteServerError(w, method, err) + return + } + + nd, err := processDocument(ctx, h.Store, filename, job, folderID, fileResult) + if err != nil { + response.WriteServerError(w, method, err) + return + } + + response.WriteJSON(w, nd) +} + +func processDocument(ctx domain.RequestContext, store *domain.Store, filename, job, folderID string, fileResult *api.DocumentConversionResponse) (newDocument doc.Document, err error) { + // Convert into database objects + document := convertFileResult(filename, fileResult) + document.Job = job + document.OrgID = ctx.OrgID + document.LabelID = folderID + document.UserID = ctx.UserID + documentID := uniqueid.Generate() + document.RefID = documentID + + err = store.Document.Add(ctx, document) + if err != nil { + ctx.Transaction.Rollback() + err = errors.Wrap(err, "cannot insert new document") + return + } + + //err = processPage(documentID, fileResult.PageFiles, fileResult.Pages.Children[0], 1, p) + + for k, v := range fileResult.Pages { + var p page.Page + p.OrgID = ctx.OrgID + p.DocumentID = documentID + p.Level = v.Level + p.Title = v.Title + p.Body = string(v.Body) + p.Sequence = float64(k+1) * 1024.0 // need to start above 0 to allow insertion before the first item + pageID := uniqueid.Generate() + p.RefID = pageID + p.ContentType = "wysiwyg" + p.PageType = "section" + + meta := page.Meta{} + meta.PageID = pageID + meta.RawBody = p.Body + meta.Config = "{}" + + model := page.NewPage{} + model.Page = p + model.Meta = meta + + err = store.Page.Add(ctx, model) + + if err != nil { + ctx.Transaction.Rollback() + err = errors.Wrap(err, "cannot insert new page for new document") + return + } + } + + for _, e := range fileResult.EmbeddedFiles { + //fmt.Println("DEBUG embedded file info", document.OrgId, document.Job, e.Name, len(e.Data), e.ID) + var a attachment.Attachment + a.DocumentID = documentID + a.Job = document.Job + a.FileID = e.ID + a.Filename = strings.Replace(e.Name, "embeddings/", "", 1) + a.Data = e.Data + refID := uniqueid.Generate() + a.RefID = refID + + err = store.Attachment.Add(ctx, a) + + if err != nil { + ctx.Transaction.Rollback() + err = errors.Wrap(err, "cannot insert attachment for new document") + return + } + } + + newDocument, err = store.Document.Get(ctx, documentID) + if err != nil { + ctx.Transaction.Rollback() + err = errors.Wrap(err, "cannot fetch new document") + return + } + + err = store.Document.Update(ctx, newDocument) // TODO review - this seems to write-back an unaltered record from that read above, but within that it calls searches.UpdateDocument() to reindex the doc. + if err != nil { + ctx.Transaction.Rollback() + err = errors.Wrap(err, "cannot updater new document") + return + } + + store.Activity.RecordUserActivity(ctx, activity.UserActivity{ + LabelID: newDocument.LabelID, + SourceID: newDocument.RefID, + SourceType: activity.SourceTypeDocument, + ActivityType: activity.TypeCreated}) + + store.Audit.Record(ctx, audit.EventTypeDocumentUpload) + + ctx.Transaction.Commit() + + return +} + +// convertFileResult takes the results of a document upload and convert, +// and creates the outline of a database record suitable for inserting into the document +// table. +func convertFileResult(filename string, fileResult *api.DocumentConversionResponse) (document doc.Document) { + document = doc.Document{} + document.RefID = "" + document.OrgID = "" + document.LabelID = "" + document.Job = "" + document.Location = filename + + if fileResult != nil { + if len(fileResult.Pages) > 0 { + document.Title = fileResult.Pages[0].Title + document.Slug = stringutil.MakeSlug(fileResult.Pages[0].Title) + } + document.Excerpt = fileResult.Excerpt + } + + document.Tags = "" // now a # separated list of tag-words, rather than JSON + + return document +} diff --git a/domain/conversion/endpoint.go b/domain/conversion/endpoint.go index 49b83ccc..4ee679ae 100644 --- a/domain/conversion/endpoint.go +++ b/domain/conversion/endpoint.go @@ -14,10 +14,9 @@ package conversion import ( "net/http" + api "github.com/documize/community/core/convapi" "github.com/documize/community/core/env" - "github.com/documize/community/core/response" "github.com/documize/community/domain" - "github.com/documize/community/model/template" ) // Handler contains the runtime information such as logging and database. @@ -26,30 +25,16 @@ type Handler struct { Store *domain.Store } -// SavedList returns all templates saved by the user -func (h *Handler) SavedList(w http.ResponseWriter, r *http.Request) { - method := "template.saved" - ctx := domain.GetRequestContext(r) - - documents, err := h.Store.Document.Templates(ctx) - if err != nil { - response.WriteServerError(w, method, err) - return +// UploadConvert is an endpoint to both upload and convert a document +func (h *Handler) UploadConvert(w http.ResponseWriter, r *http.Request) { + job, folderID, orgID := h.upload(w, r) + if job == "" { + return // error already handled } - templates := []template.Template{} - - for _, d := range documents { - var t = template.Template{} - t.ID = d.RefID - t.Title = d.Title - t.Description = d.Excerpt - t.Author = "" - t.Dated = d.Created - t.Type = template.TypePrivate - - templates = append(templates, t) - } - - response.WriteJSON(w, templates) + h.convert(w, r, job, folderID, api.ConversionJobRequest{ + Job: job, + IndexDepth: 4, + OrgID: orgID, + }) } diff --git a/domain/conversion/model.go b/domain/conversion/model.go new file mode 100644 index 00000000..377dbe64 --- /dev/null +++ b/domain/conversion/model.go @@ -0,0 +1,22 @@ +// Copyright 2016 Documize Inc. . All rights reserved. +// +// This software (Documize Community Edition) is licensed under +// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html +// +// You can operate outside the AGPL restrictions by purchasing +// Documize Enterprise Edition and obtaining a commercial license +// by contacting . +// +// https://documize.com + +package conversion + +import ( + api "github.com/documize/community/core/convapi" +) + +// StorageProvider imports and stores documents +type StorageProvider interface { + Upload(job string, filename string, file []byte) (err error) + Convert(api.ConversionJobRequest) (filename string, fileResult *api.DocumentConversionResponse, err error) +} diff --git a/domain/conversion/store/local.go b/domain/conversion/store/local.go new file mode 100644 index 00000000..df66d04a --- /dev/null +++ b/domain/conversion/store/local.go @@ -0,0 +1,124 @@ +// Copyright 2016 Documize Inc. . All rights reserved. +// +// This software (Documize Community Edition) is licensed under +// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html +// +// You can operate outside the AGPL restrictions by purchasing +// Documize Enterprise Edition and obtaining a commercial license +// by contacting . +// +// https://documize.com + +// Package store provides the implementation for a file system based storage provider. +// This enables all document upload previews to be processed AND stored locally. +package store + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "strings" + + "github.com/documize/community/core/api/convert" + api "github.com/documize/community/core/convapi" + "github.com/documize/community/core/log" +) + +var folderPath string + +func init() { + tempDir := os.TempDir() + if !strings.HasSuffix(tempDir, string(os.PathSeparator)) { + tempDir += string(os.PathSeparator) + } + folderPath = tempDir + "documize" + string(os.PathSeparator) + "_uploads" + string(os.PathSeparator) + log.Info("Temporary upload directory: " + folderPath) + log.IfErr(os.MkdirAll(folderPath, os.ModePerm)) +} + +// LocalStorageProvider provides an implementation of StorageProvider. +type LocalStorageProvider struct { +} + +// Upload a flie and store it locally. +func (store *LocalStorageProvider) Upload(job string, filename string, file []byte) (err error) { + destination := folderPath + job + string(os.PathSeparator) + + err = os.MkdirAll(destination, os.ModePerm) + + if err != nil { + log.Error(fmt.Sprintf("Cannot create local folder %s", destination), err) + return err + } + + err = ioutil.WriteFile(destination+filename, file, 0666) + + if err != nil { + log.Error(fmt.Sprintf("Cannot write to local file %s", destination+filename), err) + return err + } + + return nil +} + +// Convert a file from its native format into Documize internal format. +func (store *LocalStorageProvider) Convert(params api.ConversionJobRequest) (filename string, fileResult *api.DocumentConversionResponse, err error) { + fileResult = &api.DocumentConversionResponse{} + err = nil + path := folderPath + + if params.Job == "" { + return filename, fileResult, errors.New("no job to convert") + } + + inputFolder := path + params.Job + string(os.PathSeparator) + + list, err := ioutil.ReadDir(inputFolder) + + if err != nil { + return filename, fileResult, err + } + + if len(list) == 0 { + return filename, fileResult, errors.New("no file to convert") + } + + // remove temporary directory on exit + defer func() { log.IfErr(os.RemoveAll(inputFolder)) }() + + for _, v := range list { + + if v.Size() > 0 && !strings.HasPrefix(v.Name(), ".") && v.Mode().IsRegular() { + filename = inputFolder + v.Name() + log.Info(fmt.Sprintf("Fetching document %s", filename)) + + fileData, err := ioutil.ReadFile(filename) + + if err != nil { + log.Error(fmt.Sprintf("Unable to fetch document %s", filename), err) + return filename, fileResult, err + } + + if len(fileData) > 0 { + fileRequest := api.DocumentConversionRequest{} + fileRequest.Filename = filename + fileRequest.Filedata = fileData + fileRequest.PageBreakLevel = params.IndexDepth + fileRequest.LicenseKey = params.LicenseKey + fileRequest.LicenseSignature = params.LicenseSignature + fileRequest.ServiceEndpoint = params.ServiceEndpoint + //fileRequest.Job = params.OrgID + string(os.PathSeparator) + params.Job + //fileRequest.OrgID = params.OrgID + + bits := strings.Split(filename, ".") + xtn := strings.ToLower(bits[len(bits)-1]) + + fileResult, err = convert.Convert(nil, xtn, &fileRequest) + return filename, fileResult, err + } + } + } + + return filename, fileResult, nil +} diff --git a/domain/conversion/store/local_test.go b/domain/conversion/store/local_test.go new file mode 100644 index 00000000..49494d66 --- /dev/null +++ b/domain/conversion/store/local_test.go @@ -0,0 +1,90 @@ +// Copyright 2016 Documize Inc. . All rights reserved. +// +// This software (Documize Community Edition) is licensed under +// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html +// +// You can operate outside the AGPL restrictions by purchasing +// Documize Enterprise Edition and obtaining a commercial license +// by contacting . +// +// https://documize.com + +package store + +import ( + "github.com/documize/community/core/api/plugins" + "github.com/documize/community/core/api/util" + api "github.com/documize/community/core/convapi" + "github.com/documize/community/core/log" + "io/ioutil" + "os" + "strings" + "testing" +) + +var lsp LocalStorageProvider + +func TestUpload(t *testing.T) { + jb := "job" + uniqueid.Generate() + fn := "file.txt" + cont := "content\n" + err := lsp.Upload(jb, fn, []byte(cont)) + if err != nil { + t.Error(err) + } + b, e := ioutil.ReadFile(folderPath + jb + string(os.PathSeparator) + fn) + if e != nil { + t.Error(e) + } + if string(b) != cont { + t.Error("wrong content:" + string(b)) + } +} + +func TestConvert(t *testing.T) { + _, _, err := + lsp.Convert(api.ConversionJobRequest{}) + if err == nil { + t.Error("there should have been a convert error") + } + + err = plugins.LibSetup() + if err == nil { + // t.Error("did not error with missing config.json") + } + defer log.IfErr(plugins.Lib.KillSubProcs()) + + jb := "job" + uniqueid.Generate() + + _, _, err = + lsp.Convert(api.ConversionJobRequest{ + Job: jb, + IndexDepth: 9, + OrgID: "Documize", + }) + if err == nil { + t.Error("there should have been an error - directory not found") + } + + fn := "content.html" + cont := "content\n" + err = lsp.Upload(jb, fn, []byte(cont)) + if err != nil { + t.Error(err) + } + filename, fileResult, err := + lsp.Convert(api.ConversionJobRequest{ + Job: jb, + IndexDepth: 9, + OrgID: "Documize", + }) + if err != nil { + t.Error(err) + } + if !strings.HasSuffix(filename, fn) { + t.Error("wrong filename:" + filename) + } + if fileResult.Excerpt != "content." { + t.Error("wrong excerpt:" + fileResult.Excerpt) + } +} diff --git a/server/routing/routes.go b/server/routing/routes.go index 7231e3bc..a8f87df4 100644 --- a/server/routing/routes.go +++ b/server/routing/routes.go @@ -14,13 +14,13 @@ package routing import ( "net/http" - "github.com/documize/community/core/api/endpoint" "github.com/documize/community/core/env" "github.com/documize/community/domain" "github.com/documize/community/domain/attachment" "github.com/documize/community/domain/auth" "github.com/documize/community/domain/auth/keycloak" "github.com/documize/community/domain/block" + "github.com/documize/community/domain/conversion" "github.com/documize/community/domain/document" "github.com/documize/community/domain/link" "github.com/documize/community/domain/meta" @@ -57,6 +57,7 @@ func RegisterEndpoints(rt *env.Runtime, s *domain.Store) { template := template.Handler{Runtime: rt, Store: s} document := document.Handler{Runtime: rt, Store: s, Indexer: indexer} attachment := attachment.Handler{Runtime: rt, Store: s} + conversion := conversion.Handler{Runtime: rt, Store: s} organization := organization.Handler{Runtime: rt, Store: s} //************************************************** @@ -78,10 +79,8 @@ func RegisterEndpoints(rt *env.Runtime, s *domain.Store) { // Secure routes //************************************************** - // Import & Convert Document - Add(rt, RoutePrefixPrivate, "import/folder/{folderID}", []string{"POST", "OPTIONS"}, nil, endpoint.UploadConvertDocument) + Add(rt, RoutePrefixPrivate, "import/folder/{folderID}", []string{"POST", "OPTIONS"}, nil, conversion.UploadConvert) - // Add(rt, RoutePrefixPrivate, "documents/{documentID}/export", []string{"GET", "OPTIONS"}, nil, endpoint.GetDocumentAsDocx) Add(rt, RoutePrefixPrivate, "documents", []string{"GET", "OPTIONS"}, []string{"filter", "tag"}, document.ByTag) Add(rt, RoutePrefixPrivate, "documents", []string{"GET", "OPTIONS"}, nil, document.BySpace) Add(rt, RoutePrefixPrivate, "documents/{documentID}", []string{"GET", "OPTIONS"}, nil, document.Get)