first pass of code move to new API

domain/conversion/conversion.go

@@ -0,0 +1,263 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>.
+//
+// https://documize.com
+
+package conversion
+
+import (
+	"bytes"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/documize/community/core/api/store"
+	api "github.com/documize/community/core/convapi"
+	"github.com/documize/community/core/request"
+	"github.com/documize/community/core/response"
+	"github.com/documize/community/core/stringutil"
+	"github.com/documize/community/core/uniqueid"
+	"github.com/documize/community/domain"
+	"github.com/documize/community/domain/document"
+	"github.com/documize/community/model/activity"
+	"github.com/documize/community/model/attachment"
+	"github.com/documize/community/model/audit"
+	"github.com/documize/community/model/doc"
+	"github.com/documize/community/model/page"
+	uuid "github.com/nu7hatch/gouuid"
+	"github.com/pkg/errors"
+)
+
+var storageProvider store.StorageProvider
+
+func init() {
+	storageProvider = new(store.LocalStorageProvider)
+}
+
+func (h *Handler) upload(w http.ResponseWriter, r *http.Request) (string, string, string) {
+	method := "conversion.upload"
+	ctx := domain.GetRequestContext(r)
+
+	folderID := request.Param(r, "folderID")
+
+	if !document.CanUploadDocument(ctx, *h.Store, folderID) {
+		response.WriteForbiddenError(w)
+		return "", "", ""
+	}
+
+	// grab file
+	filedata, filename, err := r.FormFile("attachment")
+	if err != nil {
+		response.WriteMissingDataError(w, method, "attachment")
+		return "", "", ""
+	}
+
+	b := new(bytes.Buffer)
+	_, err = io.Copy(b, filedata)
+	if err != nil {
+		response.WriteServerError(w, method, err)
+		return "", "", ""
+	}
+
+	// generate job id
+	newUUID, err := uuid.NewV4()
+	if err != nil {
+		response.WriteServerError(w, method, err)
+		return "", "", ""
+	}
+
+	job := newUUID.String()
+
+	err = storageProvider.Upload(job, filename.Filename, b.Bytes())
+	if err != nil {
+		response.WriteServerError(w, method, err)
+		return "", "", ""
+	}
+
+	h.Runtime.Log.Info(fmt.Sprintf("Org %s (%s) [Uploaded] %s", ctx.OrgName, ctx.OrgID, filename.Filename))
+
+	return job, folderID, ctx.OrgID
+}
+
+func (h *Handler) convert(w http.ResponseWriter, r *http.Request, job, folderID string, conversion api.ConversionJobRequest) {
+	method := "conversion.upload"
+	ctx := domain.GetRequestContext(r)
+
+	licenseKey := h.Store.Setting.Get(ctx, "EDITION-LICENSE", "key")
+	licenseSignature := h.Store.Setting.Get(ctx, "EDITION-LICENSE", "signature")
+	k, _ := hex.DecodeString(licenseKey)
+	s, _ := hex.DecodeString(licenseSignature)
+
+	conversion.LicenseKey = k
+	conversion.LicenseSignature = s
+
+	org, err := h.Store.Organization.GetOrganization(ctx, ctx.OrgID)
+	if err != nil {
+		response.WriteServerError(w, method, err)
+		return
+	}
+
+	conversion.ServiceEndpoint = org.ConversionEndpoint
+
+	var fileResult *api.DocumentConversionResponse
+	var filename string
+	filename, fileResult, err = storageProvider.Convert(conversion)
+	if err != nil {
+		response.WriteServerError(w, method, err)
+		return
+	}
+
+	if fileResult.Err != "" {
+		response.WriteServerError(w, method, errors.New(fileResult.Err))
+		return
+	}
+
+	// NOTE: empty .docx documents trigger this error
+	if len(fileResult.Pages) == 0 {
+		response.WriteMissingDataError(w, method, "no pages in document")
+		return
+	}
+
+	ctx.Transaction, err = h.Runtime.Db.Beginx()
+	if err != nil {
+		response.WriteServerError(w, method, err)
+		return
+	}
+
+	nd, err := processDocument(ctx, h.Store, filename, job, folderID, fileResult)
+	if err != nil {
+		response.WriteServerError(w, method, err)
+		return
+	}
+
+	response.WriteJSON(w, nd)
+}
+
+func processDocument(ctx domain.RequestContext, store *domain.Store, filename, job, folderID string, fileResult *api.DocumentConversionResponse) (newDocument doc.Document, err error) {
+	// Convert into database objects
+	document := convertFileResult(filename, fileResult)
+	document.Job = job
+	document.OrgID = ctx.OrgID
+	document.LabelID = folderID
+	document.UserID = ctx.UserID
+	documentID := uniqueid.Generate()
+	document.RefID = documentID
+
+	err = store.Document.Add(ctx, document)
+	if err != nil {
+		ctx.Transaction.Rollback()
+		err = errors.Wrap(err, "cannot insert new document")
+		return
+	}
+
+	//err = processPage(documentID, fileResult.PageFiles, fileResult.Pages.Children[0], 1, p)
+
+	for k, v := range fileResult.Pages {
+		var p page.Page
+		p.OrgID = ctx.OrgID
+		p.DocumentID = documentID
+		p.Level = v.Level
+		p.Title = v.Title
+		p.Body = string(v.Body)
+		p.Sequence = float64(k+1) * 1024.0 // need to start above 0 to allow insertion before the first item
+		pageID := uniqueid.Generate()
+		p.RefID = pageID
+		p.ContentType = "wysiwyg"
+		p.PageType = "section"
+
+		meta := page.Meta{}
+		meta.PageID = pageID
+		meta.RawBody = p.Body
+		meta.Config = "{}"
+
+		model := page.NewPage{}
+		model.Page = p
+		model.Meta = meta
+
+		err = store.Page.Add(ctx, model)
+
+		if err != nil {
+			ctx.Transaction.Rollback()
+			err = errors.Wrap(err, "cannot insert new page for new document")
+			return
+		}
+	}
+
+	for _, e := range fileResult.EmbeddedFiles {
+		//fmt.Println("DEBUG embedded file info", document.OrgId, document.Job, e.Name, len(e.Data), e.ID)
+		var a attachment.Attachment
+		a.DocumentID = documentID
+		a.Job = document.Job
+		a.FileID = e.ID
+		a.Filename = strings.Replace(e.Name, "embeddings/", "", 1)
+		a.Data = e.Data
+		refID := uniqueid.Generate()
+		a.RefID = refID
+
+		err = store.Attachment.Add(ctx, a)
+
+		if err != nil {
+			ctx.Transaction.Rollback()
+			err = errors.Wrap(err, "cannot insert attachment for new document")
+			return
+		}
+	}
+
+	newDocument, err = store.Document.Get(ctx, documentID)
+	if err != nil {
+		ctx.Transaction.Rollback()
+		err = errors.Wrap(err, "cannot fetch new document")
+		return
+	}
+
+	err = store.Document.Update(ctx, newDocument) // TODO review - this seems to write-back an unaltered record from that read above, but within that it calls searches.UpdateDocument() to reindex the doc.
+	if err != nil {
+		ctx.Transaction.Rollback()
+		err = errors.Wrap(err, "cannot updater new document")
+		return
+	}
+
+	store.Activity.RecordUserActivity(ctx, activity.UserActivity{
+		LabelID:      newDocument.LabelID,
+		SourceID:     newDocument.RefID,
+		SourceType:   activity.SourceTypeDocument,
+		ActivityType: activity.TypeCreated})
+
+	store.Audit.Record(ctx, audit.EventTypeDocumentUpload)
+
+	ctx.Transaction.Commit()
+
+	return
+}
+
+// convertFileResult takes the results of a document upload and convert,
+// and creates the outline of a database record suitable for inserting into the document
+// table.
+func convertFileResult(filename string, fileResult *api.DocumentConversionResponse) (document doc.Document) {
+	document = doc.Document{}
+	document.RefID = ""
+	document.OrgID = ""
+	document.LabelID = ""
+	document.Job = ""
+	document.Location = filename
+
+	if fileResult != nil {
+		if len(fileResult.Pages) > 0 {
+			document.Title = fileResult.Pages[0].Title
+			document.Slug = stringutil.MakeSlug(fileResult.Pages[0].Title)
+		}
+		document.Excerpt = fileResult.Excerpt
+	}
+
+	document.Tags = "" // now a # separated list of tag-words, rather than JSON
+
+	return document
+}

domain/conversion/endpoint.go

@@ -14,10 +14,9 @@ package conversion
 import (
 	"net/http"
 
+	api "github.com/documize/community/core/convapi"
 	"github.com/documize/community/core/env"
-	"github.com/documize/community/core/response"
 	"github.com/documize/community/domain"
-	"github.com/documize/community/model/template"
 )
 
 // Handler contains the runtime information such as logging and database.
@@ -26,30 +25,16 @@ type Handler struct {
 	Store   *domain.Store
 }
 
-// SavedList returns all templates saved by the user
-func (h *Handler) SavedList(w http.ResponseWriter, r *http.Request) {
-	method := "template.saved"
-	ctx := domain.GetRequestContext(r)
-
-	documents, err := h.Store.Document.Templates(ctx)
-	if err != nil {
-		response.WriteServerError(w, method, err)
-		return
+// UploadConvert is an endpoint to both upload and convert a document
+func (h *Handler) UploadConvert(w http.ResponseWriter, r *http.Request) {
+	job, folderID, orgID := h.upload(w, r)
+	if job == "" {
+		return // error already handled
 	}
 
-	templates := []template.Template{}
-
-	for _, d := range documents {
-		var t = template.Template{}
-		t.ID = d.RefID
-		t.Title = d.Title
-		t.Description = d.Excerpt
-		t.Author = ""
-		t.Dated = d.Created
-		t.Type = template.TypePrivate
-
-		templates = append(templates, t)
-	}
-
-	response.WriteJSON(w, templates)
+	h.convert(w, r, job, folderID, api.ConversionJobRequest{
+		Job:        job,
+		IndexDepth: 4,
+		OrgID:      orgID,
+	})
 }

domain/conversion/model.go

@@ -0,0 +1,22 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>.
+//
+// https://documize.com
+
+package conversion
+
+import (
+	api "github.com/documize/community/core/convapi"
+)
+
+// StorageProvider imports and stores documents
+type StorageProvider interface {
+	Upload(job string, filename string, file []byte) (err error)
+	Convert(api.ConversionJobRequest) (filename string, fileResult *api.DocumentConversionResponse, err error)
+}

domain/conversion/store/local.go

@@ -0,0 +1,124 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>.
+//
+// https://documize.com
+
+// Package store provides the implementation for a file system based storage provider.
+// This enables all document upload previews to be processed AND stored locally.
+package store
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strings"
+
+	"github.com/documize/community/core/api/convert"
+	api "github.com/documize/community/core/convapi"
+	"github.com/documize/community/core/log"
+)
+
+var folderPath string
+
+func init() {
+	tempDir := os.TempDir()
+	if !strings.HasSuffix(tempDir, string(os.PathSeparator)) {
+		tempDir += string(os.PathSeparator)
+	}
+	folderPath = tempDir + "documize" + string(os.PathSeparator) + "_uploads" + string(os.PathSeparator)
+	log.Info("Temporary upload directory: " + folderPath)
+	log.IfErr(os.MkdirAll(folderPath, os.ModePerm))
+}
+
+// LocalStorageProvider provides an implementation of StorageProvider.
+type LocalStorageProvider struct {
+}
+
+// Upload a flie and store it locally.
+func (store *LocalStorageProvider) Upload(job string, filename string, file []byte) (err error) {
+	destination := folderPath + job + string(os.PathSeparator)
+
+	err = os.MkdirAll(destination, os.ModePerm)
+
+	if err != nil {
+		log.Error(fmt.Sprintf("Cannot create local folder %s", destination), err)
+		return err
+	}
+
+	err = ioutil.WriteFile(destination+filename, file, 0666)
+
+	if err != nil {
+		log.Error(fmt.Sprintf("Cannot write to local file %s", destination+filename), err)
+		return err
+	}
+
+	return nil
+}
+
+// Convert a file from its native format into Documize internal format.
+func (store *LocalStorageProvider) Convert(params api.ConversionJobRequest) (filename string, fileResult *api.DocumentConversionResponse, err error) {
+	fileResult = &api.DocumentConversionResponse{}
+	err = nil
+	path := folderPath
+
+	if params.Job == "" {
+		return filename, fileResult, errors.New("no job to convert")
+	}
+
+	inputFolder := path + params.Job + string(os.PathSeparator)
+
+	list, err := ioutil.ReadDir(inputFolder)
+
+	if err != nil {
+		return filename, fileResult, err
+	}
+
+	if len(list) == 0 {
+		return filename, fileResult, errors.New("no file to convert")
+	}
+
+	// remove temporary directory on exit
+	defer func() { log.IfErr(os.RemoveAll(inputFolder)) }()
+
+	for _, v := range list {
+
+		if v.Size() > 0 && !strings.HasPrefix(v.Name(), ".") && v.Mode().IsRegular() {
+			filename = inputFolder + v.Name()
+			log.Info(fmt.Sprintf("Fetching document %s", filename))
+
+			fileData, err := ioutil.ReadFile(filename)
+
+			if err != nil {
+				log.Error(fmt.Sprintf("Unable to fetch document %s", filename), err)
+				return filename, fileResult, err
+			}
+
+			if len(fileData) > 0 {
+				fileRequest := api.DocumentConversionRequest{}
+				fileRequest.Filename = filename
+				fileRequest.Filedata = fileData
+				fileRequest.PageBreakLevel = params.IndexDepth
+				fileRequest.LicenseKey = params.LicenseKey
+				fileRequest.LicenseSignature = params.LicenseSignature
+				fileRequest.ServiceEndpoint = params.ServiceEndpoint
+				//fileRequest.Job = params.OrgID + string(os.PathSeparator) + params.Job
+				//fileRequest.OrgID = params.OrgID
+
+				bits := strings.Split(filename, ".")
+				xtn := strings.ToLower(bits[len(bits)-1])
+
+				fileResult, err = convert.Convert(nil, xtn, &fileRequest)
+				return filename, fileResult, err
+			}
+		}
+	}
+
+	return filename, fileResult, nil
+}

domain/conversion/store/local_test.go

@@ -0,0 +1,90 @@
+// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
+//
+// This software (Documize Community Edition) is licensed under
+// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
+//
+// You can operate outside the AGPL restrictions by purchasing
+// Documize Enterprise Edition and obtaining a commercial license
+// by contacting <sales@documize.com>.
+//
+// https://documize.com
+
+package store
+
+import (
+	"github.com/documize/community/core/api/plugins"
+	"github.com/documize/community/core/api/util"
+	api "github.com/documize/community/core/convapi"
+	"github.com/documize/community/core/log"
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+)
+
+var lsp LocalStorageProvider
+
+func TestUpload(t *testing.T) {
+	jb := "job" + uniqueid.Generate()
+	fn := "file.txt"
+	cont := "content\n"
+	err := lsp.Upload(jb, fn, []byte(cont))
+	if err != nil {
+		t.Error(err)
+	}
+	b, e := ioutil.ReadFile(folderPath + jb + string(os.PathSeparator) + fn)
+	if e != nil {
+		t.Error(e)
+	}
+	if string(b) != cont {
+		t.Error("wrong content:" + string(b))
+	}
+}
+
+func TestConvert(t *testing.T) {
+	_, _, err :=
+		lsp.Convert(api.ConversionJobRequest{})
+	if err == nil {
+		t.Error("there should have been a convert error")
+	}
+
+	err = plugins.LibSetup()
+	if err == nil {
+		// t.Error("did not error with missing config.json")
+	}
+	defer log.IfErr(plugins.Lib.KillSubProcs())
+
+	jb := "job" + uniqueid.Generate()
+
+	_, _, err =
+		lsp.Convert(api.ConversionJobRequest{
+			Job:        jb,
+			IndexDepth: 9,
+			OrgID:      "Documize",
+		})
+	if err == nil {
+		t.Error("there should have been an error - directory not found")
+	}
+
+	fn := "content.html"
+	cont := "content\n"
+	err = lsp.Upload(jb, fn, []byte(cont))
+	if err != nil {
+		t.Error(err)
+	}
+	filename, fileResult, err :=
+		lsp.Convert(api.ConversionJobRequest{
+			Job:        jb,
+			IndexDepth: 9,
+			OrgID:      "Documize",
+		})
+	if err != nil {
+		t.Error(err)
+	}
+	if !strings.HasSuffix(filename, fn) {
+		t.Error("wrong filename:" + filename)
+	}
+	if fileResult.Excerpt != "content." {
+		t.Error("wrong excerpt:" + fileResult.Excerpt)
+	}
+}

server/routing/routes.go

@@ -14,13 +14,13 @@ package routing
 import (
 	"net/http"
 
-	"github.com/documize/community/core/api/endpoint"
 	"github.com/documize/community/core/env"
 	"github.com/documize/community/domain"
 	"github.com/documize/community/domain/attachment"
 	"github.com/documize/community/domain/auth"
 	"github.com/documize/community/domain/auth/keycloak"
 	"github.com/documize/community/domain/block"
+	"github.com/documize/community/domain/conversion"
 	"github.com/documize/community/domain/document"
 	"github.com/documize/community/domain/link"
 	"github.com/documize/community/domain/meta"
@@ -57,6 +57,7 @@ func RegisterEndpoints(rt *env.Runtime, s *domain.Store) {
 	template := template.Handler{Runtime: rt, Store: s}
 	document := document.Handler{Runtime: rt, Store: s, Indexer: indexer}
 	attachment := attachment.Handler{Runtime: rt, Store: s}
+	conversion := conversion.Handler{Runtime: rt, Store: s}
 	organization := organization.Handler{Runtime: rt, Store: s}
 
 	//**************************************************
@@ -78,10 +79,8 @@ func RegisterEndpoints(rt *env.Runtime, s *domain.Store) {
 	// Secure routes
 	//**************************************************
 
-	// Import & Convert Document
-	Add(rt, RoutePrefixPrivate, "import/folder/{folderID}", []string{"POST", "OPTIONS"}, nil, endpoint.UploadConvertDocument)
+	Add(rt, RoutePrefixPrivate, "import/folder/{folderID}", []string{"POST", "OPTIONS"}, nil, conversion.UploadConvert)
 
-	// Add(rt, RoutePrefixPrivate, "documents/{documentID}/export", []string{"GET", "OPTIONS"}, nil, endpoint.GetDocumentAsDocx)
 	Add(rt, RoutePrefixPrivate, "documents", []string{"GET", "OPTIONS"}, []string{"filter", "tag"}, document.ByTag)
 	Add(rt, RoutePrefixPrivate, "documents", []string{"GET", "OPTIONS"}, nil, document.BySpace)
 	Add(rt, RoutePrefixPrivate, "documents/{documentID}", []string{"GET", "OPTIONS"}, nil, document.Get)