mirror of
https://github.com/documize/community.git
synced 2025-07-23 15:19:42 +02:00
restructure directories
This commit is contained in:
parent
7e4ed6545b
commit
a2ce777762
159 changed files with 320 additions and 323 deletions
121
core/api/store/local.go
Normal file
121
core/api/store/local.go
Normal file
|
@ -0,0 +1,121 @@
|
|||
// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
|
||||
//
|
||||
// This software (Documize Community Edition) is licensed under
|
||||
// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
|
||||
//
|
||||
// You can operate outside the AGPL restrictions by purchasing
|
||||
// Documize Enterprise Edition and obtaining a commercial license
|
||||
// by contacting <sales@documize.com>.
|
||||
//
|
||||
// https://documize.com
|
||||
|
||||
// Package store provides the implementation for a file system based storage provider.
|
||||
// This enables all document upload previews to be processed AND stored locally.
|
||||
package store
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/documize/community/core/api/convert"
|
||||
api "github.com/documize/community/core/convapi"
|
||||
"github.com/documize/community/core/log"
|
||||
)
|
||||
|
||||
var folderPath string
|
||||
|
||||
func init() {
|
||||
tempDir := os.TempDir()
|
||||
if !strings.HasSuffix(tempDir, string(os.PathSeparator)) {
|
||||
tempDir += string(os.PathSeparator)
|
||||
}
|
||||
folderPath = tempDir + "documize" + string(os.PathSeparator) + "_uploads" + string(os.PathSeparator)
|
||||
log.Info("Temporary upload directory: " + folderPath)
|
||||
log.IfErr(os.MkdirAll(folderPath, os.ModePerm))
|
||||
}
|
||||
|
||||
// LocalStorageProvider provides an implementation of StorageProvider.
|
||||
type LocalStorageProvider struct {
|
||||
}
|
||||
|
||||
// Upload a flie and store it locally.
|
||||
func (store *LocalStorageProvider) Upload(job string, filename string, file []byte) (err error) {
|
||||
destination := folderPath + job + string(os.PathSeparator)
|
||||
|
||||
err = os.MkdirAll(destination, os.ModePerm)
|
||||
|
||||
if err != nil {
|
||||
log.Error(fmt.Sprintf("Cannot create local folder %s", destination), err)
|
||||
return err
|
||||
}
|
||||
|
||||
err = ioutil.WriteFile(destination+filename, file, 0666)
|
||||
|
||||
if err != nil {
|
||||
log.Error(fmt.Sprintf("Cannot write to local file %s", destination+filename), err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Convert a file from its native format into Documize internal format.
|
||||
func (store *LocalStorageProvider) Convert(params api.ConversionJobRequest) (filename string, fileResult *api.DocumentConversionResponse, err error) {
|
||||
fileResult = &api.DocumentConversionResponse{}
|
||||
err = nil
|
||||
path := folderPath
|
||||
|
||||
if params.Job == "" {
|
||||
return filename, fileResult, errors.New("no job to convert")
|
||||
}
|
||||
|
||||
inputFolder := path + params.Job + string(os.PathSeparator)
|
||||
|
||||
list, err := ioutil.ReadDir(inputFolder)
|
||||
|
||||
if err != nil {
|
||||
return filename, fileResult, err
|
||||
}
|
||||
|
||||
if len(list) == 0 {
|
||||
return filename, fileResult, errors.New("no file to convert")
|
||||
}
|
||||
|
||||
// remove temporary directory on exit
|
||||
defer func() { log.IfErr(os.RemoveAll(inputFolder)) }()
|
||||
|
||||
for _, v := range list {
|
||||
|
||||
if v.Size() > 0 && !strings.HasPrefix(v.Name(), ".") && v.Mode().IsRegular() {
|
||||
filename = inputFolder + v.Name()
|
||||
log.Info(fmt.Sprintf("Fetching document %s", filename))
|
||||
|
||||
fileData, err := ioutil.ReadFile(filename)
|
||||
|
||||
if err != nil {
|
||||
log.Error(fmt.Sprintf("Unable to fetch document %s", filename), err)
|
||||
return filename, fileResult, err
|
||||
}
|
||||
|
||||
if len(fileData) > 0 {
|
||||
fileRequest := api.DocumentConversionRequest{}
|
||||
fileRequest.Filename = filename
|
||||
fileRequest.Filedata = fileData
|
||||
fileRequest.PageBreakLevel = params.IndexDepth
|
||||
//fileRequest.Job = params.OrgID + string(os.PathSeparator) + params.Job
|
||||
//fileRequest.OrgID = params.OrgID
|
||||
|
||||
bits := strings.Split(filename, ".")
|
||||
xtn := strings.ToLower(bits[len(bits)-1])
|
||||
|
||||
fileResult, err = convert.Convert(nil, xtn, &fileRequest)
|
||||
return filename, fileResult, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return filename, fileResult, nil
|
||||
}
|
90
core/api/store/local_test.go
Normal file
90
core/api/store/local_test.go
Normal file
|
@ -0,0 +1,90 @@
|
|||
// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
|
||||
//
|
||||
// This software (Documize Community Edition) is licensed under
|
||||
// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
|
||||
//
|
||||
// You can operate outside the AGPL restrictions by purchasing
|
||||
// Documize Enterprise Edition and obtaining a commercial license
|
||||
// by contacting <sales@documize.com>.
|
||||
//
|
||||
// https://documize.com
|
||||
|
||||
package store
|
||||
|
||||
import (
|
||||
"github.com/documize/community/core/api/plugins"
|
||||
"github.com/documize/community/core/api/util"
|
||||
api "github.com/documize/community/core/convapi"
|
||||
"github.com/documize/community/core/log"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var lsp LocalStorageProvider
|
||||
|
||||
func TestUpload(t *testing.T) {
|
||||
jb := "job" + util.UniqueID()
|
||||
fn := "file.txt"
|
||||
cont := "content\n"
|
||||
err := lsp.Upload(jb, fn, []byte(cont))
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
b, e := ioutil.ReadFile(folderPath + jb + string(os.PathSeparator) + fn)
|
||||
if e != nil {
|
||||
t.Error(e)
|
||||
}
|
||||
if string(b) != cont {
|
||||
t.Error("wrong content:" + string(b))
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvert(t *testing.T) {
|
||||
_, _, err :=
|
||||
lsp.Convert(api.ConversionJobRequest{})
|
||||
if err == nil {
|
||||
t.Error("there should have been a convert error")
|
||||
}
|
||||
|
||||
err = plugins.LibSetup()
|
||||
if err == nil {
|
||||
// t.Error("did not error with missing config.json")
|
||||
}
|
||||
defer log.IfErr(plugins.Lib.KillSubProcs())
|
||||
|
||||
jb := "job" + util.UniqueID()
|
||||
|
||||
_, _, err =
|
||||
lsp.Convert(api.ConversionJobRequest{
|
||||
Job: jb,
|
||||
IndexDepth: 9,
|
||||
OrgID: "Documize",
|
||||
})
|
||||
if err == nil {
|
||||
t.Error("there should have been an error - directory not found")
|
||||
}
|
||||
|
||||
fn := "content.html"
|
||||
cont := "content\n"
|
||||
err = lsp.Upload(jb, fn, []byte(cont))
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
filename, fileResult, err :=
|
||||
lsp.Convert(api.ConversionJobRequest{
|
||||
Job: jb,
|
||||
IndexDepth: 9,
|
||||
OrgID: "Documize",
|
||||
})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if !strings.HasSuffix(filename, fn) {
|
||||
t.Error("wrong filename:" + filename)
|
||||
}
|
||||
if fileResult.Excerpt != "content." {
|
||||
t.Error("wrong excerpt:" + fileResult.Excerpt)
|
||||
}
|
||||
}
|
72
core/api/store/store.go
Normal file
72
core/api/store/store.go
Normal file
|
@ -0,0 +1,72 @@
|
|||
// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
|
||||
//
|
||||
// This software (Documize Community Edition) is licensed under
|
||||
// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
|
||||
//
|
||||
// You can operate outside the AGPL restrictions by purchasing
|
||||
// Documize Enterprise Edition and obtaining a commercial license
|
||||
// by contacting <sales@documize.com>.
|
||||
//
|
||||
// https://documize.com
|
||||
|
||||
package store
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
"github.com/documize/community/core/api/entity"
|
||||
api "github.com/documize/community/core/convapi"
|
||||
"github.com/documize/community/core/utility"
|
||||
)
|
||||
|
||||
// StorageProvider describes the interface for document conversion and take-on.
|
||||
type StorageProvider interface {
|
||||
Upload(job string, filename string, file []byte) (err error)
|
||||
Convert(api.ConversionJobRequest) (filename string, fileResult *api.DocumentConversionResponse, err error)
|
||||
}
|
||||
|
||||
// ConvertFileResult takes the results of a document upload and convert,
|
||||
// and creates the outline of a database record suitable for inserting into the document
|
||||
// table.
|
||||
func ConvertFileResult(filename string, fileResult *api.DocumentConversionResponse) (document entity.Document) {
|
||||
|
||||
document = entity.Document{}
|
||||
|
||||
document.RefID = ""
|
||||
document.OrgID = ""
|
||||
document.LabelID = ""
|
||||
document.Job = ""
|
||||
document.Location = filename
|
||||
if fileResult != nil {
|
||||
if len(fileResult.Pages) > 0 {
|
||||
document.Title = fileResult.Pages[0].Title
|
||||
document.Slug = utility.MakeSlug(fileResult.Pages[0].Title)
|
||||
}
|
||||
document.Excerpt = fileResult.Excerpt
|
||||
}
|
||||
|
||||
document.Tags = "" // now a # separated list of tag-words, rather than JSON
|
||||
|
||||
return document
|
||||
}
|
||||
|
||||
// ExportAs takes a target extension name and html to create an exported file.
|
||||
// If the target extension is "html" it simply returns the given html suitably wrapped,
|
||||
// otherwise it runs the "Export" plugin for the given target extension name.
|
||||
func ExportAs(xtn, html string) (*api.DocumentExport, error) {
|
||||
|
||||
if strings.ToLower(xtn) == "html" {
|
||||
return &api.DocumentExport{File: []byte(html), Format: "html"}, nil
|
||||
}
|
||||
return nil, errors.New("Only 'HTML' export is supported")
|
||||
|
||||
/* This functionality removed for now
|
||||
fileI, err := plugins.Lib.Run(nil, "Export", xtn, []byte(html))
|
||||
if err != nil {
|
||||
log.Error("ExportAs failed", err)
|
||||
return nil, err
|
||||
}
|
||||
return fileI.(*api.DocumentExport), nil
|
||||
*/
|
||||
}
|
63
core/api/store/store_test.go
Normal file
63
core/api/store/store_test.go
Normal file
|
@ -0,0 +1,63 @@
|
|||
// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
|
||||
//
|
||||
// This software (Documize Community Edition) is licensed under
|
||||
// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
|
||||
//
|
||||
// You can operate outside the AGPL restrictions by purchasing
|
||||
// Documize Enterprise Edition and obtaining a commercial license
|
||||
// by contacting <sales@documize.com>.
|
||||
//
|
||||
// https://documize.com
|
||||
|
||||
package store
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/documize/community/core/api/plugins"
|
||||
api "github.com/documize/community/core/convapi"
|
||||
"github.com/documize/community/core/log"
|
||||
)
|
||||
|
||||
func TestExportAs(t *testing.T) {
|
||||
err := plugins.LibSetup()
|
||||
if err == nil {
|
||||
// t.Error("did not error with missing config.json")
|
||||
}
|
||||
defer log.IfErr(plugins.Lib.KillSubProcs())
|
||||
|
||||
htm := "<p>some html</p>"
|
||||
|
||||
exp, err := ExportAs("html", htm)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if string(exp.File) != htm {
|
||||
t.Error("html returned not as expected: " + string(exp.File))
|
||||
}
|
||||
|
||||
// log.TestIfErr = true
|
||||
_, err = ExportAs("XXXXX", htm)
|
||||
if err == nil /* || log.TestIfErr */ {
|
||||
t.Error("did not error as expected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvertFileResult(t *testing.T) {
|
||||
fn := "afile"
|
||||
doc := ConvertFileResult(fn, &api.DocumentConversionResponse{}) // should not panic
|
||||
if doc.Location != fn {
|
||||
t.Error("filename not passed through")
|
||||
}
|
||||
doc = ConvertFileResult(fn, nil) // should not panic
|
||||
if doc.Location != fn {
|
||||
t.Error("filename not passed through")
|
||||
}
|
||||
rj := "Romeo & Juliet"
|
||||
doc = ConvertFileResult(fn, &api.DocumentConversionResponse{
|
||||
Pages: []api.Page{api.Page{Title: rj}},
|
||||
})
|
||||
if doc.Title != rj || doc.Slug != "romeo-juliet" {
|
||||
t.Error("title not passed through correctly")
|
||||
}
|
||||
}
|
Loading…
Add table
Add a link
Reference in a new issue