From 26bcd84958a65406f5a61079b88c0ff6600a8d14 Mon Sep 17 00:00:00 2001 From: Elliott Stoneham Date: Mon, 9 May 2016 17:32:11 +0100 Subject: [PATCH] first-cut use of config table --- documize/api/convert/apidocumizecom/init.go | 5 +- documize/api/convert/convert.go | 2 +- documize/api/convert/convert_test.go | 2 +- documize/api/endpoint/router.go | 9 +-- documize/api/mail/mailer.go | 13 ++-- documize/api/mail/smtp.go | 2 +- documize/api/plugins/glick.go | 21 +++--- documize/api/request/config.go | 54 +++++++++++++++ documize/api/request/init.go | 3 +- documize/api/request/setup.go | 8 +-- documize/api/request/user_test.go | 2 +- documize/api/store/local_test.go | 2 +- documize/api/util/password.go | 4 +- .../scripts/migrate/migrate-00002.sql | 7 ++ documize/documize.go | 2 +- documize/web/web.go | 4 +- wordsmith/environment/environment.go | 67 +++++++++++-------- wordsmith/log/logger.go | 3 +- 18 files changed, 145 insertions(+), 65 deletions(-) create mode 100644 documize/api/request/config.go create mode 100644 documize/database/scripts/migrate/migrate-00002.sql diff --git a/documize/api/convert/apidocumizecom/init.go b/documize/api/convert/apidocumizecom/init.go index 372328b7..25984174 100644 --- a/documize/api/convert/apidocumizecom/init.go +++ b/documize/api/convert/apidocumizecom/init.go @@ -5,6 +5,7 @@ import ( "errors" "net/http" + "github.com/documize/community/documize/api/request" "github.com/documize/community/wordsmith/environment" ) @@ -13,8 +14,8 @@ var endPoint = "https://api.documize.com" var token string func init() { - environment.GetString(&endPoint, "endpoint", false, "Documize end-point", nil) - environment.GetString(&token, "token", false, "Documize token", nil) + environment.GetString(&endPoint, "endpoint", false, "Documize end-point", request.FlagFromDB) + environment.GetString(&token, "token", false, "Documize token", request.FlagFromDB) } var transport = &http.Transport{ diff --git a/documize/api/convert/convert.go b/documize/api/convert/convert.go index 61e1b97e..f71d936c 100644 --- a/documize/api/convert/convert.go +++ b/documize/api/convert/convert.go @@ -2,12 +2,12 @@ package convert import ( + "errors" "github.com/documize/community/documize/api/convert/excerpt" "github.com/documize/community/documize/api/convert/html" "github.com/documize/community/documize/api/plugins" "github.com/documize/community/wordsmith/api" "github.com/documize/community/wordsmith/utility" - "errors" "golang.org/x/net/context" ) diff --git a/documize/api/convert/convert_test.go b/documize/api/convert/convert_test.go index 8a8a978d..de3f528b 100644 --- a/documize/api/convert/convert_test.go +++ b/documize/api/convert/convert_test.go @@ -13,7 +13,7 @@ import ( func TestConvert(t *testing.T) { - plugins.PluginFile = "" // no file as html is built-in + plugins.PluginFile = "" // no file as html is built-in if lerr := plugins.LibSetup(); lerr == nil { //t.Error("did not error on plugin.Libsetup() with no plugin.json file") //return diff --git a/documize/api/endpoint/router.go b/documize/api/endpoint/router.go index 7432c15f..58eb7876 100644 --- a/documize/api/endpoint/router.go +++ b/documize/api/endpoint/router.go @@ -8,6 +8,7 @@ import ( "github.com/codegangsta/negroni" "github.com/documize/community/documize/api/plugins" + "github.com/documize/community/documize/api/request" "github.com/documize/community/documize/database" "github.com/documize/community/documize/web" "github.com/documize/community/wordsmith/environment" @@ -25,10 +26,10 @@ const ( var port, certFile, keyFile, forcePort2SSL string func init() { - environment.GetString(&certFile, "cert", false, "the cert.pem file used for https", nil) - environment.GetString(&keyFile, "key", false, "the key.pem file used for https", nil) - environment.GetString(&port, "port", false, "http/https port number", nil) - environment.GetString(&forcePort2SSL, "forcesslport", false, "redirect given http port number to TLS", nil) + environment.GetString(&certFile, "cert", false, "the cert.pem file used for https", request.FlagFromDB) + environment.GetString(&keyFile, "key", false, "the key.pem file used for https", request.FlagFromDB) + environment.GetString(&port, "port", false, "http/https port number", request.FlagFromDB) + environment.GetString(&forcePort2SSL, "forcesslport", false, "redirect given http port number to TLS", request.FlagFromDB) } var testHost string // used during automated testing diff --git a/documize/api/mail/mailer.go b/documize/api/mail/mailer.go index fdf512b5..3933d403 100644 --- a/documize/api/mail/mailer.go +++ b/documize/api/mail/mailer.go @@ -8,6 +8,7 @@ import ( "html/template" "net/smtp" + "github.com/documize/community/documize/api/request" "github.com/documize/community/documize/web" "github.com/documize/community/wordsmith/environment" "github.com/documize/community/wordsmith/log" @@ -215,7 +216,7 @@ func ShareFolderNewUser(recipient, inviter, url, folder, invitationMessage strin emailTemplate := string(file) - // check inviter name + // check inviter name if inviter == "Hello You" || len(inviter) == 0 { inviter = "Your colleague" } @@ -258,11 +259,11 @@ var creds struct{ SMTPuserid, SMTPpassword, SMTPhost, SMTPport, SMTPsender strin func init() { creds.SMTPport = "587" // the default value for outgoing SMTP traffic creds.SMTPsender = "Documize " // TODO review as SAAS specific - environment.GetString(&creds.SMTPuserid, "smtpuserid", false, "SMTP username for outgoing email", nil) - environment.GetString(&creds.SMTPpassword, "smtppassword", false, "SMTP password for outgoing email", nil) - environment.GetString(&creds.SMTPhost, "smtphost", false, "SMTP host for outgoing email", nil) - environment.GetString(&creds.SMTPport, "smtpport", false, "SMTP port for outgoing email", nil) - environment.GetString(&creds.SMTPsender, "smtpsender", false, "SMTP sender's e-mail for outgoing email", nil) + environment.GetString(&creds.SMTPuserid, "smtpuserid", false, "SMTP username for outgoing email", request.FlagFromDB) + environment.GetString(&creds.SMTPpassword, "smtppassword", false, "SMTP password for outgoing email", request.FlagFromDB) + environment.GetString(&creds.SMTPhost, "smtphost", false, "SMTP host for outgoing email", request.FlagFromDB) + environment.GetString(&creds.SMTPport, "smtpport", false, "SMTP port for outgoing email", request.FlagFromDB) + environment.GetString(&creds.SMTPsender, "smtpsender", false, "SMTP sender's e-mail for outgoing email", request.FlagFromDB) } // Helper to return SMTP credentials diff --git a/documize/api/mail/smtp.go b/documize/api/mail/smtp.go index 6274c40b..cc1e2ea9 100644 --- a/documize/api/mail/smtp.go +++ b/documize/api/mail/smtp.go @@ -31,10 +31,10 @@ package mail import ( "bytes" - "github.com/documize/community/wordsmith/log" "encoding/base64" "errors" "fmt" + "github.com/documize/community/wordsmith/log" "io" "mime" "mime/multipart" diff --git a/documize/api/plugins/glick.go b/documize/api/plugins/glick.go index e25dbe5e..ebc274c6 100644 --- a/documize/api/plugins/glick.go +++ b/documize/api/plugins/glick.go @@ -10,6 +10,7 @@ import ( "github.com/documize/community/documize/api/convert/documizeapi" "github.com/documize/community/documize/api/convert/html" "github.com/documize/community/documize/api/convert/md" + "github.com/documize/community/documize/api/request" "github.com/documize/community/wordsmith/api" "github.com/documize/community/wordsmith/environment" "github.com/documize/community/wordsmith/log" @@ -22,9 +23,9 @@ var insecure = "false" func init() { environment.GetString(&PluginFile, "plugin", false, - "the JSON file describing plugins, default 'plugin.json'", nil) + "the JSON file describing plugins, default 'plugin.json' set to 'PLUGIN' to configure from database settings", request.FlagFromDB) environment.GetString(&insecure, "insecure", false, - "if 'true' allow https endpoints with invalid certificates (only for testing)", nil) + "if 'true' allow https endpoints with invalid certificates (only for testing)", request.FlagFromDB) } type infoLog struct{} @@ -98,13 +99,17 @@ func LibSetup() error { return err } - json, err := ioutil.ReadFile(PluginFile) - if err != nil { - log.Info("Plugin file '" + PluginFile + "' not found, using no plugins") - json = []byte(" [ ] \n") - err = nil + var json = make([]byte, 0) + if PluginFile == "PLUGIN" { + json = []byte(request.ConfigString(PluginFile, "")) + } else { + json, err = ioutil.ReadFile(PluginFile) + if err != nil { + log.Info("Plugin file '" + PluginFile + "' not found, using no plugins") + json = []byte(" [ ] \n") + err = nil + } } - err = Lib.Configure(json) if err != nil { return err diff --git a/documize/api/request/config.go b/documize/api/request/config.go new file mode 100644 index 00000000..9cc99aad --- /dev/null +++ b/documize/api/request/config.go @@ -0,0 +1,54 @@ +package request + +import ( + "fmt" + "strings" + + "github.com/documize/community/wordsmith/environment" + "github.com/documize/community/wordsmith/log" + "github.com/documize/community/wordsmith/utility" +) + +// FlagFromDB overrides the value in *target if it is set in the database configuration JSON. +// Function signiture must map that in environment +func FlagFromDB(target *string, name string) bool { + value := ConfigString(environment.Prefix, name) + //fmt.Println("DEBUG FlagFromDB " + value) + if value != `""` && value != "" { + *target = strings.TrimPrefix(strings.TrimSuffix(value, `"`), `"`) + return true + } + return false +} + +// ConfigString fetches a configuration JSON element from the config table. +func ConfigString(area, path string) (ret string) { + if path != "" { + path = "." + path + } + sql := `SELECT JSON_EXTRACT(details,'$` + path + `') AS item FROM config WHERE area = '` + area + `';` + + stmt, err := Db.Preparex(sql) + defer utility.Close(stmt) + + if err != nil { + log.Error(fmt.Sprintf("Unable to prepare select for ConfigString: %s", sql), err) + return "" + } + + var item = make([]uint8, 0) + + err = stmt.Get(&item) + + if err != nil { + //log.Error(fmt.Sprintf("Unable to execute select for ConfigString: %s", sql), err) + return "" + } + + if len(item) > 1 { + ret = string(item) + } + + //fmt.Println("DEBUG ConfigString " + sql + " => " + ret) + return ret +} diff --git a/documize/api/request/init.go b/documize/api/request/init.go index 4a4149c8..d3c61537 100644 --- a/documize/api/request/init.go +++ b/documize/api/request/init.go @@ -38,7 +38,7 @@ func init() { environment.GetString(&connectionString, "db", true, `"username:password@protocol(hostname:port)/databasename" for example "fred:bloggs@tcp(localhost:3306)/documize"`, - func() { + func(*string, string) bool { Db, err = sqlx.Open("mysql", stdConn(connectionString)) if err != nil { @@ -65,6 +65,7 @@ func init() { log.Info("database.Check(Db) !OK, going into setup mode") } + return false // value not changed }) } diff --git a/documize/api/request/setup.go b/documize/api/request/setup.go index 00f9621e..0c250437 100644 --- a/documize/api/request/setup.go +++ b/documize/api/request/setup.go @@ -36,9 +36,9 @@ func (p *Persister) SetupOrganization(company, title, message, domain, email str Title: title, // string `json:"title"` Message: message, // string `json:"message"` //URL: "test.domain", // string `json:"url"` - Domain: domain, // string `json:"domain"` - Email: email, // string `json:"email"` - AllowAnonymousAccess: false, // bool `json:"allowAnonymousAccess"` + Domain: domain, // string `json:"domain"` + Email: email, // string `json:"email"` + AllowAnonymousAccess: false, // bool `json:"allowAnonymousAccess"` //Serial: "123", // string `json:"-"` Active: true, // bool `json:"-"` } @@ -50,6 +50,6 @@ func (p *Persister) SetupOrganization(company, title, message, domain, email str if err != nil { return org, err } - p.Context.Transaction, err = Db.Beginx() + p.Context.Transaction, err = Db.Beginx() return org, err } diff --git a/documize/api/request/user_test.go b/documize/api/request/user_test.go index 8ed28718..8ddd7685 100644 --- a/documize/api/request/user_test.go +++ b/documize/api/request/user_test.go @@ -18,7 +18,7 @@ func testAddUser(t *testing.T, p *Persister) entity.User { //Password: "testpassword", // string `json:"-"` //Salt: "testsalt", // string `json:"-"` //Reset: "testreset", // string `json:"-"` - Accounts: nil, // []Account `json:"accounts"` + Accounts: nil, // []Account `json:"accounts"` } user.Salt = generateSalt() requestedPassword := generateRandomPassword() diff --git a/documize/api/store/local_test.go b/documize/api/store/local_test.go index 9a9a3121..04c845de 100644 --- a/documize/api/store/local_test.go +++ b/documize/api/store/local_test.go @@ -31,7 +31,7 @@ func TestUpload(t *testing.T) { } func TestConvert(t *testing.T) { - _, _, err := + _, _, err := lsp.Convert(api.ConversionJobRequest{}) if err == nil { t.Error("there should have been a convert error") diff --git a/documize/api/util/password.go b/documize/api/util/password.go index 7fd14d8e..f4e910ae 100644 --- a/documize/api/util/password.go +++ b/documize/api/util/password.go @@ -41,11 +41,11 @@ func GeneratePassword(password string, salt string) string { return string(hashedPassword) } -// MatchPassword copares a hashed password with a clear one. +// MatchPassword copares a hashed password with a clear one. func MatchPassword(hashedPassword string, password string, salt string) bool { pwd := []byte(salt + password) err := bcrypt.CompareHashAndPassword([]byte(hashedPassword), pwd) - return err == nil + return err == nil } diff --git a/documize/database/scripts/migrate/migrate-00002.sql b/documize/database/scripts/migrate/migrate-00002.sql new file mode 100644 index 00000000..ce30a8a9 --- /dev/null +++ b/documize/database/scripts/migrate/migrate-00002.sql @@ -0,0 +1,7 @@ +DROP TABLE IF EXISTS `config`; + +CREATE TABLE IF NOT EXISTS `config` ( + `area` CHAR(16) NOT NULL, + `details` JSON, + UNIQUE INDEX `idx_config_area` (`area` ASC) ) ; + \ No newline at end of file diff --git a/documize/documize.go b/documize/documize.go index 249ae6f6..7037734b 100644 --- a/documize/documize.go +++ b/documize/documize.go @@ -10,7 +10,7 @@ import ( ) func main() { - environment.Parse() + environment.Parse("db") // process the db value first ready := make(chan struct{}, 1) // channel is used for testing endpoint.Serve(ready) diff --git a/documize/web/web.go b/documize/web/web.go index 9028e3c3..80faaa19 100644 --- a/documize/web/web.go +++ b/documize/web/web.go @@ -33,8 +33,8 @@ var SiteInfo struct { } func init() { - environment.GetString(&SiteMode, "offline", false, "set to '1' for OFFLINE mode", nil) - SiteInfo.DBhash = util.GenerateRandomPassword() // do this only once + environment.GetString(&SiteMode, "offline", false, "set to '1' for OFFLINE mode", nil) // no sense overriding this setting from the DB + SiteInfo.DBhash = util.GenerateRandomPassword() // do this only once } // EmberHandler provides the webserver for pages developed using the Ember programming environment. diff --git a/wordsmith/environment/environment.go b/wordsmith/environment/environment.go index 9aff20d4..d57805cc 100644 --- a/wordsmith/environment/environment.go +++ b/wordsmith/environment/environment.go @@ -10,11 +10,14 @@ import ( "strings" ) +// CallbackT is the type signature of the callback function of GetString(). +type CallbackT func(*string, string) bool + type varT struct { target *string name, setter, value string required bool - callback func() + callback CallbackT } type varsT struct { @@ -41,10 +44,10 @@ func (v *varsT) Less(i, j int) bool { // Prefix provides the prefix for all Environment variables const Prefix = "DOCUMIZE" -const goInit = "(default)" +const goInit = "(default)" // GetString sets-up the flag for later use, it must be called before ParseOK(), usually in an init(). -func GetString(target *string, name string, required bool, usage string, callback func()) { +func GetString(target *string, name string, required bool, usage string, callback CallbackT) { name = strings.ToLower(strings.TrimSpace(name)) setter := Prefix + strings.ToUpper(name) value := os.Getenv(setter) @@ -59,37 +62,43 @@ func GetString(target *string, name string, required bool, usage string, callbac // Parse calls flag.Parse() then checks that the required environment variables are all set. // It should be the first thing called by any main() that uses this library. // If all the required variables are not present, it prints an error and calls os.Exit(2) like flag.Parse(). -func Parse() { +func Parse(doFirst string) { flag.Parse() sort.Sort(&vars) - for vi, v := range vars.vv { - typ := "Optional" - if v.value != *(v.target) || (v.value != "" && *(v.target) == "") { - vars.vv[vi].setter = "-" + v.name // v is a local copy, not the underlying data - } - if v.required { - if *(v.target) == "" { - fmt.Fprintln(os.Stderr) - fmt.Fprintln(os.Stderr, "In order to run", os.Args[0], "the following must be provided:") - for _, vv := range vars.vv { - if vv.required { - fmt.Fprintf(os.Stderr, "* setting from environment variable '%s' or flag '-%s', current value: '%s' set by '%s'\n", - Prefix+strings.ToUpper(vv.name), vv.name, *(vv.target), vv.setter) + for pass := 1; pass <= 2; pass++ { + for vi, v := range vars.vv { + if (pass == 1 && v.name == doFirst) || (pass == 2 && v.name != doFirst) { + typ := "Optional" + if v.value != *(v.target) || (v.value != "" && *(v.target) == "") { + vars.vv[vi].setter = "-" + v.name // v is a local copy, not the underlying data + } + if v.callback != nil { + if v.callback(v.target, v.name) { + vars.vv[vi].setter = "setting:" + v.name // v is a local copy, not the underlying data } } - fmt.Fprintln(os.Stderr) - flag.Usage() - os.Exit(2) - return + if v.required { + if *(v.target) == "" { + fmt.Fprintln(os.Stderr) + fmt.Fprintln(os.Stderr, "In order to run", os.Args[0], "the following must be provided:") + for _, vv := range vars.vv { + if vv.required { + fmt.Fprintf(os.Stderr, "* setting from environment variable '%s' or flag '-%s' or an application setting '%s', current value: '%s' set by '%s'\n", + Prefix+strings.ToUpper(vv.name), vv.name, vv.name, *(vv.target), vv.setter) + } + } + fmt.Fprintln(os.Stderr) + flag.Usage() + os.Exit(2) + return + } + typ = "Required" + } + if *(v.target) != "" && vars.vv[vi].setter != goInit { + fmt.Fprintf(os.Stdout, "%s setting from '%s' is: '%s'\n", + typ, vars.vv[vi].setter, *(v.target)) + } } - typ = "Required" - } - if *(v.target) != "" && v.setter != goInit { - fmt.Fprintf(os.Stdout, "%s setting from '%s' is: '%s'\n", - typ, v.setter, *(v.target)) - } - if v.callback != nil { - v.callback() } } } diff --git a/wordsmith/log/logger.go b/wordsmith/log/logger.go index 6a1c467c..c0241e56 100644 --- a/wordsmith/log/logger.go +++ b/wordsmith/log/logger.go @@ -20,8 +20,9 @@ func init() { log.SetLevel(log.DebugLevel) env.GetString(&environment, "log", false, "system being logged e.g. 'PRODUCTION'", - func() { + func(*string, string) bool { log.Infoln(environment + " environment logging enabled") + return false }) }