From 97d90662ddd5b9d3c814a4d00deb00d7d6cbe9aa Mon Sep 17 00:00:00 2001 From: Harvey Kandola Date: Fri, 14 Sep 2018 18:00:24 +0100 Subject: [PATCH] Make database boot process storage provider agonistic Moved database queries into provider specific object to ensure database checking, installation, upgrade procedures are pluggable. --- core/database/check.go | 151 ++++++----------- core/database/installer.go | 45 ++++- core/database/leader.go | 224 ------------------------- core/database/lock.go | 110 ++++++++++++ core/database/scripts.go | 4 +- core/env/runtime.go | 103 ++++++------ edition/boot/runtime.go | 19 ++- edition/storage/mysql.go | 267 ++++++++++++++++++++++++++++++ edition/storage/mysql_provider.go | 67 -------- 9 files changed, 524 insertions(+), 466 deletions(-) delete mode 100644 core/database/leader.go create mode 100644 core/database/lock.go create mode 100644 edition/storage/mysql.go delete mode 100644 edition/storage/mysql_provider.go diff --git a/core/database/check.go b/core/database/check.go index 1b523b99..78331294 100644 --- a/core/database/check.go +++ b/core/database/check.go @@ -12,9 +12,7 @@ package database import ( - "errors" "fmt" - "strconv" "strings" "github.com/documize/community/core/env" @@ -22,25 +20,21 @@ import ( "github.com/documize/community/server/web" ) -var dbCheckOK bool // default false - // Check that the database is configured correctly and that all the required tables exist. // It must be the first function called in this package. func Check(runtime *env.Runtime) bool { - runtime.Log.Info("Database checks: started") + runtime.Log.Info("Database: checking state") - csBits := strings.Split(runtime.Flags.DBConn, "/") - if len(csBits) > 1 { - web.SiteInfo.DBname = strings.Split(csBits[len(csBits)-1], "?")[0] - } + web.SiteInfo.DBname = runtime.StoreProvider.DatabaseName() - rows, err := runtime.Db.Query("SELECT VERSION() AS version, @@version_comment as comment, @@character_set_database AS charset, @@collation_database AS collation") + rows, err := runtime.Db.Query(runtime.StoreProvider.QueryMeta()) if err != nil { - runtime.Log.Error("Can't get MySQL configuration", err) - web.SiteInfo.Issue = "Can't get MySQL configuration: " + err.Error() + runtime.Log.Error("Database: unable to load meta information from database provider", err) + web.SiteInfo.Issue = "Unable to load meta information from database provider: " + err.Error() runtime.Flags.SiteMode = env.SiteModeBadDB return false } + defer streamutil.Close(rows) var version, dbComment, charset, collation string if rows.Next() { @@ -50,121 +44,70 @@ func Check(runtime *env.Runtime) bool { err = rows.Err() // get any error encountered during iteration } if err != nil { - runtime.Log.Error("no MySQL configuration returned", err) - web.SiteInfo.Issue = "no MySQL configuration return issue: " + err.Error() + runtime.Log.Error("Database: no meta data returned by database provider", err) + web.SiteInfo.Issue = "No meta data returned by database provider: " + err.Error() runtime.Flags.SiteMode = env.SiteModeBadDB return false } - // runtime.DbVariant = GetSQLVariant(runtime.Flags.DBType, dbComment) - runtime.Log.Info(fmt.Sprintf("Database checks: SQL variant %v", runtime.Storage.Type)) - runtime.Log.Info("Database checks: SQL version " + version) + runtime.Log.Info(fmt.Sprintf("Database: provider name %v", runtime.StoreProvider.Type())) + runtime.Log.Info(fmt.Sprintf("Database: provider version %s", version)) - verNums, err := GetSQLVersion(version) - if err != nil { - runtime.Log.Error("Database version check failed", err) + // Version OK? + versionOK, minVersion := runtime.StoreProvider.VerfiyVersion(version) + if !versionOK { + msg := fmt.Sprintf("*** ERROR: database version needs to be %s or above ***", minVersion) + runtime.Log.Info(msg) + web.SiteInfo.Issue = msg + runtime.Flags.SiteMode = env.SiteModeBadDB + return false } - // Check minimum MySQL version as we need JSON column type. - verInts := []int{5, 7, 10} // Minimum MySQL version - if runtime.Storage.Type == env.StoreTypeMariaDB { - verInts = []int{10, 3, 0} // Minimum MariaDB version - } - - for k, v := range verInts { - // If major release is higher then skip minor/patch checks (e.g. 8.x.x > 5.x.x) - if k == 0 && len(verNums) > 0 && verNums[0] > verInts[0] { - break - } - if verNums[k] < v { - want := fmt.Sprintf("%d.%d.%d", verInts[0], verInts[1], verInts[2]) - runtime.Log.Error("MySQL version element "+strconv.Itoa(k+1)+" of '"+version+"' not high enough, need at least version "+want, errors.New("bad MySQL version")) - web.SiteInfo.Issue = "MySQL version element " + strconv.Itoa(k+1) + " of '" + version + "' not high enough, need at least version " + want - runtime.Flags.SiteMode = env.SiteModeBadDB - return false - } - } - - { // check the MySQL character set and collation - if charset != "utf8" && charset != "utf8mb4" { - runtime.Log.Error("MySQL character set not utf8/utf8mb4:", errors.New(charset)) - web.SiteInfo.Issue = "MySQL character set not utf8/utf8mb4: " + charset - runtime.Flags.SiteMode = env.SiteModeBadDB - return false - } - if !strings.HasPrefix(collation, "utf8") { - runtime.Log.Error("MySQL collation sequence not utf8...:", errors.New(collation)) - web.SiteInfo.Issue = "MySQL collation sequence not utf8...: " + collation - runtime.Flags.SiteMode = env.SiteModeBadDB - return false - } + // Character set and collation OK? + charOK, charRequired := runtime.StoreProvider.VerfiyCharacterCollation(charset, collation) + if !charOK { + msg := fmt.Sprintf("*** ERROR: %s ***", charRequired) + runtime.Log.Info(msg) + web.SiteInfo.Issue = msg + runtime.Flags.SiteMode = env.SiteModeBadDB + return false } { // if there are no rows in the database, enter set-up mode var flds []string - if err := runtime.Db.Select(&flds, - `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = '`+web.SiteInfo.DBname+ - `' and TABLE_TYPE='BASE TABLE'`); err != nil { - runtime.Log.Error("Can't get MySQL number of tables", err) - web.SiteInfo.Issue = "Can't get MySQL number of tables: " + err.Error() + if err := runtime.Db.Select(&flds, runtime.StoreProvider.QueryTableList()); err != nil { + msg := fmt.Sprintf("Database: unable to get database table list ") + runtime.Log.Error(msg, err) + web.SiteInfo.Issue = msg + err.Error() runtime.Flags.SiteMode = env.SiteModeBadDB return false } + if strings.TrimSpace(flds[0]) == "0" { - runtime.Log.Info("Entering database set-up mode because the database is empty.....") + runtime.Log.Info("Database: starting setup mode for empty database") runtime.Flags.SiteMode = env.SiteModeSetup return false } } - { // check all the required tables exist - var tables = []string{`account`, - `attachment`, `document`, - `label`, `organization`, - `page`, `revision`, `search`, `user`} + // Ensure no missing tables. + var tables = []string{"account", "attachment", "document", + "label", "organization", "page", "revision", "search", "user"} - for _, table := range tables { - var dummy []string - if err := runtime.Db.Select(&dummy, "SELECT 1 FROM "+table+" LIMIT 1;"); err != nil { - runtime.Log.Error("Entering bad database mode because: SELECT 1 FROM "+table+" LIMIT 1;", err) - web.SiteInfo.Issue = "MySQL database is not empty, but does not contain table: " + table - runtime.Flags.SiteMode = env.SiteModeBadDB - return false - } + for _, table := range tables { + var result []string + if err := runtime.Db.Select(&result, fmt.Sprintf("SELECT COUNT(*) FROM %s ;", table)); err != nil { + msg := fmt.Sprintf("Database: missing table %s", table) + runtime.Log.Error(msg, err) + web.SiteInfo.Issue = msg + runtime.Flags.SiteMode = env.SiteModeBadDB + return false } } - runtime.Flags.SiteMode = env.SiteModeNormal // actually no need to do this (as already ""), this for documentation - web.SiteInfo.DBname = "" // do not give this info when not in set-up mode - dbCheckOK = true + // We have good database, so proceed with app boot process. + runtime.Flags.SiteMode = env.SiteModeNormal + web.SiteInfo.DBname = "" + return true } - -// GetSQLVersion returns SQL version as major,minor,patch numerics. -func GetSQLVersion(v string) (ints []int, err error) { - ints = []int{0, 0, 0} - - pos := strings.Index(v, "-") - if pos > 1 { - v = v[:pos] - } - - vs := strings.Split(v, ".") - - if len(vs) < 3 { - err = errors.New("MySQL version not of the form a.b.c") - return - } - - for key, val := range vs { - num, err := strconv.Atoi(val) - - if err != nil { - return ints, err - } - - ints[key] = num - } - - return -} diff --git a/core/database/installer.go b/core/database/installer.go index 4beb0d22..74815761 100644 --- a/core/database/installer.go +++ b/core/database/installer.go @@ -14,6 +14,7 @@ package database import ( "fmt" "regexp" + "strconv" "strings" "time" @@ -35,11 +36,11 @@ func InstallUpgrade(runtime *env.Runtime, existingDB bool) (err error) { // Filter out database specific scripts. dbTypeScripts := SpecificScripts(runtime, scripts) if len(dbTypeScripts) == 0 { - runtime.Log.Info(fmt.Sprintf("Database: unable to load scripts for database type %s", runtime.Storage.Type)) + runtime.Log.Info(fmt.Sprintf("Database: unable to load scripts for database type %s", runtime.StoreProvider.Type())) return } - runtime.Log.Info(fmt.Sprintf("Database: loaded %d SQL scripts for provider %s", len(dbTypeScripts), runtime.Storage.Type)) + runtime.Log.Info(fmt.Sprintf("Database: loaded %d SQL scripts for provider %s", len(dbTypeScripts), runtime.StoreProvider.Type())) // Get current database version. currentVersion := 0 @@ -114,13 +115,13 @@ func runScripts(runtime *env.Runtime, tx *sqlx.Tx, scripts []Script) (err error) for _, script := range scripts { runtime.Log.Info(fmt.Sprintf("Databasse: processing SQL version %d", script.Version)) - err = executeSQL(tx, runtime.Storage.Type, script.Script) + err = executeSQL(tx, runtime.StoreProvider.Type(), script.Script) if err != nil { return err } // Record the fact we have processed this database script version. - _, err = tx.Exec(recordVersionUpgradeQuery(runtime.Storage.Type, script.Version)) + _, err = tx.Exec(runtime.StoreProvider.QueryRecordVersionUpgrade(script.Version)) if err != nil { return err } @@ -170,3 +171,39 @@ func getStatements(bytes []byte) (stmts []string) { return } + +// CurrentVersion returns number that represents the current database version number. +// For example 23 represents the 23rd iteration of the database. +func CurrentVersion(runtime *env.Runtime) (version int, err error) { + row := runtime.Db.QueryRow(runtime.StoreProvider.QueryGetDatabaseVersion()) + + var currentVersion string + err = row.Scan(¤tVersion) + if err != nil { + currentVersion = "0" + } + + return extractVersionNumber(currentVersion), nil +} + +// Turns legacy "db_00021.sql" and new "21" format into version number 21. +func extractVersionNumber(s string) int { + // Good practice in case of human tampering. + s = strings.TrimSpace(s) + s = strings.ToLower(s) + + // Remove any quotes from JSON string. + s = strings.Replace(s, "\"", "", -1) + + // Remove legacy version string formatting. + // We know just store the number. + s = strings.Replace(s, "db_000", "", 1) + s = strings.Replace(s, ".sql", "", 1) + + i, err := strconv.Atoi(s) + if err != nil { + i = 0 + } + + return i +} diff --git a/core/database/leader.go b/core/database/leader.go deleted file mode 100644 index afeaeae2..00000000 --- a/core/database/leader.go +++ /dev/null @@ -1,224 +0,0 @@ -// Copyright 2016 Documize Inc. . All rights reserved. -// -// This software (Documize Community Edition) is licensed under -// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html -// -// You can operate outside the AGPL restrictions by purchasing -// Documize Enterprise Edition and obtaining a commercial license -// by contacting . -// -// https://documize.com - -package database - -import ( - "crypto/rand" - "fmt" - "os" - "strconv" - "strings" - "time" - - "github.com/documize/community/core/env" - "github.com/jmoiron/sqlx" -) - -// Lock will try to lock the database instance to the running process. -// Uses a "random" delay as a por man's database cluster-aware process. -// We skip delay if there are no scripts to process. -func Lock(runtime *env.Runtime, scriptsToProcess int) (bool, error) { - // Wait for random period of time. - b := make([]byte, 2) - _, err := rand.Read(b) - if err != nil { - return false, err - } - wait := ((time.Duration(b[0]) << 8) | time.Duration(b[1])) * time.Millisecond / 10 // up to 6.5 secs wait - - // Why delay if nothing to process? - if scriptsToProcess > 0 { - time.Sleep(wait) - } - - // Start transaction fotr lock process. - tx, err := runtime.Db.Beginx() - if err != nil { - runtime.Log.Error("Database: unable to start transaction", err) - return false, err - } - - // Lock the database. - _, err = tx.Exec(processLockStartQuery(runtime.Storage.Type)) - if err != nil { - runtime.Log.Error("Database: unable to lock tables", err) - return false, err - } - - // Unlock the database at the end of this function. - defer func() { - _, err = tx.Exec(processLockFinishQuery(runtime.Storage.Type)) - if err != nil { - runtime.Log.Error("Database: unable to unlock tables", err) - } - tx.Commit() - }() - - // Try to record this process as leader of database migration process. - _, err = tx.Exec(insertProcessIDQuery(runtime.Storage.Type)) - if err != nil { - runtime.Log.Info("Database: marked as slave process awaiting upgrade") - return false, nil - } - - // We are the leader! - runtime.Log.Info("Database: marked as database upgrade process leader") - return true, err -} - -// Unlock completes process that was started with Lock(). -func Unlock(runtime *env.Runtime, tx *sqlx.Tx, err error, amLeader bool) error { - if amLeader { - defer func() { - doUnlock(runtime) - }() - - if tx != nil { - if err == nil { - tx.Commit() - runtime.Log.Info("Database: is ready") - return nil - } - tx.Rollback() - } - - runtime.Log.Error("Database: install/upgrade failed", err) - - return err - } - - return nil // not the leader, so ignore errors -} - -// CurrentVersion returns number that represents the current database version number. -// For example 23 represents the 23rd iteration of the database. -func CurrentVersion(runtime *env.Runtime) (version int, err error) { - row := runtime.Db.QueryRow(databaseVersionQuery(runtime.Storage.Type)) - - var currentVersion string - err = row.Scan(¤tVersion) - if err != nil { - currentVersion = "0" - } - - return extractVersionNumber(currentVersion), nil -} - -// Helper method for defer function called from Unlock(). -func doUnlock(runtime *env.Runtime) error { - tx, err := runtime.Db.Beginx() - if err != nil { - return err - } - _, err = tx.Exec(deleteProcessIDQuery(runtime.Storage.Type)) - if err != nil { - return err - } - - return tx.Commit() -} - -// processLockStartQuery returns database specific query that will -// LOCK the database to this running process. -func processLockStartQuery(t env.StoreType) string { - switch t { - case env.StoreTypeMySQL, env.StoreTypeMariaDB, env.StoreTypePercona: - return "LOCK TABLE `config` WRITE;" - case env.StoreTypePostgreSQL: - return "" - case env.StoreTypeMSSQL: - return "" - } - - return "" -} - -// processLockFinishQuery returns database specific query that will -// UNLOCK the database from this running process. -func processLockFinishQuery(t env.StoreType) string { - switch t { - case env.StoreTypeMySQL, env.StoreTypeMariaDB, env.StoreTypePercona: - return "UNLOCK TABLES;" - case env.StoreTypePostgreSQL: - return "" - case env.StoreTypeMSSQL: - return "" - } - - return "" -} - -// insertProcessIDQuery returns database specific query that will -// insert ID of this running process. -func insertProcessIDQuery(t env.StoreType) string { - return "INSERT INTO `config` (`key`,`config`) " + fmt.Sprintf(`VALUES ('DBLOCK','{"pid": "%d"}');`, os.Getpid()) -} - -// deleteProcessIDQuery returns database specific query that will -// delete ID of this running process. -func deleteProcessIDQuery(t env.StoreType) string { - return "DELETE FROM `config` WHERE `key`='DBLOCK';" -} - -// recordVersionUpgradeQuery returns database specific insert statement -// that records the database version number -func recordVersionUpgradeQuery(t env.StoreType, version int) string { - // Make record that holds new database version number. - json := fmt.Sprintf("{\"database\": \"%d\"}", version) - - switch t { - case env.StoreTypeMySQL, env.StoreTypeMariaDB, env.StoreTypePercona: - return "INSERT INTO `config` (`key`,`config`) " + "VALUES ('META','" + json + "') ON DUPLICATE KEY UPDATE `config`='" + json + "';" - case env.StoreTypePostgreSQL: - return "" - case env.StoreTypeMSSQL: - return "" - } - - return "" -} - -// databaseVersionQuery returns the schema version number. -func databaseVersionQuery(t env.StoreType) string { - switch t { - case env.StoreTypeMySQL, env.StoreTypeMariaDB, env.StoreTypePercona: - return "SELECT JSON_EXTRACT(`config`,'$.database') FROM `config` WHERE `key` = 'META';" - case env.StoreTypePostgreSQL: - return "" - case env.StoreTypeMSSQL: - return "" - } - - return "" -} - -// Turns legacy "db_00021.sql" and new "21" format into version number 21. -func extractVersionNumber(s string) int { - // Good practice in case of human tampering. - s = strings.TrimSpace(s) - s = strings.ToLower(s) - - // Remove any quotes from JSON string. - s = strings.Replace(s, "\"", "", -1) - - // Remove legacy version string formatting. - // We know just store the number. - s = strings.Replace(s, "db_000", "", 1) - s = strings.Replace(s, ".sql", "", 1) - - i, err := strconv.Atoi(s) - if err != nil { - i = 0 - } - - return i -} diff --git a/core/database/lock.go b/core/database/lock.go new file mode 100644 index 00000000..86ea7ce5 --- /dev/null +++ b/core/database/lock.go @@ -0,0 +1,110 @@ +// Copyright 2016 Documize Inc. . All rights reserved. +// +// This software (Documize Community Edition) is licensed under +// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html +// +// You can operate outside the AGPL restrictions by purchasing +// Documize Enterprise Edition and obtaining a commercial license +// by contacting . +// +// https://documize.com + +package database + +import ( + "crypto/rand" + "time" + + "github.com/documize/community/core/env" + "github.com/jmoiron/sqlx" +) + +// Lock will try to lock the database instance to the running process. +// Uses a "random" delay as a por man's database cluster-aware process. +// We skip delay if there are no scripts to process. +func Lock(runtime *env.Runtime, scriptsToProcess int) (bool, error) { + // Wait for random period of time. + b := make([]byte, 2) + _, err := rand.Read(b) + if err != nil { + return false, err + } + wait := ((time.Duration(b[0]) << 8) | time.Duration(b[1])) * time.Millisecond / 10 // up to 6.5 secs wait + + // Why delay if nothing to process? + if scriptsToProcess > 0 { + time.Sleep(wait) + } + + // Start transaction fotr lock process. + tx, err := runtime.Db.Beginx() + if err != nil { + runtime.Log.Error("Database: unable to start transaction", err) + return false, err + } + + // Lock the database. + _, err = tx.Exec(runtime.StoreProvider.QueryStartLock()) + if err != nil { + runtime.Log.Error("Database: unable to lock tables", err) + return false, err + } + + // Unlock the database at the end of this function. + defer func() { + _, err = tx.Exec(runtime.StoreProvider.QueryFinishLock()) + if err != nil { + runtime.Log.Error("Database: unable to unlock tables", err) + } + tx.Commit() + }() + + // Try to record this process as leader of database migration process. + _, err = tx.Exec(runtime.StoreProvider.QueryInsertProcessID()) + if err != nil { + runtime.Log.Info("Database: marked as slave process awaiting upgrade") + return false, nil + } + + // We are the leader! + runtime.Log.Info("Database: marked as database upgrade process leader") + return true, err +} + +// Unlock completes process that was started with Lock(). +func Unlock(runtime *env.Runtime, tx *sqlx.Tx, err error, amLeader bool) error { + if amLeader { + defer func() { + doUnlock(runtime) + }() + + if tx != nil { + if err == nil { + tx.Commit() + runtime.Log.Info("Database: is ready") + return nil + } + tx.Rollback() + } + + runtime.Log.Error("Database: install/upgrade failed", err) + + return err + } + + return nil // not the leader, so ignore errors +} + +// Helper method for defer function called from Unlock(). +func doUnlock(runtime *env.Runtime) error { + tx, err := runtime.Db.Beginx() + if err != nil { + return err + } + _, err = tx.Exec(runtime.StoreProvider.QueryDeleteProcessID()) + if err != nil { + return err + } + + return tx.Commit() +} diff --git a/core/database/scripts.go b/core/database/scripts.go index 8dd4b11a..8e6b7aed 100644 --- a/core/database/scripts.go +++ b/core/database/scripts.go @@ -13,9 +13,9 @@ package database import ( "fmt" - "github.com/documize/community/core/env" "sort" + "github.com/documize/community/core/env" "github.com/documize/community/server/web" ) @@ -47,7 +47,7 @@ func LoadScripts() (s Scripts, err error) { // SpecificScripts returns SQL scripts for current databasse provider. func SpecificScripts(runtime *env.Runtime, all Scripts) (s []Script) { - switch runtime.Storage.Type { + switch runtime.StoreProvider.Type() { case env.StoreTypeMySQL, env.StoreTypeMariaDB, env.StoreTypePercona: return all.MySQLScripts case env.StoreTypePostgreSQL: diff --git a/core/env/runtime.go b/core/env/runtime.go index 341b7d3c..de9dbc16 100644 --- a/core/env/runtime.go +++ b/core/env/runtime.go @@ -14,7 +14,6 @@ package env import ( "github.com/jmoiron/sqlx" - "strings" ) // SQL-STORE: DbVariant needs to be struct like: name, delims, std params and conn string method @@ -22,11 +21,11 @@ import ( // Runtime provides access to database, logger and other server-level scoped objects. // Use Context for per-request values. type Runtime struct { - Flags Flags - Db *sqlx.DB - Storage StoreProvider - Log Logger - Product ProdInfo + Flags Flags + Db *sqlx.DB + StoreProvider StoreProvider + Log Logger + Product ProdInfo } const ( @@ -63,69 +62,61 @@ const ( StoreTypeMSSQL StoreType = "MSSQL" ) -// StoreProvider contains database specific details -type StoreProvider struct { - // Type identifies storage provider - Type StoreType +// StoreProvider defines a database provider. +type StoreProvider interface { + // Name of provider + Type() StoreType // SQL driver name used to open DB connection. - DriverName string + DriverName() string - // Database connection string parameters that must be present before connecting to DB - Params map[string]string + // Database connection string parameters that must be present before connecting to DB. + Params() map[string]string - // Example holds storage provider specific connection string format + // Example holds storage provider specific connection string format. // used in error messages - Example string -} + Example() string -// ConnectionString returns provider specific DB connection string -// complete with default parameters. -func (s *StoreProvider) ConnectionString(cs string) string { - switch s.Type { + // DatabaseName holds the SQL database name where Documize tables live. + DatabaseName() string - case StoreTypePostgreSQL: - return "pg" + // Make connection string with default parameters. + MakeConnectionString() string - case StoreTypeMSSQL: - return "sql server" + // QueryMeta is how to extract version number, collation, character set from database provider. + QueryMeta() string - case StoreTypeMySQL, StoreTypeMariaDB, StoreTypePercona: - queryBits := strings.Split(cs, "?") - ret := queryBits[0] + "?" - retFirst := true + // QueryStartLock locks database tables. + QueryStartLock() string - if len(queryBits) == 2 { - paramBits := strings.Split(queryBits[1], "&") - for _, pb := range paramBits { - found := false - if assignBits := strings.Split(pb, "="); len(assignBits) == 2 { - _, found = s.Params[strings.TrimSpace(assignBits[0])] - } - if !found { // if we can't work out what it is, put it through - if retFirst { - retFirst = false - } else { - ret += "&" - } - ret += pb - } - } - } + // QueryFinishLock unlocks database tables. + QueryFinishLock() string - for k, v := range s.Params { - if retFirst { - retFirst = false - } else { - ret += "&" - } - ret += k + "=" + v - } + // QueryInsertProcessID returns database specific query that will + // insert ID of this running process. + QueryInsertProcessID() string - return ret - } + // QueryInsertProcessID returns database specific query that will + // delete ID of this running process. + QueryDeleteProcessID() string - return "" + // QueryRecordVersionUpgrade returns database specific insert statement + // that records the database version number. + QueryRecordVersionUpgrade(version int) string + + // QueryGetDatabaseVersion returns the schema version number. + QueryGetDatabaseVersion() string + + // QueryTableList returns a list tables in Documize database. + QueryTableList() string + + // VerfiyVersion checks to see if actual database meets + // minimum version requirements. + VerfiyVersion(dbVersion string) (versionOK bool, minVerRequired string) + + // VerfiyCharacterCollation checks to see if actual database + // has correct character set and collation settings. + VerfiyCharacterCollation(charset, collation string) (charOK bool, requirements string) } const ( diff --git a/edition/boot/runtime.go b/edition/boot/runtime.go index 159f9b67..06dc0ce5 100644 --- a/edition/boot/runtime.go +++ b/edition/boot/runtime.go @@ -47,41 +47,42 @@ func InitRuntime(r *env.Runtime, s *domain.Store) bool { } } - // Work out required storage provider set it up. + // Set up required storage provider. switch r.Flags.DBType { case "mysql": - r.Storage = env.StoreProvider{Type: env.StoreTypeMySQL, DriverName: "mysql"} storage.SetMySQLProvider(r, s) case "mariadb": - r.Storage = env.StoreProvider{Type: env.StoreTypeMariaDB, DriverName: "mysql"} storage.SetMySQLProvider(r, s) case "percona": - r.Storage = env.StoreProvider{Type: env.StoreTypePercona, DriverName: "mysql"} storage.SetMySQLProvider(r, s) case "pggg": - r.Storage = env.StoreProvider{Type: env.StoreTypePercona, DriverName: "pgggggg"} // storage.SetPostgresSQLProvider(r, s) case "mssql": - r.Storage = env.StoreProvider{Type: env.StoreTypePercona, DriverName: "sqlserver"} // storage.SetSQLServerProvider(r, s) } // Open connection to database - db, err := sqlx.Open(r.Storage.DriverName, r.Storage.ConnectionString(r.Flags.DBConn)) + db, err := sqlx.Open(r.StoreProvider.DriverName(), r.StoreProvider.MakeConnectionString()) //r.Flags.DBConn if err != nil { r.Log.Error("unable to setup database", err) } + + // Database handle r.Db = db + + // Database connection defaults r.Db.SetMaxIdleConns(30) r.Db.SetMaxOpenConns(100) r.Db.SetConnMaxLifetime(time.Second * 14400) + + // Database good? err = r.Db.Ping() if err != nil { - r.Log.Error("unable to connect to database - "+r.Storage.Example, err) + r.Log.Error("unable to connect to database - "+r.StoreProvider.Example(), err) return false } - // Go into setup mode if required. + // Check database and upgrade if required. if r.Flags.SiteMode != env.SiteModeOffline { if database.Check(r) { if err := database.InstallUpgrade(r, true); err != nil { diff --git a/edition/storage/mysql.go b/edition/storage/mysql.go new file mode 100644 index 00000000..4ee2c1ab --- /dev/null +++ b/edition/storage/mysql.go @@ -0,0 +1,267 @@ +// Copyright 2016 Documize Inc. . All rights reserved. +// +// This software (Documize Community Edition) is licensed under +// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html +// +// You can operate outside the AGPL restrictions by purchasing +// Documize Enterprise Edition and obtaining a commercial license +// by contacting . +// +// https://documize.com + +// Package storage sets up database persistence providers. +package storage + +import ( + "errors" + "fmt" + "os" + "strconv" + "strings" + + "github.com/documize/community/core/env" + "github.com/documize/community/domain" + account "github.com/documize/community/domain/account/mysql" + activity "github.com/documize/community/domain/activity/mysql" + attachment "github.com/documize/community/domain/attachment/mysql" + audit "github.com/documize/community/domain/audit/mysql" + block "github.com/documize/community/domain/block/mysql" + category "github.com/documize/community/domain/category/mysql" + doc "github.com/documize/community/domain/document/mysql" + group "github.com/documize/community/domain/group/mysql" + link "github.com/documize/community/domain/link/mysql" + meta "github.com/documize/community/domain/meta/mysql" + org "github.com/documize/community/domain/organization/mysql" + page "github.com/documize/community/domain/page/mysql" + permission "github.com/documize/community/domain/permission/mysql" + pin "github.com/documize/community/domain/pin/mysql" + search "github.com/documize/community/domain/search/mysql" + setting "github.com/documize/community/domain/setting/mysql" + space "github.com/documize/community/domain/space/mysql" + user "github.com/documize/community/domain/user/mysql" +) + +// SetMySQLProvider creates MySQL provider +func SetMySQLProvider(r *env.Runtime, s *domain.Store) { + // Set up provider specific details. + r.StoreProvider = MySQLProvider{ + ConnectionString: r.Flags.DBConn, + Variant: r.Flags.DBType, + } + + // Wire up data providers! + s.Account = account.Scope{Runtime: r} + s.Activity = activity.Scope{Runtime: r} + s.Attachment = attachment.Scope{Runtime: r} + s.Audit = audit.Scope{Runtime: r} + s.Block = block.Scope{Runtime: r} + s.Category = category.Scope{Runtime: r} + s.Document = doc.Scope{Runtime: r} + s.Group = group.Scope{Runtime: r} + s.Link = link.Scope{Runtime: r} + s.Meta = meta.Scope{Runtime: r} + s.Organization = org.Scope{Runtime: r} + s.Page = page.Scope{Runtime: r} + s.Pin = pin.Scope{Runtime: r} + s.Permission = permission.Scope{Runtime: r} + s.Search = search.Scope{Runtime: r} + s.Setting = setting.Scope{Runtime: r} + s.Space = space.Scope{Runtime: r} + s.User = user.Scope{Runtime: r} +} + +// MySQLProvider supports MySQL 5.7.x and 8.0.x versions. +type MySQLProvider struct { + // User specified connection string. + ConnectionString string + + // User specified db type (mysql, percona or mariadb). + Variant string +} + +// Type returns name of provider +func (p MySQLProvider) Type() env.StoreType { + return env.StoreTypeMySQL +} + +// DriverName returns database/sql driver name. +func (p MySQLProvider) DriverName() string { + return "mysql" +} + +// Params returns connection string parameters that must be present before connecting to DB. +func (p MySQLProvider) Params() map[string]string { + return map[string]string{ + "charset": "utf8mb4", + "parseTime": "True", + "maxAllowedPacket": "104857600", // 4194304 // 16777216 = 16MB // 104857600 = 100MB + } +} + +// Example holds storage provider specific connection string format. +// used in error messages +func (p MySQLProvider) Example() string { + return "database connection string format is 'username:password@tcp(host:3306)/database'" +} + +// DatabaseName holds the SQL database name where Documize tables live. +func (p MySQLProvider) DatabaseName() string { + bits := strings.Split(p.ConnectionString, "/") + if len(bits) > 1 { + return strings.Split(bits[len(bits)-1], "?")[0] + } + + return "" +} + +// MakeConnectionString returns provider specific DB connection string +// complete with default parameters. +func (p MySQLProvider) MakeConnectionString() string { + queryBits := strings.Split(p.ConnectionString, "?") + ret := queryBits[0] + "?" + retFirst := true + + params := p.Params() + + if len(queryBits) == 2 { + paramBits := strings.Split(queryBits[1], "&") + for _, pb := range paramBits { + found := false + if assignBits := strings.Split(pb, "="); len(assignBits) == 2 { + _, found = params[strings.TrimSpace(assignBits[0])] + } + if !found { // if we can't work out what it is, put it through + if retFirst { + retFirst = false + } else { + ret += "&" + } + ret += pb + } + } + } + + for k, v := range params { + if retFirst { + retFirst = false + } else { + ret += "&" + } + ret += k + "=" + v + } + + return ret +} + +// QueryMeta is how to extract version number, collation, character set from database provider. +func (p MySQLProvider) QueryMeta() string { + return "SELECT VERSION() AS version, @@version_comment as comment, @@character_set_database AS charset, @@collation_database AS collation" +} + +// QueryStartLock locks database tables. +func (p MySQLProvider) QueryStartLock() string { + return "LOCK TABLE `config` WRITE;" +} + +// QueryFinishLock unlocks database tables. +func (p MySQLProvider) QueryFinishLock() string { + return "UNLOCK TABLES;" +} + +// QueryInsertProcessID returns database specific query that will +// insert ID of this running process. +func (p MySQLProvider) QueryInsertProcessID() string { + return "INSERT INTO `config` (`key`,`config`) " + fmt.Sprintf(`VALUES ('DBLOCK','{"pid": "%d"}');`, os.Getpid()) +} + +// QueryDeleteProcessID returns database specific query that will +// delete ID of this running process. +func (p MySQLProvider) QueryDeleteProcessID() string { + return "DELETE FROM `config` WHERE `key`='DBLOCK';" +} + +// QueryRecordVersionUpgrade returns database specific insert statement +// that records the database version number. +func (p MySQLProvider) QueryRecordVersionUpgrade(version int) string { + // Make record that holds new database version number. + json := fmt.Sprintf("{\"database\": \"%d\"}", version) + return "INSERT INTO `config` (`key`,`config`) " + "VALUES ('META','" + json + "') ON DUPLICATE KEY UPDATE `config`='" + json + "';" +} + +// QueryGetDatabaseVersion returns the schema version number. +func (p MySQLProvider) QueryGetDatabaseVersion() string { + return "SELECT JSON_EXTRACT(`config`,'$.database') FROM `config` WHERE `key` = 'META';" +} + +// QueryTableList returns a list tables in Documize database. +func (p MySQLProvider) QueryTableList() string { + return `SELECT COUNT(*) FROM information_schema.tables + WHERE table_schema = '` + p.DatabaseName() + `' and TABLE_TYPE='BASE TABLE'` +} + +// VerfiyVersion checks to see if actual database meets +// minimum version requirements. +func (p MySQLProvider) VerfiyVersion(dbVersion string) (bool, string) { + // Minimum MySQL / MariaDB version. + minVer := []int{5, 7, 10} + if p.Variant == "mariadb" { + minVer = []int{10, 3, 0} + } + + // Convert string to semver. + dbSemver, _ := convertDatabaseVersion(dbVersion) + + for k, v := range minVer { + // If major release is higher then skip minor/patch checks (e.g. 8.x.x > 5.x.x) + if k == 0 && len(dbSemver) > 0 && dbSemver[0] > minVer[0] { + break + } + if dbSemver[k] < v { + want := fmt.Sprintf("%d.%d.%d", minVer[0], minVer[1], minVer[2]) + return false, want + } + } + + return true, "" +} + +// VerfiyCharacterCollation needs to ensure utf8/utf8mb4. +func (p MySQLProvider) VerfiyCharacterCollation(charset, collation string) (charOK bool, requirements string) { + if charset != "utf8" && charset != "utf8mb4" { + return false, "MySQL character needs to be utf8/utf8mb4" + } + if !strings.HasPrefix(collation, "utf8") { + return false, "MySQL collation sequence needs to be utf8" + } + + return true, "" +} + +// convertDatabaseVersion turns database version string as major,minor,patch numerics. +func convertDatabaseVersion(v string) (ints []int, err error) { + ints = []int{0, 0, 0} + + pos := strings.Index(v, "-") + if pos > 1 { + v = v[:pos] + } + + vs := strings.Split(v, ".") + + if len(vs) < 3 { + err = errors.New("MySQL version not of the form a.b.c") + return + } + + for key, val := range vs { + num, err := strconv.Atoi(val) + + if err != nil { + return ints, err + } + + ints[key] = num + } + + return +} diff --git a/edition/storage/mysql_provider.go b/edition/storage/mysql_provider.go deleted file mode 100644 index 0a6ecc63..00000000 --- a/edition/storage/mysql_provider.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2016 Documize Inc. . All rights reserved. -// -// This software (Documize Community Edition) is licensed under -// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html -// -// You can operate outside the AGPL restrictions by purchasing -// Documize Enterprise Edition and obtaining a commercial license -// by contacting . -// -// https://documize.com - -// Package storage sets up database persistence providers. -package storage - -import ( - "github.com/documize/community/core/env" - "github.com/documize/community/domain" - account "github.com/documize/community/domain/account/mysql" - activity "github.com/documize/community/domain/activity/mysql" - attachment "github.com/documize/community/domain/attachment/mysql" - audit "github.com/documize/community/domain/audit/mysql" - block "github.com/documize/community/domain/block/mysql" - category "github.com/documize/community/domain/category/mysql" - doc "github.com/documize/community/domain/document/mysql" - group "github.com/documize/community/domain/group/mysql" - link "github.com/documize/community/domain/link/mysql" - meta "github.com/documize/community/domain/meta/mysql" - org "github.com/documize/community/domain/organization/mysql" - page "github.com/documize/community/domain/page/mysql" - permission "github.com/documize/community/domain/permission/mysql" - pin "github.com/documize/community/domain/pin/mysql" - search "github.com/documize/community/domain/search/mysql" - setting "github.com/documize/community/domain/setting/mysql" - space "github.com/documize/community/domain/space/mysql" - user "github.com/documize/community/domain/user/mysql" -) - -// SetMySQLProvider creates MySQL provider -func SetMySQLProvider(r *env.Runtime, s *domain.Store) { - // Required connection string parameters and defaults. - r.Storage.Params = map[string]string{ - "charset": "utf8mb4", - "parseTime": "True", - "maxAllowedPacket": "104857600", // 4194304 // 16777216 = 16MB // 104857600 = 100MB - } - - r.Storage.Example = "database connection string format is 'username:password@tcp(host:3306)/database'" - - s.Account = account.Scope{Runtime: r} - s.Activity = activity.Scope{Runtime: r} - s.Attachment = attachment.Scope{Runtime: r} - s.Audit = audit.Scope{Runtime: r} - s.Block = block.Scope{Runtime: r} - s.Category = category.Scope{Runtime: r} - s.Document = doc.Scope{Runtime: r} - s.Group = group.Scope{Runtime: r} - s.Link = link.Scope{Runtime: r} - s.Meta = meta.Scope{Runtime: r} - s.Organization = org.Scope{Runtime: r} - s.Page = page.Scope{Runtime: r} - s.Pin = pin.Scope{Runtime: r} - s.Permission = permission.Scope{Runtime: r} - s.Search = search.Scope{Runtime: r} - s.Setting = setting.Scope{Runtime: r} - s.Space = space.Scope{Runtime: r} - s.User = user.Scope{Runtime: r} -}