Docker/documize
Docker/documize
1
0
Fork 0
mirror of https://github.com/documize/community.git synced 2025-07-21 22:29:41 +02:00
Code Releases Activity

Make database boot process storage provider agonistic

Browse source 
Moved database queries into provider specific object to ensure database checking, installation, upgrade procedures are pluggable.
This commit is contained in:
Harvey Kandola 2018-09-14 18:00:24 +01:00
parent 4c733ce581
commit 97d90662dd
9 changed files with 524 additions and 466 deletions
Download patch file Download diff file Expand all files Collapse all files

137
core/database/check.go
View file

@ -12,9 +12,7 @@
package database package database
import ( import (
"errors"
"fmt" "fmt"
"strconv"
"strings" "strings"
"github.com/documize/community/core/env" "github.com/documize/community/core/env"
@ -22,25 +20,21 @@ import (
"github.com/documize/community/server/web" "github.com/documize/community/server/web"
) )
var dbCheckOK bool // default false
// Check that the database is configured correctly and that all the required tables exist. // Check that the database is configured correctly and that all the required tables exist.
// It must be the first function called in this package. // It must be the first function called in this package.
func Check(runtime *env.Runtime) bool { func Check(runtime *env.Runtime) bool {
runtime.Log.Info("Database checks: started") runtime.Log.Info("Database: checking state")
csBits := strings.Split(runtime.Flags.DBConn, "/") web.SiteInfo.DBname = runtime.StoreProvider.DatabaseName()
if len(csBits) > 1 {
web.SiteInfo.DBname = strings.Split(csBits[len(csBits)-1], "?")[0]
}
rows, err := runtime.Db.Query("SELECT VERSION() AS version, @@version_comment as comment, @@character_set_database AS charset, @@collation_database AS collation") rows, err := runtime.Db.Query(runtime.StoreProvider.QueryMeta())
if err != nil { if err != nil {
runtime.Log.Error("Can't get MySQL configuration", err) runtime.Log.Error("Database: unable to load meta information from database provider", err)
web.SiteInfo.Issue = "Can't get MySQL configuration: " + err.Error() web.SiteInfo.Issue = "Unable to load meta information from database provider: " + err.Error()
runtime.Flags.SiteMode = env.SiteModeBadDB runtime.Flags.SiteMode = env.SiteModeBadDB
return false return false
} }
defer streamutil.Close(rows) defer streamutil.Close(rows)
var version, dbComment, charset, collation string var version, dbComment, charset, collation string
if rows.Next() { if rows.Next() {
@ -50,121 +44,70 @@ func Check(runtime *env.Runtime) bool {
err = rows.Err() // get any error encountered during iteration err = rows.Err() // get any error encountered during iteration
} }
if err != nil { if err != nil {
runtime.Log.Error("no MySQL configuration returned", err) runtime.Log.Error("Database: no meta data returned by database provider", err)
web.SiteInfo.Issue = "no MySQL configuration return issue: " + err.Error() web.SiteInfo.Issue = "No meta data returned by database provider: " + err.Error()
runtime.Flags.SiteMode = env.SiteModeBadDB runtime.Flags.SiteMode = env.SiteModeBadDB
return false return false
} }
// runtime.DbVariant = GetSQLVariant(runtime.Flags.DBType, dbComment)
runtime.Log.Info(fmt.Sprintf("Database checks: SQL variant %v", runtime.Storage.Type)) runtime.Log.Info(fmt.Sprintf("Database: provider name %v", runtime.StoreProvider.Type()))
runtime.Log.Info("Database checks: SQL version " + version) runtime.Log.Info(fmt.Sprintf("Database: provider version %s", version))
verNums, err := GetSQLVersion(version) // Version OK?
if err != nil { versionOK, minVersion := runtime.StoreProvider.VerfiyVersion(version)
runtime.Log.Error("Database version check failed", err) if !versionOK {
} msg := fmt.Sprintf("*** ERROR: database version needs to be %s or above ***", minVersion)
runtime.Log.Info(msg)
// Check minimum MySQL version as we need JSON column type. web.SiteInfo.Issue = msg
verInts := []int{5, 7, 10} // Minimum MySQL version
if runtime.Storage.Type == env.StoreTypeMariaDB {
verInts = []int{10, 3, 0} // Minimum MariaDB version
}
for k, v := range verInts {
// If major release is higher then skip minor/patch checks (e.g. 8.x.x > 5.x.x)
if k == 0 && len(verNums) > 0 && verNums[0] > verInts[0] {
break
}
if verNums[k] < v {
want := fmt.Sprintf("%d.%d.%d", verInts[0], verInts[1], verInts[2])
runtime.Log.Error("MySQL version element "+strconv.Itoa(k+1)+" of '"+version+"' not high enough, need at least version "+want, errors.New("bad MySQL version"))
web.SiteInfo.Issue = "MySQL version element " + strconv.Itoa(k+1) + " of '" + version + "' not high enough, need at least version " + want
runtime.Flags.SiteMode = env.SiteModeBadDB runtime.Flags.SiteMode = env.SiteModeBadDB
return false return false
} }
}
{ // check the MySQL character set and collation // Character set and collation OK?
if charset != "utf8" && charset != "utf8mb4" { charOK, charRequired := runtime.StoreProvider.VerfiyCharacterCollation(charset, collation)
runtime.Log.Error("MySQL character set not utf8/utf8mb4:", errors.New(charset)) if !charOK {
web.SiteInfo.Issue = "MySQL character set not utf8/utf8mb4: " + charset msg := fmt.Sprintf("*** ERROR: %s ***", charRequired)
runtime.Log.Info(msg)
web.SiteInfo.Issue = msg
runtime.Flags.SiteMode = env.SiteModeBadDB runtime.Flags.SiteMode = env.SiteModeBadDB
return false return false
} }
if !strings.HasPrefix(collation, "utf8") {
runtime.Log.Error("MySQL collation sequence not utf8...:", errors.New(collation))
web.SiteInfo.Issue = "MySQL collation sequence not utf8...: " + collation
runtime.Flags.SiteMode = env.SiteModeBadDB
return false
}
}
{ // if there are no rows in the database, enter set-up mode { // if there are no rows in the database, enter set-up mode
var flds []string var flds []string
if err := runtime.Db.Select(&flds, if err := runtime.Db.Select(&flds, runtime.StoreProvider.QueryTableList()); err != nil {
`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = '`+web.SiteInfo.DBname+ msg := fmt.Sprintf("Database: unable to get database table list ")
`' and TABLE_TYPE='BASE TABLE'`); err != nil { runtime.Log.Error(msg, err)
runtime.Log.Error("Can't get MySQL number of tables", err) web.SiteInfo.Issue = msg + err.Error()
web.SiteInfo.Issue = "Can't get MySQL number of tables: " + err.Error()
runtime.Flags.SiteMode = env.SiteModeBadDB runtime.Flags.SiteMode = env.SiteModeBadDB
return false return false
} }
if strings.TrimSpace(flds[0]) == "0" { if strings.TrimSpace(flds[0]) == "0" {
runtime.Log.Info("Entering database set-up mode because the database is empty.....") runtime.Log.Info("Database: starting setup mode for empty database")
runtime.Flags.SiteMode = env.SiteModeSetup runtime.Flags.SiteMode = env.SiteModeSetup
return false return false
} }
} }
{ // check all the required tables exist // Ensure no missing tables.
var tables = []string{`account`, var tables = []string{"account", "attachment", "document",
`attachment`, `document`, "label", "organization", "page", "revision", "search", "user"}
`label`, `organization`,
`page`, `revision`, `search`, `user`}
for _, table := range tables { for _, table := range tables {
var dummy []string var result []string
if err := runtime.Db.Select(&dummy, "SELECT 1 FROM "+table+" LIMIT 1;"); err != nil { if err := runtime.Db.Select(&result, fmt.Sprintf("SELECT COUNT(*) FROM %s ;", table)); err != nil {
runtime.Log.Error("Entering bad database mode because: SELECT 1 FROM "+table+" LIMIT 1;", err) msg := fmt.Sprintf("Database: missing table %s", table)
web.SiteInfo.Issue = "MySQL database is not empty, but does not contain table: " + table runtime.Log.Error(msg, err)
web.SiteInfo.Issue = msg
runtime.Flags.SiteMode = env.SiteModeBadDB runtime.Flags.SiteMode = env.SiteModeBadDB
return false return false
} }
} }
}
runtime.Flags.SiteMode = env.SiteModeNormal // actually no need to do this (as already ""), this for documentation // We have good database, so proceed with app boot process.
web.SiteInfo.DBname = "" // do not give this info when not in set-up mode runtime.Flags.SiteMode = env.SiteModeNormal
dbCheckOK = true web.SiteInfo.DBname = ""
return true return true
} }
// GetSQLVersion returns SQL version as major,minor,patch numerics.
func GetSQLVersion(v string) (ints []int, err error) {
ints = []int{0, 0, 0}
pos := strings.Index(v, "-")
if pos > 1 {
v = v[:pos]
}
vs := strings.Split(v, ".")
if len(vs) < 3 {
err = errors.New("MySQL version not of the form a.b.c")
return
}
for key, val := range vs {
num, err := strconv.Atoi(val)
if err != nil {
return ints, err
}
ints[key] = num
}
return
}

45
core/database/installer.go
View file

@ -14,6 +14,7 @@ package database
import ( import (
"fmt" "fmt"
"regexp" "regexp"
"strconv"
"strings" "strings"
"time" "time"
@ -35,11 +36,11 @@ func InstallUpgrade(runtime *env.Runtime, existingDB bool) (err error) {
// Filter out database specific scripts. // Filter out database specific scripts.
dbTypeScripts := SpecificScripts(runtime, scripts) dbTypeScripts := SpecificScripts(runtime, scripts)
if len(dbTypeScripts) == 0 { if len(dbTypeScripts) == 0 {
runtime.Log.Info(fmt.Sprintf("Database: unable to load scripts for database type %s", runtime.Storage.Type)) runtime.Log.Info(fmt.Sprintf("Database: unable to load scripts for database type %s", runtime.StoreProvider.Type()))
return return
} }
runtime.Log.Info(fmt.Sprintf("Database: loaded %d SQL scripts for provider %s", len(dbTypeScripts), runtime.Storage.Type)) runtime.Log.Info(fmt.Sprintf("Database: loaded %d SQL scripts for provider %s", len(dbTypeScripts), runtime.StoreProvider.Type()))
// Get current database version. // Get current database version.
currentVersion := 0 currentVersion := 0
@ -114,13 +115,13 @@ func runScripts(runtime *env.Runtime, tx *sqlx.Tx, scripts []Script) (err error)
for _, script := range scripts { for _, script := range scripts {
runtime.Log.Info(fmt.Sprintf("Databasse: processing SQL version %d", script.Version)) runtime.Log.Info(fmt.Sprintf("Databasse: processing SQL version %d", script.Version))
err = executeSQL(tx, runtime.Storage.Type, script.Script) err = executeSQL(tx, runtime.StoreProvider.Type(), script.Script)
if err != nil { if err != nil {
return err return err
} }
// Record the fact we have processed this database script version. // Record the fact we have processed this database script version.
_, err = tx.Exec(recordVersionUpgradeQuery(runtime.Storage.Type, script.Version)) _, err = tx.Exec(runtime.StoreProvider.QueryRecordVersionUpgrade(script.Version))
if err != nil { if err != nil {
return err return err
} }
@ -170,3 +171,39 @@ func getStatements(bytes []byte) (stmts []string) {
return return
} }
// CurrentVersion returns number that represents the current database version number.
// For example 23 represents the 23rd iteration of the database.
func CurrentVersion(runtime *env.Runtime) (version int, err error) {
row := runtime.Db.QueryRow(runtime.StoreProvider.QueryGetDatabaseVersion())
var currentVersion string
err = row.Scan(&currentVersion)
if err != nil {
currentVersion = "0"
}
return extractVersionNumber(currentVersion), nil
}
// Turns legacy "db_00021.sql" and new "21" format into version number 21.
func extractVersionNumber(s string) int {
// Good practice in case of human tampering.
s = strings.TrimSpace(s)
s = strings.ToLower(s)
// Remove any quotes from JSON string.
s = strings.Replace(s, "\"", "", -1)
// Remove legacy version string formatting.
// We know just store the number.
s = strings.Replace(s, "db_000", "", 1)
s = strings.Replace(s, ".sql", "", 1)
i, err := strconv.Atoi(s)
if err != nil {
i = 0
}
return i
}

224
core/database/leader.go
View file

@ -1,224 +0,0 @@
// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
//
// This software (Documize Community Edition) is licensed under
// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
//
// You can operate outside the AGPL restrictions by purchasing
// Documize Enterprise Edition and obtaining a commercial license
// by contacting <sales@documize.com>.
//
// https://documize.com
package database
import (
"crypto/rand"
"fmt"
"os"
"strconv"
"strings"
"time"
"github.com/documize/community/core/env"
"github.com/jmoiron/sqlx"
)
// Lock will try to lock the database instance to the running process.
// Uses a "random" delay as a por man's database cluster-aware process.
// We skip delay if there are no scripts to process.
func Lock(runtime *env.Runtime, scriptsToProcess int) (bool, error) {
// Wait for random period of time.
b := make([]byte, 2)
_, err := rand.Read(b)
if err != nil {
return false, err
}
wait := ((time.Duration(b[0]) << 8) | time.Duration(b[1])) * time.Millisecond / 10 // up to 6.5 secs wait
// Why delay if nothing to process?
if scriptsToProcess > 0 {
time.Sleep(wait)
}
// Start transaction fotr lock process.
tx, err := runtime.Db.Beginx()
if err != nil {
runtime.Log.Error("Database: unable to start transaction", err)
return false, err
}
// Lock the database.
_, err = tx.Exec(processLockStartQuery(runtime.Storage.Type))
if err != nil {
runtime.Log.Error("Database: unable to lock tables", err)
return false, err
}
// Unlock the database at the end of this function.
defer func() {
_, err = tx.Exec(processLockFinishQuery(runtime.Storage.Type))
if err != nil {
runtime.Log.Error("Database: unable to unlock tables", err)
}
tx.Commit()
}()
// Try to record this process as leader of database migration process.
_, err = tx.Exec(insertProcessIDQuery(runtime.Storage.Type))
if err != nil {
runtime.Log.Info("Database: marked as slave process awaiting upgrade")
return false, nil
}
// We are the leader!
runtime.Log.Info("Database: marked as database upgrade process leader")
return true, err
}
// Unlock completes process that was started with Lock().
func Unlock(runtime *env.Runtime, tx *sqlx.Tx, err error, amLeader bool) error {
if amLeader {
defer func() {
doUnlock(runtime)
}()
if tx != nil {
if err == nil {
tx.Commit()
runtime.Log.Info("Database: is ready")
return nil
}
tx.Rollback()
}
runtime.Log.Error("Database: install/upgrade failed", err)
return err
}
return nil // not the leader, so ignore errors
}
// CurrentVersion returns number that represents the current database version number.
// For example 23 represents the 23rd iteration of the database.
func CurrentVersion(runtime *env.Runtime) (version int, err error) {
row := runtime.Db.QueryRow(databaseVersionQuery(runtime.Storage.Type))
var currentVersion string
err = row.Scan(&currentVersion)
if err != nil {
currentVersion = "0"
}
return extractVersionNumber(currentVersion), nil
}
// Helper method for defer function called from Unlock().
func doUnlock(runtime *env.Runtime) error {
tx, err := runtime.Db.Beginx()
if err != nil {
return err
}
_, err = tx.Exec(deleteProcessIDQuery(runtime.Storage.Type))
if err != nil {
return err
}
return tx.Commit()
}
// processLockStartQuery returns database specific query that will
// LOCK the database to this running process.
func processLockStartQuery(t env.StoreType) string {
switch t {
case env.StoreTypeMySQL, env.StoreTypeMariaDB, env.StoreTypePercona:
return "LOCK TABLE `config` WRITE;"
case env.StoreTypePostgreSQL:
return ""
case env.StoreTypeMSSQL:
return ""
}
return ""
}
// processLockFinishQuery returns database specific query that will
// UNLOCK the database from this running process.
func processLockFinishQuery(t env.StoreType) string {
switch t {
case env.StoreTypeMySQL, env.StoreTypeMariaDB, env.StoreTypePercona:
return "UNLOCK TABLES;"
case env.StoreTypePostgreSQL:
return ""
case env.StoreTypeMSSQL:
return ""
}
return ""
}
// insertProcessIDQuery returns database specific query that will
// insert ID of this running process.
func insertProcessIDQuery(t env.StoreType) string {
return "INSERT INTO `config` (`key`,`config`) " + fmt.Sprintf(`VALUES ('DBLOCK','{"pid": "%d"}');`, os.Getpid())
}
// deleteProcessIDQuery returns database specific query that will
// delete ID of this running process.
func deleteProcessIDQuery(t env.StoreType) string {
return "DELETE FROM `config` WHERE `key`='DBLOCK';"
}
// recordVersionUpgradeQuery returns database specific insert statement
// that records the database version number
func recordVersionUpgradeQuery(t env.StoreType, version int) string {
// Make record that holds new database version number.
json := fmt.Sprintf("{\"database\": \"%d\"}", version)
switch t {
case env.StoreTypeMySQL, env.StoreTypeMariaDB, env.StoreTypePercona:
return "INSERT INTO `config` (`key`,`config`) " + "VALUES ('META','" + json + "') ON DUPLICATE KEY UPDATE `config`='" + json + "';"
case env.StoreTypePostgreSQL:
return ""
case env.StoreTypeMSSQL:
return ""
}
return ""
}
// databaseVersionQuery returns the schema version number.
func databaseVersionQuery(t env.StoreType) string {
switch t {
case env.StoreTypeMySQL, env.StoreTypeMariaDB, env.StoreTypePercona:
return "SELECT JSON_EXTRACT(`config`,'$.database') FROM `config` WHERE `key` = 'META';"
case env.StoreTypePostgreSQL:
return ""
case env.StoreTypeMSSQL:
return ""
}
return ""
}
// Turns legacy "db_00021.sql" and new "21" format into version number 21.
func extractVersionNumber(s string) int {
// Good practice in case of human tampering.
s = strings.TrimSpace(s)
s = strings.ToLower(s)
// Remove any quotes from JSON string.
s = strings.Replace(s, "\"", "", -1)
// Remove legacy version string formatting.
// We know just store the number.
s = strings.Replace(s, "db_000", "", 1)
s = strings.Replace(s, ".sql", "", 1)
i, err := strconv.Atoi(s)
if err != nil {
i = 0
}
return i
}

110
core/database/lock.go Normal file
View file

@ -0,0 +1,110 @@
// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
//
// This software (Documize Community Edition) is licensed under
// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
//
// You can operate outside the AGPL restrictions by purchasing
// Documize Enterprise Edition and obtaining a commercial license
// by contacting <sales@documize.com>.
//
// https://documize.com
package database
import (
"crypto/rand"
"time"
"github.com/documize/community/core/env"
"github.com/jmoiron/sqlx"
)
// Lock will try to lock the database instance to the running process.
// Uses a "random" delay as a por man's database cluster-aware process.
// We skip delay if there are no scripts to process.
func Lock(runtime *env.Runtime, scriptsToProcess int) (bool, error) {
// Wait for random period of time.
b := make([]byte, 2)
_, err := rand.Read(b)
if err != nil {
return false, err
}
wait := ((time.Duration(b[0]) << 8) | time.Duration(b[1])) * time.Millisecond / 10 // up to 6.5 secs wait
// Why delay if nothing to process?
if scriptsToProcess > 0 {
time.Sleep(wait)
}
// Start transaction fotr lock process.
tx, err := runtime.Db.Beginx()
if err != nil {
runtime.Log.Error("Database: unable to start transaction", err)
return false, err
}
// Lock the database.
_, err = tx.Exec(runtime.StoreProvider.QueryStartLock())
if err != nil {
runtime.Log.Error("Database: unable to lock tables", err)
return false, err
}
// Unlock the database at the end of this function.
defer func() {
_, err = tx.Exec(runtime.StoreProvider.QueryFinishLock())
if err != nil {
runtime.Log.Error("Database: unable to unlock tables", err)
}
tx.Commit()
}()
// Try to record this process as leader of database migration process.
_, err = tx.Exec(runtime.StoreProvider.QueryInsertProcessID())
if err != nil {
runtime.Log.Info("Database: marked as slave process awaiting upgrade")
return false, nil
}
// We are the leader!
runtime.Log.Info("Database: marked as database upgrade process leader")
return true, err
}
// Unlock completes process that was started with Lock().
func Unlock(runtime *env.Runtime, tx *sqlx.Tx, err error, amLeader bool) error {
if amLeader {
defer func() {
doUnlock(runtime)
}()
if tx != nil {
if err == nil {
tx.Commit()
runtime.Log.Info("Database: is ready")
return nil
}
tx.Rollback()
}
runtime.Log.Error("Database: install/upgrade failed", err)
return err
}
return nil // not the leader, so ignore errors
}
// Helper method for defer function called from Unlock().
func doUnlock(runtime *env.Runtime) error {
tx, err := runtime.Db.Beginx()
if err != nil {
return err
}
_, err = tx.Exec(runtime.StoreProvider.QueryDeleteProcessID())
if err != nil {
return err
}
return tx.Commit()
}

4
core/database/scripts.go
View file

@ -13,9 +13,9 @@ package database
import ( import (
"fmt" "fmt"
"github.com/documize/community/core/env"
"sort" "sort"
"github.com/documize/community/core/env"
"github.com/documize/community/server/web" "github.com/documize/community/server/web"
) )
@ -47,7 +47,7 @@ func LoadScripts() (s Scripts, err error) {
// SpecificScripts returns SQL scripts for current databasse provider. // SpecificScripts returns SQL scripts for current databasse provider.
func SpecificScripts(runtime *env.Runtime, all Scripts) (s []Script) { func SpecificScripts(runtime *env.Runtime, all Scripts) (s []Script) {
switch runtime.Storage.Type { switch runtime.StoreProvider.Type() {
case env.StoreTypeMySQL, env.StoreTypeMariaDB, env.StoreTypePercona: case env.StoreTypeMySQL, env.StoreTypeMariaDB, env.StoreTypePercona:
return all.MySQLScripts return all.MySQLScripts
case env.StoreTypePostgreSQL: case env.StoreTypePostgreSQL:

95
core/env/runtime.go vendored
View file

@ -14,7 +14,6 @@ package env
import ( import (
"github.com/jmoiron/sqlx" "github.com/jmoiron/sqlx"
"strings"
) )
// SQL-STORE: DbVariant needs to be struct like: name, delims, std params and conn string method // SQL-STORE: DbVariant needs to be struct like: name, delims, std params and conn string method
@ -24,7 +23,7 @@ import (
type Runtime struct { type Runtime struct {
Flags Flags Flags Flags
Db *sqlx.DB Db *sqlx.DB
Storage StoreProvider StoreProvider StoreProvider
Log Logger Log Logger
Product ProdInfo Product ProdInfo
} }
@ -63,69 +62,61 @@ const (
StoreTypeMSSQL StoreType = "MSSQL" StoreTypeMSSQL StoreType = "MSSQL"
) )
// StoreProvider contains database specific details // StoreProvider defines a database provider.
type StoreProvider struct { type StoreProvider interface {
// Type identifies storage provider // Name of provider
Type StoreType Type() StoreType
// SQL driver name used to open DB connection. // SQL driver name used to open DB connection.
DriverName string DriverName() string
// Database connection string parameters that must be present before connecting to DB // Database connection string parameters that must be present before connecting to DB.
Params map[string]string Params() map[string]string
// Example holds storage provider specific connection string format // Example holds storage provider specific connection string format.
// used in error messages // used in error messages
Example string Example() string
}
// ConnectionString returns provider specific DB connection string // DatabaseName holds the SQL database name where Documize tables live.
// complete with default parameters. DatabaseName() string
func (s *StoreProvider) ConnectionString(cs string) string {
switch s.Type {
case StoreTypePostgreSQL: // Make connection string with default parameters.
return "pg" MakeConnectionString() string
case StoreTypeMSSQL: // QueryMeta is how to extract version number, collation, character set from database provider.
return "sql server" QueryMeta() string
case StoreTypeMySQL, StoreTypeMariaDB, StoreTypePercona: // QueryStartLock locks database tables.
queryBits := strings.Split(cs, "?") QueryStartLock() string
ret := queryBits[0] + "?"
retFirst := true
if len(queryBits) == 2 { // QueryFinishLock unlocks database tables.
paramBits := strings.Split(queryBits[1], "&") QueryFinishLock() string
for _, pb := range paramBits {
found := false
if assignBits := strings.Split(pb, "="); len(assignBits) == 2 {
_, found = s.Params[strings.TrimSpace(assignBits[0])]
}
if !found { // if we can't work out what it is, put it through
if retFirst {
retFirst = false
} else {
ret += "&"
}
ret += pb
}
}
}
for k, v := range s.Params { // QueryInsertProcessID returns database specific query that will
if retFirst { // insert ID of this running process.
retFirst = false QueryInsertProcessID() string
} else {
ret += "&"
}
ret += k + "=" + v
}
return ret // QueryInsertProcessID returns database specific query that will
} // delete ID of this running process.
QueryDeleteProcessID() string
return "" // QueryRecordVersionUpgrade returns database specific insert statement
// that records the database version number.
QueryRecordVersionUpgrade(version int) string
// QueryGetDatabaseVersion returns the schema version number.
QueryGetDatabaseVersion() string
// QueryTableList returns a list tables in Documize database.
QueryTableList() string
// VerfiyVersion checks to see if actual database meets
// minimum version requirements.
VerfiyVersion(dbVersion string) (versionOK bool, minVerRequired string)
// VerfiyCharacterCollation checks to see if actual database
// has correct character set and collation settings.
VerfiyCharacterCollation(charset, collation string) (charOK bool, requirements string)
} }
const ( const (

19
edition/boot/runtime.go
View file

@ -47,41 +47,42 @@ func InitRuntime(r *env.Runtime, s *domain.Store) bool {
} }
} }
// Work out required storage provider set it up. // Set up required storage provider.
switch r.Flags.DBType { switch r.Flags.DBType {
case "mysql": case "mysql":
r.Storage = env.StoreProvider{Type: env.StoreTypeMySQL, DriverName: "mysql"}
storage.SetMySQLProvider(r, s) storage.SetMySQLProvider(r, s)
case "mariadb": case "mariadb":
r.Storage = env.StoreProvider{Type: env.StoreTypeMariaDB, DriverName: "mysql"}
storage.SetMySQLProvider(r, s) storage.SetMySQLProvider(r, s)
case "percona": case "percona":
r.Storage = env.StoreProvider{Type: env.StoreTypePercona, DriverName: "mysql"}
storage.SetMySQLProvider(r, s) storage.SetMySQLProvider(r, s)
case "pggg": case "pggg":
r.Storage = env.StoreProvider{Type: env.StoreTypePercona, DriverName: "pgggggg"}
// storage.SetPostgresSQLProvider(r, s) // storage.SetPostgresSQLProvider(r, s)
case "mssql": case "mssql":
r.Storage = env.StoreProvider{Type: env.StoreTypePercona, DriverName: "sqlserver"}
// storage.SetSQLServerProvider(r, s) // storage.SetSQLServerProvider(r, s)
} }
// Open connection to database // Open connection to database
db, err := sqlx.Open(r.Storage.DriverName, r.Storage.ConnectionString(r.Flags.DBConn)) db, err := sqlx.Open(r.StoreProvider.DriverName(), r.StoreProvider.MakeConnectionString()) //r.Flags.DBConn
if err != nil { if err != nil {
r.Log.Error("unable to setup database", err) r.Log.Error("unable to setup database", err)
} }
// Database handle
r.Db = db r.Db = db
// Database connection defaults
r.Db.SetMaxIdleConns(30) r.Db.SetMaxIdleConns(30)
r.Db.SetMaxOpenConns(100) r.Db.SetMaxOpenConns(100)
r.Db.SetConnMaxLifetime(time.Second * 14400) r.Db.SetConnMaxLifetime(time.Second * 14400)
// Database good?
err = r.Db.Ping() err = r.Db.Ping()
if err != nil { if err != nil {
r.Log.Error("unable to connect to database - "+r.Storage.Example, err) r.Log.Error("unable to connect to database - "+r.StoreProvider.Example(), err)
return false return false
} }
// Go into setup mode if required. // Check database and upgrade if required.
if r.Flags.SiteMode != env.SiteModeOffline { if r.Flags.SiteMode != env.SiteModeOffline {
if database.Check(r) { if database.Check(r) {
if err := database.InstallUpgrade(r, true); err != nil { if err := database.InstallUpgrade(r, true); err != nil {

267
edition/storage/mysql.go Normal file
View file

@ -0,0 +1,267 @@
// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
//
// This software (Documize Community Edition) is licensed under
// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
//
// You can operate outside the AGPL restrictions by purchasing
// Documize Enterprise Edition and obtaining a commercial license
// by contacting <sales@documize.com>.
//
// https://documize.com
// Package storage sets up database persistence providers.
package storage
import (
"errors"
"fmt"
"os"
"strconv"
"strings"
"github.com/documize/community/core/env"
"github.com/documize/community/domain"
account "github.com/documize/community/domain/account/mysql"
activity "github.com/documize/community/domain/activity/mysql"
attachment "github.com/documize/community/domain/attachment/mysql"
audit "github.com/documize/community/domain/audit/mysql"
block "github.com/documize/community/domain/block/mysql"
category "github.com/documize/community/domain/category/mysql"
doc "github.com/documize/community/domain/document/mysql"
group "github.com/documize/community/domain/group/mysql"
link "github.com/documize/community/domain/link/mysql"
meta "github.com/documize/community/domain/meta/mysql"
org "github.com/documize/community/domain/organization/mysql"
page "github.com/documize/community/domain/page/mysql"
permission "github.com/documize/community/domain/permission/mysql"
pin "github.com/documize/community/domain/pin/mysql"
search "github.com/documize/community/domain/search/mysql"
setting "github.com/documize/community/domain/setting/mysql"
space "github.com/documize/community/domain/space/mysql"
user "github.com/documize/community/domain/user/mysql"
)
// SetMySQLProvider creates MySQL provider
func SetMySQLProvider(r *env.Runtime, s *domain.Store) {
// Set up provider specific details.
r.StoreProvider = MySQLProvider{
ConnectionString: r.Flags.DBConn,
Variant: r.Flags.DBType,
}
// Wire up data providers!
s.Account = account.Scope{Runtime: r}
s.Activity = activity.Scope{Runtime: r}
s.Attachment = attachment.Scope{Runtime: r}
s.Audit = audit.Scope{Runtime: r}
s.Block = block.Scope{Runtime: r}
s.Category = category.Scope{Runtime: r}
s.Document = doc.Scope{Runtime: r}
s.Group = group.Scope{Runtime: r}
s.Link = link.Scope{Runtime: r}
s.Meta = meta.Scope{Runtime: r}
s.Organization = org.Scope{Runtime: r}
s.Page = page.Scope{Runtime: r}
s.Pin = pin.Scope{Runtime: r}
s.Permission = permission.Scope{Runtime: r}
s.Search = search.Scope{Runtime: r}
s.Setting = setting.Scope{Runtime: r}
s.Space = space.Scope{Runtime: r}
s.User = user.Scope{Runtime: r}
}
// MySQLProvider supports MySQL 5.7.x and 8.0.x versions.
type MySQLProvider struct {
// User specified connection string.
ConnectionString string
// User specified db type (mysql, percona or mariadb).
Variant string
}
// Type returns name of provider
func (p MySQLProvider) Type() env.StoreType {
return env.StoreTypeMySQL
}
// DriverName returns database/sql driver name.
func (p MySQLProvider) DriverName() string {
return "mysql"
}
// Params returns connection string parameters that must be present before connecting to DB.
func (p MySQLProvider) Params() map[string]string {
return map[string]string{
"charset": "utf8mb4",
"parseTime": "True",
"maxAllowedPacket": "104857600", // 4194304 // 16777216 = 16MB // 104857600 = 100MB
}
}
// Example holds storage provider specific connection string format.
// used in error messages
func (p MySQLProvider) Example() string {
return "database connection string format is 'username:password@tcp(host:3306)/database'"
}
// DatabaseName holds the SQL database name where Documize tables live.
func (p MySQLProvider) DatabaseName() string {
bits := strings.Split(p.ConnectionString, "/")
if len(bits) > 1 {
return strings.Split(bits[len(bits)-1], "?")[0]
}
return ""
}
// MakeConnectionString returns provider specific DB connection string
// complete with default parameters.
func (p MySQLProvider) MakeConnectionString() string {
queryBits := strings.Split(p.ConnectionString, "?")
ret := queryBits[0] + "?"
retFirst := true
params := p.Params()
if len(queryBits) == 2 {
paramBits := strings.Split(queryBits[1], "&")
for _, pb := range paramBits {
found := false
if assignBits := strings.Split(pb, "="); len(assignBits) == 2 {
_, found = params[strings.TrimSpace(assignBits[0])]
}
if !found { // if we can't work out what it is, put it through
if retFirst {
retFirst = false
} else {
ret += "&"
}
ret += pb
}
}
}
for k, v := range params {
if retFirst {
retFirst = false
} else {
ret += "&"
}
ret += k + "=" + v
}
return ret
}
// QueryMeta is how to extract version number, collation, character set from database provider.
func (p MySQLProvider) QueryMeta() string {
return "SELECT VERSION() AS version, @@version_comment as comment, @@character_set_database AS charset, @@collation_database AS collation"
}
// QueryStartLock locks database tables.
func (p MySQLProvider) QueryStartLock() string {
return "LOCK TABLE `config` WRITE;"
}
// QueryFinishLock unlocks database tables.
func (p MySQLProvider) QueryFinishLock() string {
return "UNLOCK TABLES;"
}
// QueryInsertProcessID returns database specific query that will
// insert ID of this running process.
func (p MySQLProvider) QueryInsertProcessID() string {
return "INSERT INTO `config` (`key`,`config`) " + fmt.Sprintf(`VALUES ('DBLOCK','{"pid": "%d"}');`, os.Getpid())
}
// QueryDeleteProcessID returns database specific query that will
// delete ID of this running process.
func (p MySQLProvider) QueryDeleteProcessID() string {
return "DELETE FROM `config` WHERE `key`='DBLOCK';"
}
// QueryRecordVersionUpgrade returns database specific insert statement
// that records the database version number.
func (p MySQLProvider) QueryRecordVersionUpgrade(version int) string {
// Make record that holds new database version number.
json := fmt.Sprintf("{\"database\": \"%d\"}", version)
return "INSERT INTO `config` (`key`,`config`) " + "VALUES ('META','" + json + "') ON DUPLICATE KEY UPDATE `config`='" + json + "';"
}
// QueryGetDatabaseVersion returns the schema version number.
func (p MySQLProvider) QueryGetDatabaseVersion() string {
return "SELECT JSON_EXTRACT(`config`,'$.database') FROM `config` WHERE `key` = 'META';"
}
// QueryTableList returns a list tables in Documize database.
func (p MySQLProvider) QueryTableList() string {
return `SELECT COUNT(*) FROM information_schema.tables
WHERE table_schema = '` + p.DatabaseName() + `' and TABLE_TYPE='BASE TABLE'`
}
// VerfiyVersion checks to see if actual database meets
// minimum version requirements.
func (p MySQLProvider) VerfiyVersion(dbVersion string) (bool, string) {
// Minimum MySQL / MariaDB version.
minVer := []int{5, 7, 10}
if p.Variant == "mariadb" {
minVer = []int{10, 3, 0}
}
// Convert string to semver.
dbSemver, _ := convertDatabaseVersion(dbVersion)
for k, v := range minVer {
// If major release is higher then skip minor/patch checks (e.g. 8.x.x > 5.x.x)
if k == 0 && len(dbSemver) > 0 && dbSemver[0] > minVer[0] {
break
}
if dbSemver[k] < v {
want := fmt.Sprintf("%d.%d.%d", minVer[0], minVer[1], minVer[2])
return false, want
}
}
return true, ""
}
// VerfiyCharacterCollation needs to ensure utf8/utf8mb4.
func (p MySQLProvider) VerfiyCharacterCollation(charset, collation string) (charOK bool, requirements string) {
if charset != "utf8" && charset != "utf8mb4" {
return false, "MySQL character needs to be utf8/utf8mb4"
}
if !strings.HasPrefix(collation, "utf8") {
return false, "MySQL collation sequence needs to be utf8"
}
return true, ""
}
// convertDatabaseVersion turns database version string as major,minor,patch numerics.
func convertDatabaseVersion(v string) (ints []int, err error) {
ints = []int{0, 0, 0}
pos := strings.Index(v, "-")
if pos > 1 {
v = v[:pos]
}
vs := strings.Split(v, ".")
if len(vs) < 3 {
err = errors.New("MySQL version not of the form a.b.c")
return
}
for key, val := range vs {
num, err := strconv.Atoi(val)
if err != nil {
return ints, err
}
ints[key] = num
}
return
}

67
edition/storage/mysql_provider.go
View file

@ -1,67 +0,0 @@
// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
//
// This software (Documize Community Edition) is licensed under
// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
//
// You can operate outside the AGPL restrictions by purchasing
// Documize Enterprise Edition and obtaining a commercial license
// by contacting <sales@documize.com>.
//
// https://documize.com
// Package storage sets up database persistence providers.
package storage
import (
"github.com/documize/community/core/env"
"github.com/documize/community/domain"
account "github.com/documize/community/domain/account/mysql"
activity "github.com/documize/community/domain/activity/mysql"
attachment "github.com/documize/community/domain/attachment/mysql"
audit "github.com/documize/community/domain/audit/mysql"
block "github.com/documize/community/domain/block/mysql"
category "github.com/documize/community/domain/category/mysql"
doc "github.com/documize/community/domain/document/mysql"
group "github.com/documize/community/domain/group/mysql"
link "github.com/documize/community/domain/link/mysql"
meta "github.com/documize/community/domain/meta/mysql"
org "github.com/documize/community/domain/organization/mysql"
page "github.com/documize/community/domain/page/mysql"
permission "github.com/documize/community/domain/permission/mysql"
pin "github.com/documize/community/domain/pin/mysql"
search "github.com/documize/community/domain/search/mysql"
setting "github.com/documize/community/domain/setting/mysql"
space "github.com/documize/community/domain/space/mysql"
user "github.com/documize/community/domain/user/mysql"
)
// SetMySQLProvider creates MySQL provider
func SetMySQLProvider(r *env.Runtime, s *domain.Store) {
// Required connection string parameters and defaults.
r.Storage.Params = map[string]string{
"charset": "utf8mb4",
"parseTime": "True",
"maxAllowedPacket": "104857600", // 4194304 // 16777216 = 16MB // 104857600 = 100MB
}
r.Storage.Example = "database connection string format is 'username:password@tcp(host:3306)/database'"
s.Account = account.Scope{Runtime: r}
s.Activity = activity.Scope{Runtime: r}
s.Attachment = attachment.Scope{Runtime: r}
s.Audit = audit.Scope{Runtime: r}
s.Block = block.Scope{Runtime: r}
s.Category = category.Scope{Runtime: r}
s.Document = doc.Scope{Runtime: r}
s.Group = group.Scope{Runtime: r}
s.Link = link.Scope{Runtime: r}
s.Meta = meta.Scope{Runtime: r}
s.Organization = org.Scope{Runtime: r}
s.Page = page.Scope{Runtime: r}
s.Pin = pin.Scope{Runtime: r}
s.Permission = permission.Scope{Runtime: r}
s.Search = search.Scope{Runtime: r}
s.Setting = setting.Scope{Runtime: r}
s.Space = space.Scope{Runtime: r}
s.User = user.Scope{Runtime: r}
}