1
0
Fork 0
mirror of https://github.com/documize/community.git synced 2025-07-24 23:59:47 +02:00

Make database boot process storage provider agonistic

Moved database queries into provider specific object to ensure database checking, installation, upgrade procedures are pluggable.
This commit is contained in:
Harvey Kandola 2018-09-14 18:00:24 +01:00
parent 4c733ce581
commit 97d90662dd
9 changed files with 524 additions and 466 deletions

View file

@ -12,9 +12,7 @@
package database
import (
"errors"
"fmt"
"strconv"
"strings"
"github.com/documize/community/core/env"
@ -22,25 +20,21 @@ import (
"github.com/documize/community/server/web"
)
var dbCheckOK bool // default false
// Check that the database is configured correctly and that all the required tables exist.
// It must be the first function called in this package.
func Check(runtime *env.Runtime) bool {
runtime.Log.Info("Database checks: started")
runtime.Log.Info("Database: checking state")
csBits := strings.Split(runtime.Flags.DBConn, "/")
if len(csBits) > 1 {
web.SiteInfo.DBname = strings.Split(csBits[len(csBits)-1], "?")[0]
}
web.SiteInfo.DBname = runtime.StoreProvider.DatabaseName()
rows, err := runtime.Db.Query("SELECT VERSION() AS version, @@version_comment as comment, @@character_set_database AS charset, @@collation_database AS collation")
rows, err := runtime.Db.Query(runtime.StoreProvider.QueryMeta())
if err != nil {
runtime.Log.Error("Can't get MySQL configuration", err)
web.SiteInfo.Issue = "Can't get MySQL configuration: " + err.Error()
runtime.Log.Error("Database: unable to load meta information from database provider", err)
web.SiteInfo.Issue = "Unable to load meta information from database provider: " + err.Error()
runtime.Flags.SiteMode = env.SiteModeBadDB
return false
}
defer streamutil.Close(rows)
var version, dbComment, charset, collation string
if rows.Next() {
@ -50,121 +44,70 @@ func Check(runtime *env.Runtime) bool {
err = rows.Err() // get any error encountered during iteration
}
if err != nil {
runtime.Log.Error("no MySQL configuration returned", err)
web.SiteInfo.Issue = "no MySQL configuration return issue: " + err.Error()
runtime.Log.Error("Database: no meta data returned by database provider", err)
web.SiteInfo.Issue = "No meta data returned by database provider: " + err.Error()
runtime.Flags.SiteMode = env.SiteModeBadDB
return false
}
// runtime.DbVariant = GetSQLVariant(runtime.Flags.DBType, dbComment)
runtime.Log.Info(fmt.Sprintf("Database checks: SQL variant %v", runtime.Storage.Type))
runtime.Log.Info("Database checks: SQL version " + version)
runtime.Log.Info(fmt.Sprintf("Database: provider name %v", runtime.StoreProvider.Type()))
runtime.Log.Info(fmt.Sprintf("Database: provider version %s", version))
verNums, err := GetSQLVersion(version)
if err != nil {
runtime.Log.Error("Database version check failed", err)
// Version OK?
versionOK, minVersion := runtime.StoreProvider.VerfiyVersion(version)
if !versionOK {
msg := fmt.Sprintf("*** ERROR: database version needs to be %s or above ***", minVersion)
runtime.Log.Info(msg)
web.SiteInfo.Issue = msg
runtime.Flags.SiteMode = env.SiteModeBadDB
return false
}
// Check minimum MySQL version as we need JSON column type.
verInts := []int{5, 7, 10} // Minimum MySQL version
if runtime.Storage.Type == env.StoreTypeMariaDB {
verInts = []int{10, 3, 0} // Minimum MariaDB version
}
for k, v := range verInts {
// If major release is higher then skip minor/patch checks (e.g. 8.x.x > 5.x.x)
if k == 0 && len(verNums) > 0 && verNums[0] > verInts[0] {
break
}
if verNums[k] < v {
want := fmt.Sprintf("%d.%d.%d", verInts[0], verInts[1], verInts[2])
runtime.Log.Error("MySQL version element "+strconv.Itoa(k+1)+" of '"+version+"' not high enough, need at least version "+want, errors.New("bad MySQL version"))
web.SiteInfo.Issue = "MySQL version element " + strconv.Itoa(k+1) + " of '" + version + "' not high enough, need at least version " + want
runtime.Flags.SiteMode = env.SiteModeBadDB
return false
}
}
{ // check the MySQL character set and collation
if charset != "utf8" && charset != "utf8mb4" {
runtime.Log.Error("MySQL character set not utf8/utf8mb4:", errors.New(charset))
web.SiteInfo.Issue = "MySQL character set not utf8/utf8mb4: " + charset
runtime.Flags.SiteMode = env.SiteModeBadDB
return false
}
if !strings.HasPrefix(collation, "utf8") {
runtime.Log.Error("MySQL collation sequence not utf8...:", errors.New(collation))
web.SiteInfo.Issue = "MySQL collation sequence not utf8...: " + collation
runtime.Flags.SiteMode = env.SiteModeBadDB
return false
}
// Character set and collation OK?
charOK, charRequired := runtime.StoreProvider.VerfiyCharacterCollation(charset, collation)
if !charOK {
msg := fmt.Sprintf("*** ERROR: %s ***", charRequired)
runtime.Log.Info(msg)
web.SiteInfo.Issue = msg
runtime.Flags.SiteMode = env.SiteModeBadDB
return false
}
{ // if there are no rows in the database, enter set-up mode
var flds []string
if err := runtime.Db.Select(&flds,
`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = '`+web.SiteInfo.DBname+
`' and TABLE_TYPE='BASE TABLE'`); err != nil {
runtime.Log.Error("Can't get MySQL number of tables", err)
web.SiteInfo.Issue = "Can't get MySQL number of tables: " + err.Error()
if err := runtime.Db.Select(&flds, runtime.StoreProvider.QueryTableList()); err != nil {
msg := fmt.Sprintf("Database: unable to get database table list ")
runtime.Log.Error(msg, err)
web.SiteInfo.Issue = msg + err.Error()
runtime.Flags.SiteMode = env.SiteModeBadDB
return false
}
if strings.TrimSpace(flds[0]) == "0" {
runtime.Log.Info("Entering database set-up mode because the database is empty.....")
runtime.Log.Info("Database: starting setup mode for empty database")
runtime.Flags.SiteMode = env.SiteModeSetup
return false
}
}
{ // check all the required tables exist
var tables = []string{`account`,
`attachment`, `document`,
`label`, `organization`,
`page`, `revision`, `search`, `user`}
// Ensure no missing tables.
var tables = []string{"account", "attachment", "document",
"label", "organization", "page", "revision", "search", "user"}
for _, table := range tables {
var dummy []string
if err := runtime.Db.Select(&dummy, "SELECT 1 FROM "+table+" LIMIT 1;"); err != nil {
runtime.Log.Error("Entering bad database mode because: SELECT 1 FROM "+table+" LIMIT 1;", err)
web.SiteInfo.Issue = "MySQL database is not empty, but does not contain table: " + table
runtime.Flags.SiteMode = env.SiteModeBadDB
return false
}
for _, table := range tables {
var result []string
if err := runtime.Db.Select(&result, fmt.Sprintf("SELECT COUNT(*) FROM %s ;", table)); err != nil {
msg := fmt.Sprintf("Database: missing table %s", table)
runtime.Log.Error(msg, err)
web.SiteInfo.Issue = msg
runtime.Flags.SiteMode = env.SiteModeBadDB
return false
}
}
runtime.Flags.SiteMode = env.SiteModeNormal // actually no need to do this (as already ""), this for documentation
web.SiteInfo.DBname = "" // do not give this info when not in set-up mode
dbCheckOK = true
// We have good database, so proceed with app boot process.
runtime.Flags.SiteMode = env.SiteModeNormal
web.SiteInfo.DBname = ""
return true
}
// GetSQLVersion returns SQL version as major,minor,patch numerics.
func GetSQLVersion(v string) (ints []int, err error) {
ints = []int{0, 0, 0}
pos := strings.Index(v, "-")
if pos > 1 {
v = v[:pos]
}
vs := strings.Split(v, ".")
if len(vs) < 3 {
err = errors.New("MySQL version not of the form a.b.c")
return
}
for key, val := range vs {
num, err := strconv.Atoi(val)
if err != nil {
return ints, err
}
ints[key] = num
}
return
}

View file

@ -14,6 +14,7 @@ package database
import (
"fmt"
"regexp"
"strconv"
"strings"
"time"
@ -35,11 +36,11 @@ func InstallUpgrade(runtime *env.Runtime, existingDB bool) (err error) {
// Filter out database specific scripts.
dbTypeScripts := SpecificScripts(runtime, scripts)
if len(dbTypeScripts) == 0 {
runtime.Log.Info(fmt.Sprintf("Database: unable to load scripts for database type %s", runtime.Storage.Type))
runtime.Log.Info(fmt.Sprintf("Database: unable to load scripts for database type %s", runtime.StoreProvider.Type()))
return
}
runtime.Log.Info(fmt.Sprintf("Database: loaded %d SQL scripts for provider %s", len(dbTypeScripts), runtime.Storage.Type))
runtime.Log.Info(fmt.Sprintf("Database: loaded %d SQL scripts for provider %s", len(dbTypeScripts), runtime.StoreProvider.Type()))
// Get current database version.
currentVersion := 0
@ -114,13 +115,13 @@ func runScripts(runtime *env.Runtime, tx *sqlx.Tx, scripts []Script) (err error)
for _, script := range scripts {
runtime.Log.Info(fmt.Sprintf("Databasse: processing SQL version %d", script.Version))
err = executeSQL(tx, runtime.Storage.Type, script.Script)
err = executeSQL(tx, runtime.StoreProvider.Type(), script.Script)
if err != nil {
return err
}
// Record the fact we have processed this database script version.
_, err = tx.Exec(recordVersionUpgradeQuery(runtime.Storage.Type, script.Version))
_, err = tx.Exec(runtime.StoreProvider.QueryRecordVersionUpgrade(script.Version))
if err != nil {
return err
}
@ -170,3 +171,39 @@ func getStatements(bytes []byte) (stmts []string) {
return
}
// CurrentVersion returns number that represents the current database version number.
// For example 23 represents the 23rd iteration of the database.
func CurrentVersion(runtime *env.Runtime) (version int, err error) {
row := runtime.Db.QueryRow(runtime.StoreProvider.QueryGetDatabaseVersion())
var currentVersion string
err = row.Scan(&currentVersion)
if err != nil {
currentVersion = "0"
}
return extractVersionNumber(currentVersion), nil
}
// Turns legacy "db_00021.sql" and new "21" format into version number 21.
func extractVersionNumber(s string) int {
// Good practice in case of human tampering.
s = strings.TrimSpace(s)
s = strings.ToLower(s)
// Remove any quotes from JSON string.
s = strings.Replace(s, "\"", "", -1)
// Remove legacy version string formatting.
// We know just store the number.
s = strings.Replace(s, "db_000", "", 1)
s = strings.Replace(s, ".sql", "", 1)
i, err := strconv.Atoi(s)
if err != nil {
i = 0
}
return i
}

View file

@ -1,224 +0,0 @@
// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
//
// This software (Documize Community Edition) is licensed under
// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
//
// You can operate outside the AGPL restrictions by purchasing
// Documize Enterprise Edition and obtaining a commercial license
// by contacting <sales@documize.com>.
//
// https://documize.com
package database
import (
"crypto/rand"
"fmt"
"os"
"strconv"
"strings"
"time"
"github.com/documize/community/core/env"
"github.com/jmoiron/sqlx"
)
// Lock will try to lock the database instance to the running process.
// Uses a "random" delay as a por man's database cluster-aware process.
// We skip delay if there are no scripts to process.
func Lock(runtime *env.Runtime, scriptsToProcess int) (bool, error) {
// Wait for random period of time.
b := make([]byte, 2)
_, err := rand.Read(b)
if err != nil {
return false, err
}
wait := ((time.Duration(b[0]) << 8) | time.Duration(b[1])) * time.Millisecond / 10 // up to 6.5 secs wait
// Why delay if nothing to process?
if scriptsToProcess > 0 {
time.Sleep(wait)
}
// Start transaction fotr lock process.
tx, err := runtime.Db.Beginx()
if err != nil {
runtime.Log.Error("Database: unable to start transaction", err)
return false, err
}
// Lock the database.
_, err = tx.Exec(processLockStartQuery(runtime.Storage.Type))
if err != nil {
runtime.Log.Error("Database: unable to lock tables", err)
return false, err
}
// Unlock the database at the end of this function.
defer func() {
_, err = tx.Exec(processLockFinishQuery(runtime.Storage.Type))
if err != nil {
runtime.Log.Error("Database: unable to unlock tables", err)
}
tx.Commit()
}()
// Try to record this process as leader of database migration process.
_, err = tx.Exec(insertProcessIDQuery(runtime.Storage.Type))
if err != nil {
runtime.Log.Info("Database: marked as slave process awaiting upgrade")
return false, nil
}
// We are the leader!
runtime.Log.Info("Database: marked as database upgrade process leader")
return true, err
}
// Unlock completes process that was started with Lock().
func Unlock(runtime *env.Runtime, tx *sqlx.Tx, err error, amLeader bool) error {
if amLeader {
defer func() {
doUnlock(runtime)
}()
if tx != nil {
if err == nil {
tx.Commit()
runtime.Log.Info("Database: is ready")
return nil
}
tx.Rollback()
}
runtime.Log.Error("Database: install/upgrade failed", err)
return err
}
return nil // not the leader, so ignore errors
}
// CurrentVersion returns number that represents the current database version number.
// For example 23 represents the 23rd iteration of the database.
func CurrentVersion(runtime *env.Runtime) (version int, err error) {
row := runtime.Db.QueryRow(databaseVersionQuery(runtime.Storage.Type))
var currentVersion string
err = row.Scan(&currentVersion)
if err != nil {
currentVersion = "0"
}
return extractVersionNumber(currentVersion), nil
}
// Helper method for defer function called from Unlock().
func doUnlock(runtime *env.Runtime) error {
tx, err := runtime.Db.Beginx()
if err != nil {
return err
}
_, err = tx.Exec(deleteProcessIDQuery(runtime.Storage.Type))
if err != nil {
return err
}
return tx.Commit()
}
// processLockStartQuery returns database specific query that will
// LOCK the database to this running process.
func processLockStartQuery(t env.StoreType) string {
switch t {
case env.StoreTypeMySQL, env.StoreTypeMariaDB, env.StoreTypePercona:
return "LOCK TABLE `config` WRITE;"
case env.StoreTypePostgreSQL:
return ""
case env.StoreTypeMSSQL:
return ""
}
return ""
}
// processLockFinishQuery returns database specific query that will
// UNLOCK the database from this running process.
func processLockFinishQuery(t env.StoreType) string {
switch t {
case env.StoreTypeMySQL, env.StoreTypeMariaDB, env.StoreTypePercona:
return "UNLOCK TABLES;"
case env.StoreTypePostgreSQL:
return ""
case env.StoreTypeMSSQL:
return ""
}
return ""
}
// insertProcessIDQuery returns database specific query that will
// insert ID of this running process.
func insertProcessIDQuery(t env.StoreType) string {
return "INSERT INTO `config` (`key`,`config`) " + fmt.Sprintf(`VALUES ('DBLOCK','{"pid": "%d"}');`, os.Getpid())
}
// deleteProcessIDQuery returns database specific query that will
// delete ID of this running process.
func deleteProcessIDQuery(t env.StoreType) string {
return "DELETE FROM `config` WHERE `key`='DBLOCK';"
}
// recordVersionUpgradeQuery returns database specific insert statement
// that records the database version number
func recordVersionUpgradeQuery(t env.StoreType, version int) string {
// Make record that holds new database version number.
json := fmt.Sprintf("{\"database\": \"%d\"}", version)
switch t {
case env.StoreTypeMySQL, env.StoreTypeMariaDB, env.StoreTypePercona:
return "INSERT INTO `config` (`key`,`config`) " + "VALUES ('META','" + json + "') ON DUPLICATE KEY UPDATE `config`='" + json + "';"
case env.StoreTypePostgreSQL:
return ""
case env.StoreTypeMSSQL:
return ""
}
return ""
}
// databaseVersionQuery returns the schema version number.
func databaseVersionQuery(t env.StoreType) string {
switch t {
case env.StoreTypeMySQL, env.StoreTypeMariaDB, env.StoreTypePercona:
return "SELECT JSON_EXTRACT(`config`,'$.database') FROM `config` WHERE `key` = 'META';"
case env.StoreTypePostgreSQL:
return ""
case env.StoreTypeMSSQL:
return ""
}
return ""
}
// Turns legacy "db_00021.sql" and new "21" format into version number 21.
func extractVersionNumber(s string) int {
// Good practice in case of human tampering.
s = strings.TrimSpace(s)
s = strings.ToLower(s)
// Remove any quotes from JSON string.
s = strings.Replace(s, "\"", "", -1)
// Remove legacy version string formatting.
// We know just store the number.
s = strings.Replace(s, "db_000", "", 1)
s = strings.Replace(s, ".sql", "", 1)
i, err := strconv.Atoi(s)
if err != nil {
i = 0
}
return i
}

110
core/database/lock.go Normal file
View file

@ -0,0 +1,110 @@
// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
//
// This software (Documize Community Edition) is licensed under
// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
//
// You can operate outside the AGPL restrictions by purchasing
// Documize Enterprise Edition and obtaining a commercial license
// by contacting <sales@documize.com>.
//
// https://documize.com
package database
import (
"crypto/rand"
"time"
"github.com/documize/community/core/env"
"github.com/jmoiron/sqlx"
)
// Lock will try to lock the database instance to the running process.
// Uses a "random" delay as a por man's database cluster-aware process.
// We skip delay if there are no scripts to process.
func Lock(runtime *env.Runtime, scriptsToProcess int) (bool, error) {
// Wait for random period of time.
b := make([]byte, 2)
_, err := rand.Read(b)
if err != nil {
return false, err
}
wait := ((time.Duration(b[0]) << 8) | time.Duration(b[1])) * time.Millisecond / 10 // up to 6.5 secs wait
// Why delay if nothing to process?
if scriptsToProcess > 0 {
time.Sleep(wait)
}
// Start transaction fotr lock process.
tx, err := runtime.Db.Beginx()
if err != nil {
runtime.Log.Error("Database: unable to start transaction", err)
return false, err
}
// Lock the database.
_, err = tx.Exec(runtime.StoreProvider.QueryStartLock())
if err != nil {
runtime.Log.Error("Database: unable to lock tables", err)
return false, err
}
// Unlock the database at the end of this function.
defer func() {
_, err = tx.Exec(runtime.StoreProvider.QueryFinishLock())
if err != nil {
runtime.Log.Error("Database: unable to unlock tables", err)
}
tx.Commit()
}()
// Try to record this process as leader of database migration process.
_, err = tx.Exec(runtime.StoreProvider.QueryInsertProcessID())
if err != nil {
runtime.Log.Info("Database: marked as slave process awaiting upgrade")
return false, nil
}
// We are the leader!
runtime.Log.Info("Database: marked as database upgrade process leader")
return true, err
}
// Unlock completes process that was started with Lock().
func Unlock(runtime *env.Runtime, tx *sqlx.Tx, err error, amLeader bool) error {
if amLeader {
defer func() {
doUnlock(runtime)
}()
if tx != nil {
if err == nil {
tx.Commit()
runtime.Log.Info("Database: is ready")
return nil
}
tx.Rollback()
}
runtime.Log.Error("Database: install/upgrade failed", err)
return err
}
return nil // not the leader, so ignore errors
}
// Helper method for defer function called from Unlock().
func doUnlock(runtime *env.Runtime) error {
tx, err := runtime.Db.Beginx()
if err != nil {
return err
}
_, err = tx.Exec(runtime.StoreProvider.QueryDeleteProcessID())
if err != nil {
return err
}
return tx.Commit()
}

View file

@ -13,9 +13,9 @@ package database
import (
"fmt"
"github.com/documize/community/core/env"
"sort"
"github.com/documize/community/core/env"
"github.com/documize/community/server/web"
)
@ -47,7 +47,7 @@ func LoadScripts() (s Scripts, err error) {
// SpecificScripts returns SQL scripts for current databasse provider.
func SpecificScripts(runtime *env.Runtime, all Scripts) (s []Script) {
switch runtime.Storage.Type {
switch runtime.StoreProvider.Type() {
case env.StoreTypeMySQL, env.StoreTypeMariaDB, env.StoreTypePercona:
return all.MySQLScripts
case env.StoreTypePostgreSQL: