Docker/documize
Docker/documize
1
0
Fork 0
mirror of https://github.com/documize/community.git synced 2025-07-21 14:19:43 +02:00
Code Releases Activity

Refactoring of database init code

Browse source
This commit is contained in:
Harvey Kandola 2018-09-12 20:03:34 +01:00
parent d7fea2125f
commit 3bccd6a537
7 changed files with 155 additions and 134 deletions
Download patch file Download diff file Expand all files Collapse all files

55
core/database/check.go
View file

@ -46,23 +46,18 @@ func Check(runtime *env.Runtime) bool {
if rows.Next() {
err = rows.Scan(&version, &dbComment, &charset, &collation)
}
if err == nil {
err = rows.Err() // get any error encountered during iteration
}
if err != nil {
runtime.Log.Error("no MySQL configuration returned", err)
web.SiteInfo.Issue = "no MySQL configuration return issue: " + err.Error()
runtime.Flags.SiteMode = env.SiteModeBadDB
return false
}
// runtime.DbVariant = GetSQLVariant(runtime.Flags.DBType, dbComment)
// Get SQL variant as this affects minimum version checking logic.
// MySQL and Percona share same version scheme (e..g 5.7.10).
// MariaDB starts at 10.2.x
runtime.DbVariant = GetSQLVariant(runtime.Flags.DBType, dbComment)
runtime.Log.Info(fmt.Sprintf("Database checks: SQL variant %v", runtime.DbVariant))
runtime.Log.Info(fmt.Sprintf("Database checks: SQL variant %v", runtime.Storage.Type))
runtime.Log.Info("Database checks: SQL version " + version)
verNums, err := GetSQLVersion(version)
@ -72,7 +67,7 @@ func Check(runtime *env.Runtime) bool {
// Check minimum MySQL version as we need JSON column type.
verInts := []int{5, 7, 10} // Minimum MySQL version
if runtime.DbVariant == env.DBVariantMariaDB {
if runtime.Storage.Type == env.StoreTypeMariaDB {
verInts = []int{10, 3, 0} // Minimum MariaDB version
}
@ -146,31 +141,31 @@ func Check(runtime *env.Runtime) bool {
}
// GetSQLVariant uses database value form @@version_comment to deduce MySQL variant.
func GetSQLVariant(dbType, vc string) env.DbVariant {
vc = strings.ToLower(vc)
dbType = strings.ToLower(dbType)
// func GetSQLVariant(dbType, vc string) env.DbVariant {
// vc = strings.ToLower(vc)
// dbType = strings.ToLower(dbType)
// determine type from database
if strings.Contains(vc, "mariadb") {
return env.DBVariantMariaDB
} else if strings.Contains(vc, "percona") {
return env.DBVariantPercona
} else if strings.Contains(vc, "mysql") {
return env.DbVariantMySQL
}
// // determine type from database
// if strings.Contains(vc, "mariadb") {
// return env.DBVariantMariaDB
// } else if strings.Contains(vc, "percona") {
// return env.DBVariantPercona
// } else if strings.Contains(vc, "mysql") {
// return env.DbVariantMySQL
// }
// now determine type from command line switch
if strings.Contains(dbType, "mariadb") {
return env.DBVariantMariaDB
} else if strings.Contains(dbType, "percona") {
return env.DBVariantPercona
} else if strings.Contains(dbType, "mysql") {
return env.DbVariantMySQL
}
// // now determine type from command line switch
// if strings.Contains(dbType, "mariadb") {
// return env.DBVariantMariaDB
// } else if strings.Contains(dbType, "percona") {
// return env.DBVariantPercona
// } else if strings.Contains(dbType, "mysql") {
// return env.DbVariantMySQL
// }
// horrid default could cause app to crash
return env.DbVariantMySQL
}
// // horrid default could cause app to crash
// return env.DbVariantMySQL
// }
// GetSQLVersion returns SQL version as major,minor,patch numerics.
func GetSQLVersion(v string) (ints []int, err error) {

6
core/database/migrate.go
View file

@ -75,7 +75,7 @@ func (m migrationsT) migrate(runtime *env.Runtime, tx *sqlx.Tx) error {
return err
}
err = processSQLfile(tx, runtime.DbVariant, buf)
err = processSQLfile(tx, runtime.Storage.Type, buf)
if err != nil {
return err
}
@ -244,12 +244,12 @@ func Migrate(runtime *env.Runtime, ConfigTableExists bool) error {
return migrateEnd(runtime, tx, nil, amLeader)
}
func processSQLfile(tx *sqlx.Tx, v env.DbVariant, buf []byte) error {
func processSQLfile(tx *sqlx.Tx, v env.StoreType, buf []byte) error {
stmts := getStatements(buf)
for _, stmt := range stmts {
// MariaDB has no specific JSON column type (but has JSON queries)
if v == env.DBVariantMariaDB {
if v == env.StoreTypeMariaDB {
stmt = strings.Replace(stmt, "` JSON", "` TEXT", -1)
}

6
core/env/flags.go vendored
View file

@ -80,8 +80,8 @@ func ParseFlags() (f Flags) {
register(&port, "port", false, "http/https port number")
register(&forcePort2SSL, "forcesslport", false, "redirect given http port number to TLS")
register(&siteMode, "offline", false, "set to '1' for OFFLINE mode")
register(&dbType, "dbtype", false, "set to database type mysql|percona|mariadb")
register(&dbConn, "db", true, `'username:password@protocol(hostname:port)/databasename" for example "fred:bloggs@tcp(localhost:3306)/documize"`)
register(&dbType, "dbtype", true, "specify the database provider: mysql|percona|mariadb|postgressql")
register(&dbConn, "db", true, `'database specific connection string for example "user:password@tcp(localhost:3306)/dbname"`)
parse("db")
@ -92,7 +92,7 @@ func ParseFlags() (f Flags) {
f.SiteMode = siteMode
f.SSLCertFile = certFile
f.SSLKeyFile = keyFile
f.DBType = dbType
f.DBType = strings.ToLower(dbType)
return f
}

114
core/env/runtime.go vendored
View file

@ -12,48 +12,126 @@
// Package env provides runtime, server level setup and configuration
package env
import "github.com/jmoiron/sqlx"
import (
"github.com/jmoiron/sqlx"
"strings"
)
// SQL-STORE: DbVariant needs to be struct like: name, delims, std params and conn string method
// Runtime provides access to database, logger and other server-level scoped objects.
// Use Context for per-request values.
type Runtime struct {
Flags Flags
Db *sqlx.DB
DbVariant DbVariant
Log Logger
Product ProdInfo
Flags Flags
Db *sqlx.DB
Storage StoreProvider
Log Logger
Product ProdInfo
}
const (
// SiteModeNormal serves app
SiteModeNormal = ""
// SiteModeOffline serves offline.html
SiteModeOffline = "1"
// SiteModeSetup tells Ember to serve setup route
SiteModeSetup = "2"
// SiteModeBadDB redirects to db-error.html page
SiteModeBadDB = "3"
)
// DbVariant details SQL database variant
type DbVariant string
// StoreType represents name of database system
type StoreType string
const (
// DbVariantMySQL is MySQL
DbVariantMySQL DbVariant = "MySQL"
// DBVariantPercona is Percona
DBVariantPercona DbVariant = "Percona"
// DBVariantMariaDB is MariaDB
DBVariantMariaDB DbVariant = "MariaDB"
// DBVariantMSSQL is Microsoft SQL Server
DBVariantMSSQL DbVariant = "MSSQL"
// DBVariantPostgreSQL is PostgreSQL
DBVariantPostgreSQL DbVariant = "PostgreSQL"
// StoreTypeMySQL is MySQL
StoreTypeMySQL StoreType = "MySQL"
// StoreTypePercona is Percona
StoreTypePercona StoreType = "Percona"
// StoreTypeMariaDB is MariaDB
StoreTypeMariaDB StoreType = "MariaDB"
// StoreTypePostgreSQL is PostgreSQL
StoreTypePostgreSQL StoreType = "PostgreSQL"
// StoreTypeMSSQL is Microsoft SQL Server
StoreTypeMSSQL StoreType = "MSSQL"
)
// StoreProvider contains database specific details
type StoreProvider struct {
// Type identifies storage provider
Type StoreType
// SQL driver name used to open DB connection.
DriverName string
// Database connection string parameters that must be present before connecting to DB
Params map[string]string
// Example holds storage provider specific connection string format
// used in error messages
Example string
}
// ConnectionString returns provider specific DB connection string
// complete with default parameters.
func (s *StoreProvider) ConnectionString(cs string) string {
switch s.Type {
case StoreTypePostgreSQL:
return "pg"
case StoreTypeMSSQL:
return "sql server"
case StoreTypeMySQL, StoreTypeMariaDB, StoreTypePercona:
queryBits := strings.Split(cs, "?")
ret := queryBits[0] + "?"
retFirst := true
if len(queryBits) == 2 {
paramBits := strings.Split(queryBits[1], "&")
for _, pb := range paramBits {
found := false
if assignBits := strings.Split(pb, "="); len(assignBits) == 2 {
_, found = s.Params[strings.TrimSpace(assignBits[0])]
}
if !found { // if we can't work out what it is, put it through
if retFirst {
retFirst = false
} else {
ret += "&"
}
ret += pb
}
}
}
for k, v := range s.Params {
if retFirst {
retFirst = false
} else {
ret += "&"
}
ret += k + "=" + v
}
return ret
}
return ""
}
const (
// CommunityEdition is AGPL product variant
CommunityEdition = "Community"
// EnterpriseEdition is commercial licensed product variant
EnterpriseEdition = "Enterprise"
)

9
edition/boot/mysql.go
View file

@ -37,6 +37,15 @@ import (
// StoreMySQL creates MySQL provider
func StoreMySQL(r *env.Runtime, s *domain.Store) {
// Required connection string parameters and defaults.
r.Storage.Params = map[string]string{
"charset": "utf8mb4",
"parseTime": "True",
"maxAllowedPacket": "104857600", // 4194304 // 16777216 = 16MB // 104857600 = 100MB
}
r.Storage.Example = "database connection string format is 'username:password@tcp(host:3306)/database'"
s.Account = account.Scope{Runtime: r}
s.Activity = activity.Scope{Runtime: r}
s.Attachment = attachment.Scope{Runtime: r}

67
edition/boot/runtime.go
View file

@ -13,7 +13,6 @@
package boot
import (
"strings"
"time"
"github.com/documize/community/core/database"
@ -47,25 +46,35 @@ func InitRuntime(r *env.Runtime, s *domain.Store) bool {
}
}
// Prepare DB
db, err := sqlx.Open("mysql", stdConn(r.Flags.DBConn))
// Work out required storage provider set it up.
switch r.Flags.DBType {
case "mysql":
r.Storage = env.StoreProvider{Type: env.StoreTypeMySQL, DriverName: "mysql"}
StoreMySQL(r, s)
case "mariadb":
r.Storage = env.StoreProvider{Type: env.StoreTypeMariaDB, DriverName: "mysql"}
StoreMySQL(r, s)
case "percona":
r.Storage = env.StoreProvider{Type: env.StoreTypePercona, DriverName: "mysql"}
StoreMySQL(r, s)
}
// Open connection to database
db, err := sqlx.Open(r.Storage.DriverName, r.Storage.ConnectionString(r.Flags.DBConn))
if err != nil {
r.Log.Error("unable to setup database", err)
}
r.Db = db
r.Db.SetMaxIdleConns(30)
r.Db.SetMaxOpenConns(100)
r.Db.SetConnMaxLifetime(time.Second * 14400)
err = r.Db.Ping()
if err != nil {
r.Log.Error("unable to connect to database, connection string should be of the form: '"+
"username:password@tcp(host:3306)/database'", err)
r.Log.Error("unable to connect to database - "+r.Storage.Example, err)
return false
}
// go into setup mode if required
// Go into setup mode if required.
if r.Flags.SiteMode != env.SiteModeOffline {
if database.Check(r) {
if err := database.Migrate(r, true /* the config table exists */); err != nil {
@ -75,46 +84,8 @@ func InitRuntime(r *env.Runtime, s *domain.Store) bool {
}
}
// setup store based upon database type
AttachStore(r, s)
return true
}
var stdParams = map[string]string{
"charset": "utf8mb4",
"parseTime": "True",
"maxAllowedPacket": "104857600", // 4194304 // 16777216 = 16MB // 104857600 = 100MB
}
func stdConn(cs string) string {
queryBits := strings.Split(cs, "?")
ret := queryBits[0] + "?"
retFirst := true
if len(queryBits) == 2 {
paramBits := strings.Split(queryBits[1], "&")
for _, pb := range paramBits {
found := false
if assignBits := strings.Split(pb, "="); len(assignBits) == 2 {
_, found = stdParams[strings.TrimSpace(assignBits[0])]
}
if !found { // if we can't work out what it is, put it through
if retFirst {
retFirst = false
} else {
ret += "&"
}
ret += pb
}
}
}
for k, v := range stdParams {
if retFirst {
retFirst = false
} else {
ret += "&"
}
ret += k + "=" + v
}
return ret
}
// Clever way to detect database type:
// https://github.com/golang-sql/sqlexp/blob/c2488a8be21d20d31abf0d05c2735efd2d09afe4/quoter.go#L46

32
edition/boot/store.go
View file

@ -1,32 +0,0 @@
// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
//
// This software (Documize Community Edition) is licensed under
// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
//
// You can operate outside the AGPL restrictions by purchasing
// Documize Enterprise Edition and obtaining a commercial license
// by contacting <sales@documize.com>.
//
// https://documize.com
// Package boot prepares runtime environment.
package boot
import (
"github.com/documize/community/core/env"
"github.com/documize/community/domain"
)
// AttachStore selects database persistence layer
func AttachStore(r *env.Runtime, s *domain.Store) {
switch r.DbVariant {
case env.DbVariantMySQL, env.DBVariantPercona, env.DBVariantMariaDB:
StoreMySQL(r, s)
case env.DBVariantMSSQL:
// todo
case env.DBVariantPostgreSQL:
// todo
}
}
// https://github.com/golang-sql/sqlexp/blob/c2488a8be21d20d31abf0d05c2735efd2d09afe4/quoter.go#L46