Run multiple sql files to update the db as required

Locks the config table to make sure only one instance does the update.
Refactors the start-up against an empty database code to also use the
same update mechanism.

app/app/pods/setup/route.js

@@ -17,8 +17,6 @@ export default Ember.Route.extend({
 		if (pwd.length === 0 || pwd === "{{.DBhash}}") {
 			this.transitionTo('auth.login'); // don't allow access to this page if we are not in setup mode, kick them out altogether
 		}
-
-		this.session.clearSession();
 	},
 
 	model() {

app/app/services/app-meta.js

@@ -20,6 +20,7 @@ const {
 
 export default Ember.Service.extend({
 	ajax: service(),
+	localStorage: service(),
 
 	endpoint: `${config.apiHost}/${config.apiNamespace}`,
 	orgId: '',
@@ -27,6 +28,7 @@ export default Ember.Service.extend({
 	version: '',
 	message: '',
 	allowAnonymousAccess: false,
+	setupMode: false,
 
 	getBaseUrl(endpoint) {
 		return [this.get('host'), endpoint].join('/');
@@ -40,12 +42,14 @@ export default Ember.Service.extend({
 
 		let isInSetupMode = dbhash && dbhash !== "{{.DBhash}}";
 		if (isInSetupMode) {
-			this.setProperites({
+			this.setProperties({
 				title: htmlSafe("Documize Setup"),
-				allowAnonymousAccess: false
+				allowAnonymousAccess: true,
+				setupMode: true
 			});
+			this.get('localStorage').clearAll();
 
-			return resolve();
+			return resolve(this);
 		}
 
 		return this.get('ajax').request('public/meta').then((response) => {

app/app/services/local-storage.js

@@ -22,5 +22,9 @@ export default Ember.Service.extend({
 
 	clearSessionItem: function (key) {
 		delete localStorage[key];
+	},
+
+	clearAll() {
+		localStorage.clear();
 	}
 });

build.sh

@@ -37,7 +37,7 @@ go generate
 
 echo "Compiling app..."
 cd ../..
-for arch in amd64 386 ; do
+for arch in amd64 ; do
     for os in darwin linux windows ; do
         if [ "$os" == "windows" ] ; then
             echo "Compiling documize-$os-$arch.exe"

documize/api/endpoint/authentication_endpoint.go

@@ -28,6 +28,7 @@ import (
 	"github.com/documize/community/documize/api/request"
 	"github.com/documize/community/documize/api/util"
 	"github.com/documize/community/documize/section/provider"
+	"github.com/documize/community/documize/web"
 	"github.com/documize/community/wordsmith/environment"
 	"github.com/documize/community/wordsmith/log"
 	"github.com/documize/community/wordsmith/utility"
@@ -293,7 +294,8 @@ func preAuthorizeStaticAssets(r *http.Request) bool {
 		strings.ToLower(r.URL.Path) == "/favicon.ico" ||
 		strings.ToLower(r.URL.Path) == "/robots.txt" ||
 		strings.ToLower(r.URL.Path) == "/version" ||
-		strings.HasPrefix(strings.ToLower(r.URL.Path), "/api/public/") {
+		strings.HasPrefix(strings.ToLower(r.URL.Path), "/api/public/") ||
+		((web.SiteMode == web.SiteModeSetup) && (strings.ToLower(r.URL.Path) == "/api/setup")) {
 
 		return true
 	}

documize/api/endpoint/router.go

@@ -134,6 +134,10 @@ func buildUnsecureRoutes() *mux.Router {
 func buildSecureRoutes() *mux.Router {
 	router := mux.NewRouter()
 
+	if web.SiteMode == web.SiteModeSetup {
+		router.HandleFunc("/api/setup", database.Create).Methods("POST", "OPTIONS")
+	}
+
 	// Import & Convert Document
 	router.HandleFunc("/api/import/folder/{folderID}", UploadConvertDocument).Methods("POST", "OPTIONS")
 
@@ -254,7 +258,6 @@ func AppRouter() *mux.Router {
 		log.Info("Serving OFFLINE web app")
 	case web.SiteModeSetup:
 		log.Info("Serving SETUP web app")
-		router.HandleFunc("/setup", database.Create).Methods("POST", "OPTIONS")
 	case web.SiteModeBadDB:
 		log.Info("Serving BAD DATABASE web app")
 	default:

documize/api/request/account_test.go

@@ -10,6 +10,7 @@
 // https://documize.com
 
 package request
+
 /* TODO(Elliott)
 import (
 	"github.com/documize/community/documize/api/entity"

documize/api/request/config.go

@@ -67,6 +67,29 @@ func ConfigString(area, path string) (ret string) {
 	return ret
 }
 
+// ConfigSet writes a configuration JSON element to the config table.
+func ConfigSet(area, json string) error {
+	if Db == nil {
+		return errors.New("no database")
+	}
+	if area == "" {
+		return errors.New("no area")
+	}
+	sql := "INSERT INTO `config` (`key`,`config`) " +
+		"VALUES ('" + area + "','" + json +
+		"') ON DUPLICATE KEY UPDATE `config`='" + json + "';"
+
+	stmt, err := Db.Preparex(sql)
+	if err != nil {
+		//fmt.Printf("DEBUG: Unable to prepare select SQL for ConfigSet: %s -- error: %v\n", sql, err)
+		return err
+	}
+	defer utility.Close(stmt)
+
+	_, err = stmt.Exec()
+	return err
+}
+
 // UserConfigGetJSON fetches a configuration JSON element from the userconfig table for a given orgid/userid combination.
 // Errors return the empty string. A blank path returns the whole JSON object, as JSON.
 func UserConfigGetJSON(orgid, userid, area, path string) (ret string) {

documize/api/request/context_test.go

@@ -10,6 +10,7 @@
 // https://documize.com
 
 package request
+
 /* TODO(Elliott)
 import (
 	"github.com/documize/community/wordsmith/environment"

documize/api/request/document_test.go

@@ -10,6 +10,7 @@
 // https://documize.com
 
 package request
+
 /* TODO(Elliott)
 import (
 	"github.com/documize/community/documize/api/entity"

documize/api/request/domain_test.go

@@ -10,6 +10,7 @@
 // https://documize.com
 
 package request
+
 /* TODO(Elliott)
 import "testing"
 import "net/http"

documize/api/request/init.go

@@ -12,7 +12,6 @@
 package request
 
 import (
-	"errors"
 	"fmt"
 	"os"
 	"strings"
@@ -69,18 +68,8 @@ func init() {
 			}
 
 			// go into setup mode if required
-			if database.Check(Db, connectionString,
+			if database.Check(Db, connectionString) {
-				func() (bool, error) {
+				if err := database.Migrate(true /* the config table exists */); err != nil {
-					// LockDB locks the database for migrations, returning if locked and an error.
-					// TODO, and if lock fails, wait here until it unlocks
-					return false, errors.New("LockDB TODO")
-				},
-				func() {
-					// UnlockDB unlocks the database for migrations.
-					// Reports errors in the log.
-					// TODO
-				}) {
-				if err := database.Migrate(ConfigString("META", "database")); err != nil {
 					log.Error("Unable to run database migration: ", err)
 					os.Exit(2)
 				}

documize/api/request/init_test.go

@@ -10,6 +10,7 @@
 // https://documize.com
 
 package request
+
 /* TODO(Elliott)
 import (
 	"fmt"

documize/api/request/label_test.go

@@ -10,6 +10,7 @@
 // https://documize.com
 
 package request
+
 /* TODO(Elliott)
 import (
 	"testing"

documize/api/request/labelrole_test.go

@@ -10,6 +10,7 @@
 // https://documize.com
 
 package request
+
 /* TODO(Elliott)
 import (
 	"testing"

documize/api/request/page_test.go

@@ -10,6 +10,7 @@
 // https://documize.com
 
 package request
+
 /* TODO(Elliott)
 import (
 	"strings"

documize/database/check.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/documize/community/documize/web"
 	"github.com/documize/community/wordsmith/log"
+	"github.com/documize/community/wordsmith/utility"
 	"github.com/jmoiron/sqlx"
 )
 
@@ -27,18 +28,12 @@ var dbCheckOK bool // default false
 // dbPtr is a pointer to the central connection to the database, used by all database requests.
 var dbPtr **sqlx.DB
 
-// lockDB locks the database
-var lockDB func() (bool, error)
-
-// unlockDB unlocks the database
-var unlockDB func()
-
 // Check that the database is configured correctly and that all the required tables exist.
-// It must be the first function called in the
+// It must be the first function called in this package.
-func Check(Db *sqlx.DB, connectionString string, lDB func() (bool, error), ulDB func()) bool {
+func Check(Db *sqlx.DB, connectionString string) bool {
 	dbPtr = &Db
-	lockDB = lDB
+
-	unlockDB = ulDB
+	log.Info("Running database checks, this may take a while...")
 	
 	csBits := strings.Split(connectionString, "/")
 	if len(csBits) > 1 {
@@ -52,7 +47,7 @@ func Check(Db *sqlx.DB, connectionString string, lDB func() (bool, error), ulDB
 		web.SiteMode = web.SiteModeBadDB
 		return false
 	}
-	defer rows.Close() // ignore error
+	defer utility.Close(rows)
 	var version, charset, collation string
 	if rows.Next() {
 		err = rows.Scan(&version, &charset, &collation)

documize/database/create.go

@@ -15,7 +15,6 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
-	"regexp"
 	"strings"
 	"time"
 
@@ -25,6 +24,7 @@ import (
 	"github.com/documize/community/wordsmith/utility"
 )
 
+// runSQL creates a transaction per call
 func runSQL(sql string) (id uint64, err error) {
 
 	if strings.TrimSpace(sql) == "" {
@@ -41,7 +41,7 @@ func runSQL(sql string) (id uint64, err error) {
 	result, err := tx.Exec(sql)
 
 	if err != nil {
-		tx.Rollback() // ignore error as already in an error state
+		log.IfErr(tx.Rollback())
 		log.Error("runSql - unable to run sql", err)
 		return
 	}
@@ -59,14 +59,6 @@ func runSQL(sql string) (id uint64, err error) {
 
 // Create the tables in a blank database
 func Create(w http.ResponseWriter, r *http.Request) {
-	txt := "database.Create()"
-	//defer func(){fmt.Println("DEBUG"+txt)}()
-
-	if dbCheckOK {
-		txt += " Check OK"
-	} else {
-		txt += " Check not OK"
-	}
 
 	defer func() {
 		target := "/setup"
@@ -92,14 +84,9 @@ func Create(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	txt += fmt.Sprintf("\n%#v\n", r.Form)
-
 	dbname := r.Form.Get("dbname")
 	dbhash := r.Form.Get("dbhash")
 
-	txt += fmt.Sprintf("DBname:%s (want:%s) DBhash: %s (want:%s)\n",
-		dbname, web.SiteInfo.DBname, dbhash, web.SiteInfo.DBhash)
-
 	if dbname != web.SiteInfo.DBname || dbhash != web.SiteInfo.DBhash {
 		log.Error("database.Create()'s security credentials error ", errors.New("bad db name or validation code"))
 		return
@@ -117,8 +104,6 @@ func Create(w http.ResponseWriter, r *http.Request) {
 		Revised:     time.Now(),
 	}
 
-	txt += fmt.Sprintf("\n%#v\n", details)
-
 	if details.Company == "" ||
 		details.CompanyLong == "" ||
 		details.Message == "" ||
@@ -126,43 +111,12 @@ func Create(w http.ResponseWriter, r *http.Request) {
 		details.Password == "" ||
 		details.Firstname == "" ||
 		details.Lastname == "" {
-		txt += "ERROR: required field blank"
+		log.Error("database.Create() error ",
+			errors.New("required field in database set-up form blank"))
 		return
 	}
 
-	firstSQL := "db_00000.sql"
+	if err = Migrate(false /* no tables exist yet */); err != nil {
-
-	buf, err := web.ReadFile("scripts/" + firstSQL)
-	if err != nil {
-		log.Error("database.Create()'s web.ReadFile()", err)
-		return
-	}
-
-	tx, err := (*dbPtr).Beginx()
-	if err != nil {
-		log.Error(" failed to get transaction", err)
-		return
-	}
-
-	stmts := getStatements(buf)
-
-	for i, stmt := range stmts {
-		_, err = tx.Exec(stmt)
-		txt += fmt.Sprintf("%d: %s\nResult: %v\n\n", i, stmt, err)
-		if err != nil {
-			tx.Rollback() // ignore error as already in an error state
-			log.Error("database.Create() unable to run table create sql", err)
-			return
-		}
-	}
-
-	err = tx.Commit()
-	if err != nil {
-		log.Error("database.Create()", err)
-		return
-	}
-
-	if err := Migrate(firstSQL); err != nil {
 		log.Error("database.Create()", err)
 		return
 	}
@@ -174,7 +128,6 @@ func Create(w http.ResponseWriter, r *http.Request) {
 	}
 
 	web.SiteMode = web.SiteModeNormal
-	txt += "\n Success!\n"
 }
 
 // The result of completing the onboarding process.
@@ -219,7 +172,6 @@ func setupAccount(completion onboardRequest, serial string) (err error) {
 		log.Error("Failed with error", err)
 		return err
 	}
-	//}
 
 	// Link user to organization.
 	accountID := util.UniqueID()
@@ -250,19 +202,3 @@ func setupAccount(completion onboardRequest, serial string) (err error) {
 
 	return
 }
-
-// getStatement strips out the comments and returns all the individual SQL commands (apart from "USE") as a []string.
-func getStatements(bytes []byte) []string {
-	/* Strip comments of the form '-- comment' or like this one */
-	stripped := regexp.MustCompile("(?s)--.*?\n|/\\*.*?\\*/").ReplaceAll(bytes, []byte("\n"))
-	sqls := strings.Split(string(stripped), ";")
-	ret := make([]string, 0, len(sqls))
-	for _, v := range sqls {
-		trimmed := strings.TrimSpace(v)
-		if len(trimmed) > 0 &&
-			!strings.HasPrefix(strings.ToUpper(trimmed), "USE ") { // make sure we don't USE the wrong database
-			ret = append(ret, trimmed+";")
-		}
-	}
-	return ret
-}

documize/database/migrate.go

@@ -12,11 +12,17 @@
 package database
 
 import (
-	"fmt"
+	"bytes"
+	"database/sql"
+	"regexp"
 	"sort"
 	"strings"
 
+	"github.com/jmoiron/sqlx"
+
 	"github.com/documize/community/documize/web"
+	"github.com/documize/community/wordsmith/log"
+	"github.com/documize/community/wordsmith/utility"
 )
 
 const migrationsDir = "bindata/scripts"
@@ -41,6 +47,10 @@ func migrations(lastMigration string) (migrationsT, error) {
 
 	hadLast := false
 
+	if len(lastMigration) == 0 {
+		hadLast = true
+	}
+
 	for _, v := range files {
 		if v == lastMigration {
 			hadLast = true
@@ -56,35 +66,130 @@ func migrations(lastMigration string) (migrationsT, error) {
 }
 
 // migrate the database as required, by applying the migrations.
-func (m migrationsT) migrate() error {
+func (m migrationsT) migrate(tx *sqlx.Tx) error {
 	for _, v := range m {
+		log.Info("Processing migration file: " + v)
 		buf, err := web.Asset(migrationsDir + "/" + v)
 		if err != nil {
 			return err
 		}
-		fmt.Println("DEBUG database.Migrate() ", v, ":\n", string(buf)) // TODO actually run the SQL
+		//fmt.Println("DEBUG database.Migrate() ", v, ":\n", string(buf)) // TODO actually run the SQL
+		err = processSQLfile(tx, buf)
+		if err != nil {
+			return err
+		}
+		json := `{"database":"` + v + `"}`
+		sql := "INSERT INTO `config` (`key`,`config`) " +
+			"VALUES ('META','" + json +
+			"') ON DUPLICATE KEY UPDATE `config`='" + json + "';"
+
+		_, err = tx.Exec(sql)
+		if err != nil {
+			return err
+		}
+
+		//fmt.Println("DEBUG insert 10s wait for testing")
+		//time.Sleep(10 * time.Second)
 	}
 	return nil
 }
 
-// Migrate the database as required, consolidated action.
+func migrateEnd(tx *sqlx.Tx, err error) error {
-func Migrate(lastMigration string) error {
+	if tx != nil {
-	mig, err := migrations(lastMigration)
+		_, ulerr := tx.Exec("UNLOCK TABLES;")
-	if err != nil {
+		log.IfErr(ulerr)
-		return err
+		if err == nil {
-	}
+			log.IfErr(tx.Commit())
-	if len(mig) == 0 {
+			log.Info("Database migration completed.")
-		return nil // no migrations to perform
-	}
-	locked, err := lockDB()
-	if err != nil {
-		return err
-	}
-	if locked {
-		defer unlockDB()
-		if err := mig.migrate(); err != nil {
-			return err
-		}
-	}
 			return nil
 		}
+		log.IfErr(tx.Rollback())
+	}
+	log.Error("Database migration failed: ", err)
+	return err
+}
+
+// Migrate the database as required, consolidated action.
+func Migrate(ConfigTableExists bool) error {
+
+	lastMigration := ""
+
+	tx, err := (*dbPtr).Beginx()
+	if err != nil {
+		return migrateEnd(tx, err)
+	}
+
+	if ConfigTableExists {
+		_, err = tx.Exec("LOCK TABLE `config` WRITE;")
+		if err != nil {
+			return migrateEnd(tx, err)
+		}
+
+		log.Info("Database migration lock taken.")
+
+		var stmt *sql.Stmt
+		stmt, err = tx.Prepare("SELECT JSON_EXTRACT(`config`,'$.database') FROM `config` WHERE `key` = 'META';")
+		if err == nil {
+			defer utility.Close(stmt)
+			var item = make([]uint8, 0)
+
+			row := stmt.QueryRow()
+
+			err = row.Scan(&item)
+			if err != nil {
+				return migrateEnd(tx, err)
+			}
+
+			if len(item) > 1 {
+				q := []byte(`"`)
+				lastMigration = string(bytes.TrimPrefix(bytes.TrimSuffix(item, q), q))
+			}
+		}
+		log.Info("Database migration last previously applied file was: " + lastMigration)
+	}
+
+	mig, err := migrations(lastMigration)
+	if err != nil {
+		return migrateEnd(tx, err)
+	}
+
+	if len(mig) == 0 {
+		log.Info("Database migration no updates to perform.")
+		return migrateEnd(tx, nil) // no migrations to perform
+	}
+	log.Info("Database migration will execute the following update files: " + strings.Join([]string(mig), ", "))
+
+	return migrateEnd(tx, mig.migrate(tx))
+}
+
+func processSQLfile(tx *sqlx.Tx, buf []byte) error {
+
+	stmts := getStatements(buf)
+
+	for _, stmt := range stmts {
+
+		_, err := tx.Exec(stmt)
+		if err != nil {
+			return err
+		}
+
+	}
+
+	return nil
+}
+
+// getStatement strips out the comments and returns all the individual SQL commands (apart from "USE") as a []string.
+func getStatements(bytes []byte) []string {
+	/* Strip comments of the form '-- comment' or like this one */
+	stripped := regexp.MustCompile("(?s)--.*?\n|/\\*.*?\\*/").ReplaceAll(bytes, []byte("\n"))
+	sqls := strings.Split(string(stripped), ";")
+	ret := make([]string, 0, len(sqls))
+	for _, v := range sqls {
+		trimmed := strings.TrimSpace(v)
+		if len(trimmed) > 0 &&
+			!strings.HasPrefix(strings.ToUpper(trimmed), "USE ") { // make sure we don't USE the wrong database
+			ret = append(ret, trimmed+";")
+		}
+	}
+	return ret
+}

wordsmith/environment/environment.go

@@ -19,6 +19,7 @@ import (
 	"os"
 	"sort"
 	"strings"
+	"sync"
 )
 
 // CallbackT is the type signature of the callback function of GetString().
@@ -36,6 +37,7 @@ type varsT struct {
 }
 
 var vars varsT
+var varsMutex sync.Mutex
 
 // Len is part of sort.Interface.
 func (v *varsT) Len() int {
@@ -59,6 +61,8 @@ const goInit = "(default)"
 
 // GetString sets-up the flag for later use, it must be called before ParseOK(), usually in an init().
 func GetString(target *string, name string, required bool, usage string, callback CallbackT) {
+	varsMutex.Lock()
+	defer varsMutex.Unlock()
 	name = strings.ToLower(strings.TrimSpace(name))
 	setter := Prefix + strings.ToUpper(name)
 	value := os.Getenv(setter)
@@ -76,6 +80,8 @@ var showSettings = flag.Bool("showsettings", false, "if true, show settings in t
 // It should be the first thing called by any main() that uses this library.
 // If all the required variables are not present, it prints an error and calls os.Exit(2) like flag.Parse().
 func Parse(doFirst string) {
+	varsMutex.Lock()
+	defer varsMutex.Unlock()
 	flag.Parse()
 	sort.Sort(&vars)
 	for pass := 1; pass <= 2; pass++ {