WIP: migrations

migrations/00010_go.go

@@ -5,12 +5,17 @@ import (
 	"crypto/rand"
 	"crypto/sha1"
 	"database/sql"
-	"errors"
+	"encoding/csv"
 	"fmt"
+	"io"
+	"net/http"
+	"strings"
 	"time"
 
 	"github.com/pressly/goose"
 	log "github.com/sirupsen/logrus"
+	"golang.org/x/text/encoding"
+	"golang.org/x/text/encoding/charmap"
 )
 
 func init() {
@@ -19,14 +24,28 @@ func init() {
 
 func Up20201214193523(tx *sql.Tx) error {
 	// This code is executed when the migration is applied.
-	var row *sql.Row
+	var (
+		data                            int
+		countryId, regionId, locationId int64
+		row                             *sql.Row
+		result                          sql.Result
+		err                             error
+	)
+
 	row = tx.QueryRow("SELECT COUNT(*) FROM users WHERE admin=1")
-	var data int
+	if err = row.Scan(&data); err != nil {
-	if err := row.Scan(&data); err != nil {
 		return err
 	}
 	log.Infof("%d admins found\n", data)
 
+	var countryFipsCodeMap map[string]int64
+	if countryFipsCodeMap, err = updateCountries(tx); err != nil {
+		return err
+	}
+	if err = updateRegions(tx, &countryFipsCodeMap); err != nil {
+		return err
+	}
+
 	if data == 0 {
 		location, err := time.LoadLocation("Europe/Berlin")
 		if err != nil {
@@ -40,14 +59,16 @@ func Up20201214193523(tx *sql.Tx) error {
 		if err != nil {
 			return err
 		}
-		row = tx.QueryRow("SELECT id FROM countries WHERE name='Germany'")
+
-		var ccid int
+		if countryId, err = getCountryId(tx, "Germany"); err != nil {
-		if err := row.Scan(&ccid); err != nil {
+			return err
+		}
+		if regionId, err = getRegionId(tx, "Sachsen", countryId); err != nil {
+			return err
+		}
+		if locationId, err = getLocationId(tx, "Dresden", countryId, regionId); err != nil {
 			return err
 		}
-
-		regid := 0
-		locid := 0
 
 		random64Bytes := make([]byte, 64)
 		_, err = rand.Read(random64Bytes)
@@ -55,20 +76,32 @@ func Up20201214193523(tx *sql.Tx) error {
 			return err
 		}
 
-		result, err := tx.Exec(`INSERT INTO users (email, password, fname, mname,
+		result, err = tx.Exec(`INSERT INTO users (email, password, fname, mname,
 				   lname, suffix, dob, verified, ccid,
 				   regid, locid, listme, codesign, 1024bit, contactinfo, admin, orgadmin,
 				   ttpadmin, adadmin, board, tverify, locadmin, language,
 				   Q1, Q2, Q3, Q4, Q5,
 				   A1, A2, A3, A4, A5,
-				   created, modified, deleted, locked, uniqueID, otphash,
+				   created, modified, locked, uniqueID,
-				   otppin, assurer, assurer_blocked, lastLoginAttempt)
+				   otphash, otppin, assurer, assurer_blocked, lastLoginAttempt)
-VALUES (?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, 0, 1, 0, ?, 1, 0, 0, 0, 0, 0, 0, ?, '', '', '', '', '', '', '', '', '', '', ?,
+VALUES (?, ?, ?, '', ?, '', ?, 0,
-		?, NULL, 0, SHA1(CONCAT(NOW(), ?)), '', 0, 0, 0, NULL)`,
+		?, ?, ?, 0, 1, 0, ?,
+		1, 0, 0, 0, 0, 0, 0, ?,
+		'', '', '', '', '', '', '', '', '', '',
+		CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0,
+		SHA1(CONCAT(NOW(), ?)),
+		'', 0, 0, 0, NULL)`,
 			"jandd@cacert.org",
-			fmt.Sprintf("%x", sha1.Sum([]byte("abcdefghijklmn")),
+			fmt.Sprintf("%x", sha1.Sum([]byte("abcdefghijklmn"))),
-				"Jan", "", "Dittberner", "", dob, ccid, regid, locid, "Somewhere over the rainbow",
+			"Jan",
-				time.Now(), time.Now(), fmt.Sprintf("%x", md5.Sum(random64Bytes))))
+			"Dittberner",
+			dob,
+			countryId,
+			regionId,
+			locationId,
+			"Somewhere over the rainbow",
+			"de_DE",
+			fmt.Sprintf("%x", md5.Sum(random64Bytes)))
 		if err != nil {
 			return err
 		}
@@ -78,8 +111,197 @@ VALUES (?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, 0, 1, 0, ?, 1, 0, 0, 0, 0, 0, 0, ?, '',
 		}
 		log.Infof("new user id is %d", lastId)
 	}
+	return nil
+}
 
-	return errors.New("TODO: implement")
+func updateRegions(tx *sql.Tx, codeMap *map[string]int64) error {
+	client := &http.Client{}
+	var (
+		err       error
+		request   *http.Request
+		response  *http.Response
+		csvReader *csv.Reader
+	)
+	request, err = http.NewRequest("GET", "https://raw.githubusercontent.com/datasets/fips-10-4/master/data/data.csv", nil)
+	if err != nil {
+		return err
+	}
+	response, err = client.Do(request)
+	if err != nil {
+		return err
+	}
+	if response.StatusCode != 200 {
+		return fmt.Errorf("got unexpected HTTP status %d %s", response.StatusCode, response.Status)
+	}
+	csvReader = csv.NewReader(response.Body)
+	headings, err := csvReader.Read()
+	log.Infof("CSV headings %s", strings.Join(headings, ","))
+	for {
+		record, err := csvReader.Read()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+		regionCode := record[0]
+		regionDivision := record[1]
+		regionName := record[2]
+		fipsCode := regionCode[:2]
+		log.Infof("read %s %s %s", regionCode, regionName, fipsCode)
+		var countryId int64
+		var exists bool
+		if countryId, exists = (*codeMap)[fipsCode]; exists {
+			log.Infof("country id %d", countryId)
+		} else if regionDivision == "country" {
+			countryId, err = getCountryId(tx, regionName)
+			(*codeMap)[fipsCode] = countryId
+		} else {
+			return fmt.Errorf("could not find country for %s %s", fipsCode, regionName)
+		}
+		_, err = getRegionId(tx, regionName, countryId)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func updateCountries(tx *sql.Tx) (map[string]int64, error) {
+	client := &http.Client{}
+	var (
+		err       error
+		request   *http.Request
+		response  *http.Response
+		csvReader *csv.Reader
+	)
+	request, err = http.NewRequest("GET", "https://raw.githubusercontent.com/datasets/country-codes/master/data/country-codes.csv", nil)
+	if err != nil {
+		return nil, err
+	}
+	response, err = client.Do(request)
+	if err != nil {
+		return nil, err
+	}
+	if response.StatusCode != 200 {
+		return nil, fmt.Errorf("got unexpected HTTP status %d %s", response.StatusCode, response.Status)
+	}
+	csvReader = csv.NewReader(response.Body)
+
+	headings, err := csvReader.Read()
+	log.Infof("CSV headings %s", strings.Join(headings, ","))
+
+	countryFipsMapping := make(map[string]int64, 0)
+	var count int64 = 0
+	for {
+		record, err := csvReader.Read()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+
+		name := strings.TrimSpace(record[54])
+		if len(name) > 0 {
+			countryId, err := getCountryId(tx, name)
+			if err != nil {
+				return nil, err
+			}
+			countryFipsMapping[record[7]] = countryId
+			count++
+		}
+	}
+	log.Infof("read %d countries", count)
+
+	return countryFipsMapping, nil
+}
+
+func getLocationId(tx *sql.Tx, name string, countryId, regionId int64) (int64, error) {
+	var (
+		row        *sql.Row
+		result     sql.Result
+		locationId int64
+		err        error
+	)
+	row = tx.QueryRow("SELECT id FROM locations WHERE name=? AND ccid=? AND regid=?", name, countryId, regionId)
+	if err := row.Scan(&locationId); err != nil {
+		if err != sql.ErrNoRows {
+			return 0, err
+		}
+	} else {
+		return locationId, nil
+	}
+	result, err = tx.Exec("INSERT INTO locations (regid, ccid, name, acount) VALUES (?, ?, ?, ?)", regionId, countryId, name, 0)
+	if err != nil {
+		return 0, err
+	}
+	locationId, err = result.LastInsertId()
+	if err != nil {
+		return 0, err
+	}
+	return locationId, nil
+
+}
+
+func getRegionId(tx *sql.Tx, name string, countryId int64) (int64, error) {
+	var (
+		row      *sql.Row
+		result   sql.Result
+		regionId int64
+		err      error
+	)
+	encoder := charmap.ISO8859_1.NewEncoder()
+	cutDownName, err := encoding.HTMLEscapeUnsupported(encoder).String(name)
+	if err != nil {
+		return 0, err
+	}
+	if len(cutDownName) > 50 {
+		cutDownName = cutDownName[:50]
+	}
+	row = tx.QueryRow("SELECT id FROM regions WHERE name=? AND ccid=?", cutDownName, countryId)
+	if err := row.Scan(&regionId); err != nil {
+		if err != sql.ErrNoRows {
+			return 0, err
+		}
+	} else {
+		return regionId, nil
+	}
+	result, err = tx.Exec("INSERT INTO regions (ccid, name, acount) VALUES (?, ?, ?)", countryId, cutDownName, 0)
+	if err != nil {
+		return 0, err
+	}
+	regionId, err = result.LastInsertId()
+	if err != nil {
+		return 0, err
+	}
+	return regionId, nil
+}
+
+func getCountryId(tx *sql.Tx, name string) (int64, error) {
+	var (
+		row       *sql.Row
+		result    sql.Result
+		countryId int64
+		err       error
+	)
+	row = tx.QueryRow("SELECT id FROM countries WHERE name=?", name)
+	if err := row.Scan(&countryId); err != nil {
+		if err != sql.ErrNoRows {
+			return 0, err
+		}
+	} else {
+		return countryId, nil
+	}
+	result, err = tx.Exec("INSERT INTO countries (countries.name, countries.acount) VALUES (?, ?)", name, 0)
+	if err != nil {
+		return 0, err
+	}
+	countryId, err = result.LastInsertId()
+	if err != nil {
+		return 0, err
+	}
+	return countryId, nil
 }
 
 func Down20201214193523(tx *sql.Tx) error {