community-website/migrations/00010_go.go

package migrations

import (
	"crypto/md5"
	"crypto/rand"
	"crypto/sha1"
	"database/sql"
	"encoding/csv"
	"fmt"
	"io"
	"net/http"
	"strings"
	"time"

	"github.com/pressly/goose"
	log "github.com/sirupsen/logrus"
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/charmap"
)

func init() {
	goose.AddMigration(Up20201214193523, Down20201214193523)
}

func Up20201214193523(tx *sql.Tx) error {
	// This code is executed when the migration is applied.
	var (
		data                            int
		countryId, regionId, locationId int64
		row                             *sql.Row
		result                          sql.Result
		err                             error
	)

	row = tx.QueryRow("SELECT COUNT(*) FROM users WHERE admin=1")
	if err = row.Scan(&data); err != nil {
		return err
	}
	log.Infof("%d admins found\n", data)

	var countryFipsCodeMap map[string]int64
	if countryFipsCodeMap, err = updateCountries(tx); err != nil {
		return err
	}
	if err = updateRegions(tx, &countryFipsCodeMap); err != nil {
		return err
	}

	if data == 0 {
		location, err := time.LoadLocation("Europe/Berlin")
		if err != nil {
			return err
		}
		dob, err := time.ParseInLocation(
			"2006-01-02",
			"1977-08-08",
			location,
		)
		if err != nil {
			return err
		}

		if countryId, err = getCountryId(tx, "Germany"); err != nil {
			return err
		}
		if regionId, err = getRegionId(tx, "Sachsen", countryId); err != nil {
			return err
		}
		if locationId, err = getLocationId(tx, "Dresden", countryId, regionId); err != nil {
			return err
		}

		random64Bytes := make([]byte, 64)
		_, err = rand.Read(random64Bytes)
		if err != nil {
			return err
		}

		result, err = tx.Exec(`INSERT INTO users (email, password, fname, mname,
				   lname, suffix, dob, verified, ccid,
				   regid, locid, listme, codesign, 1024bit, contactinfo, admin, orgadmin,
				   ttpadmin, adadmin, board, tverify, locadmin, language,
				   Q1, Q2, Q3, Q4, Q5,
				   A1, A2, A3, A4, A5,
				   created, modified, locked, uniqueID,
				   otphash, otppin, assurer, assurer_blocked, lastLoginAttempt)
VALUES (?, ?, ?, '', ?, '', ?, 0,
		?, ?, ?, 0, 1, 0, ?,
		1, 0, 0, 0, 0, 0, 0, ?,
		'', '', '', '', '', '', '', '', '', '',
		CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0,
		SHA1(CONCAT(NOW(), ?)),
		'', 0, 0, 0, NULL)`,
			"jandd@cacert.org",
			fmt.Sprintf("%x", sha1.Sum([]byte("abcdefghijklmn"))),
			"Jan",
			"Dittberner",
			dob,
			countryId,
			regionId,
			locationId,
			"Somewhere over the rainbow",
			"de_DE",
			fmt.Sprintf("%x", md5.Sum(random64Bytes)))
		if err != nil {
			return err
		}
		lastId, err := result.LastInsertId()
		if err != nil {
			return err
		}
		log.Infof("new user id is %d", lastId)
	}
	return nil
}

func updateRegions(tx *sql.Tx, codeMap *map[string]int64) error {
	client := &http.Client{}
	var (
		err       error
		request   *http.Request
		response  *http.Response
		csvReader *csv.Reader
	)
	request, err = http.NewRequest("GET", "https://raw.githubusercontent.com/datasets/fips-10-4/master/data/data.csv", nil)
	if err != nil {
		return err
	}
	response, err = client.Do(request)
	if err != nil {
		return err
	}
	if response.StatusCode != 200 {
		return fmt.Errorf("got unexpected HTTP status %d %s", response.StatusCode, response.Status)
	}
	csvReader = csv.NewReader(response.Body)
	headings, err := csvReader.Read()
	log.Infof("CSV headings %s", strings.Join(headings, ","))
	for {
		record, err := csvReader.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		regionCode := record[0]
		regionDivision := record[1]
		regionName := record[2]
		fipsCode := regionCode[:2]
		log.Infof("read %s %s %s", regionCode, regionName, fipsCode)
		var countryId int64
		var exists bool
		if countryId, exists = (*codeMap)[fipsCode]; exists {
			log.Infof("country id %d", countryId)
		} else if regionDivision == "country" {
			countryId, err = getCountryId(tx, regionName)
			(*codeMap)[fipsCode] = countryId
		} else {
			return fmt.Errorf("could not find country for %s %s", fipsCode, regionName)
		}
		_, err = getRegionId(tx, regionName, countryId)
		if err != nil {
			return err
		}
	}
	return nil
}

func updateCountries(tx *sql.Tx) (map[string]int64, error) {
	client := &http.Client{}
	var (
		err       error
		request   *http.Request
		response  *http.Response
		csvReader *csv.Reader
	)
	request, err = http.NewRequest("GET", "https://raw.githubusercontent.com/datasets/country-codes/master/data/country-codes.csv", nil)
	if err != nil {
		return nil, err
	}
	response, err = client.Do(request)
	if err != nil {
		return nil, err
	}
	if response.StatusCode != 200 {
		return nil, fmt.Errorf("got unexpected HTTP status %d %s", response.StatusCode, response.Status)
	}
	csvReader = csv.NewReader(response.Body)

	headings, err := csvReader.Read()
	log.Infof("CSV headings %s", strings.Join(headings, ","))

	countryFipsMapping := make(map[string]int64, 0)
	var count int64 = 0
	for {
		record, err := csvReader.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}

		name := strings.TrimSpace(record[54])
		if len(name) > 0 {
			countryId, err := getCountryId(tx, name)
			if err != nil {
				return nil, err
			}
			countryFipsMapping[record[7]] = countryId
			count++
		}
	}
	log.Infof("read %d countries", count)

	return countryFipsMapping, nil
}

func getLocationId(tx *sql.Tx, name string, countryId, regionId int64) (int64, error) {
	var (
		row        *sql.Row
		result     sql.Result
		locationId int64
		err        error
	)
	row = tx.QueryRow("SELECT id FROM locations WHERE name=? AND ccid=? AND regid=?", name, countryId, regionId)
	if err := row.Scan(&locationId); err != nil {
		if err != sql.ErrNoRows {
			return 0, err
		}
	} else {
		return locationId, nil
	}
	result, err = tx.Exec("INSERT INTO locations (regid, ccid, name, acount) VALUES (?, ?, ?, ?)", regionId, countryId, name, 0)
	if err != nil {
		return 0, err
	}
	locationId, err = result.LastInsertId()
	if err != nil {
		return 0, err
	}
	return locationId, nil

}

func getRegionId(tx *sql.Tx, name string, countryId int64) (int64, error) {
	var (
		row      *sql.Row
		result   sql.Result
		regionId int64
		err      error
	)
	encoder := charmap.ISO8859_1.NewEncoder()
	cutDownName, err := encoding.HTMLEscapeUnsupported(encoder).String(name)
	if err != nil {
		return 0, err
	}
	if len(cutDownName) > 50 {
		cutDownName = cutDownName[:50]
	}
	row = tx.QueryRow("SELECT id FROM regions WHERE name=? AND ccid=?", cutDownName, countryId)
	if err := row.Scan(&regionId); err != nil {
		if err != sql.ErrNoRows {
			return 0, err
		}
	} else {
		return regionId, nil
	}
	result, err = tx.Exec("INSERT INTO regions (ccid, name, acount) VALUES (?, ?, ?)", countryId, cutDownName, 0)
	if err != nil {
		return 0, err
	}
	regionId, err = result.LastInsertId()
	if err != nil {
		return 0, err
	}
	return regionId, nil
}

func getCountryId(tx *sql.Tx, name string) (int64, error) {
	var (
		row       *sql.Row
		result    sql.Result
		countryId int64
		err       error
	)
	row = tx.QueryRow("SELECT id FROM countries WHERE name=?", name)
	if err := row.Scan(&countryId); err != nil {
		if err != sql.ErrNoRows {
			return 0, err
		}
	} else {
		return countryId, nil
	}
	result, err = tx.Exec("INSERT INTO countries (countries.name, countries.acount) VALUES (?, ?)", name, 0)
	if err != nil {
		return 0, err
	}
	countryId, err = result.LastInsertId()
	if err != nil {
		return 0, err
	}
	return countryId, nil
}

func Down20201214193523(tx *sql.Tx) error {
	// This code is executed when the migration is rolled back.
	return nil
}