package migrations import ( "crypto/md5" "crypto/rand" "crypto/sha1" "database/sql" "encoding/csv" "fmt" "io" "net/http" "strings" "time" "github.com/pressly/goose" log "github.com/sirupsen/logrus" "golang.org/x/text/encoding" "golang.org/x/text/encoding/charmap" ) func init() { goose.AddMigration(Up20201214193523, Down20201214193523) } func Up20201214193523(tx *sql.Tx) error { // This code is executed when the migration is applied. var ( data int countryId, regionId, locationId int64 row *sql.Row result sql.Result err error ) row = tx.QueryRow("SELECT COUNT(*) FROM users WHERE admin=1") if err = row.Scan(&data); err != nil { return err } log.Infof("%d admins found\n", data) var countryFipsCodeMap map[string]int64 if countryFipsCodeMap, err = updateCountries(tx); err != nil { return err } if err = updateRegions(tx, &countryFipsCodeMap); err != nil { return err } if data == 0 { location, err := time.LoadLocation("Europe/Berlin") if err != nil { return err } dob, err := time.ParseInLocation( "2006-01-02", "1977-08-08", location, ) if err != nil { return err } if countryId, err = getCountryId(tx, "Germany"); err != nil { return err } if regionId, err = getRegionId(tx, "Sachsen", countryId); err != nil { return err } if locationId, err = getLocationId(tx, "Dresden", countryId, regionId); err != nil { return err } random64Bytes := make([]byte, 64) _, err = rand.Read(random64Bytes) if err != nil { return err } result, err = tx.Exec(`INSERT INTO users (email, password, fname, mname, lname, suffix, dob, verified, ccid, regid, locid, listme, codesign, 1024bit, contactinfo, admin, orgadmin, ttpadmin, adadmin, board, tverify, locadmin, language, Q1, Q2, Q3, Q4, Q5, A1, A2, A3, A4, A5, created, modified, locked, uniqueID, otphash, otppin, assurer, assurer_blocked, lastLoginAttempt) VALUES (?, ?, ?, '', ?, '', ?, 0, ?, ?, ?, 0, 1, 0, ?, 1, 0, 0, 0, 0, 0, 0, ?, '', '', '', '', '', '', '', '', '', '', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0, SHA1(CONCAT(NOW(), ?)), '', 0, 0, 0, NULL)`, "jandd@cacert.org", fmt.Sprintf("%x", sha1.Sum([]byte("abcdefghijklmn"))), "Jan", "Dittberner", dob, countryId, regionId, locationId, "Somewhere over the rainbow", "de_DE", fmt.Sprintf("%x", md5.Sum(random64Bytes))) if err != nil { return err } lastId, err := result.LastInsertId() if err != nil { return err } log.Infof("new user id is %d", lastId) } return nil } func updateRegions(tx *sql.Tx, codeMap *map[string]int64) error { client := &http.Client{} var ( err error request *http.Request response *http.Response csvReader *csv.Reader ) request, err = http.NewRequest("GET", "https://raw.githubusercontent.com/datasets/fips-10-4/master/data/data.csv", nil) if err != nil { return err } response, err = client.Do(request) if err != nil { return err } if response.StatusCode != 200 { return fmt.Errorf("got unexpected HTTP status %d %s", response.StatusCode, response.Status) } csvReader = csv.NewReader(response.Body) headings, err := csvReader.Read() log.Infof("CSV headings %s", strings.Join(headings, ",")) for { record, err := csvReader.Read() if err == io.EOF { break } if err != nil { return err } regionCode := record[0] regionDivision := record[1] regionName := record[2] fipsCode := regionCode[:2] log.Infof("read %s %s %s", regionCode, regionName, fipsCode) var countryId int64 var exists bool if countryId, exists = (*codeMap)[fipsCode]; exists { log.Infof("country id %d", countryId) } else if regionDivision == "country" { countryId, err = getCountryId(tx, regionName) (*codeMap)[fipsCode] = countryId } else { return fmt.Errorf("could not find country for %s %s", fipsCode, regionName) } _, err = getRegionId(tx, regionName, countryId) if err != nil { return err } } return nil } func updateCountries(tx *sql.Tx) (map[string]int64, error) { client := &http.Client{} var ( err error request *http.Request response *http.Response csvReader *csv.Reader ) request, err = http.NewRequest("GET", "https://raw.githubusercontent.com/datasets/country-codes/master/data/country-codes.csv", nil) if err != nil { return nil, err } response, err = client.Do(request) if err != nil { return nil, err } if response.StatusCode != 200 { return nil, fmt.Errorf("got unexpected HTTP status %d %s", response.StatusCode, response.Status) } csvReader = csv.NewReader(response.Body) headings, err := csvReader.Read() log.Infof("CSV headings %s", strings.Join(headings, ",")) countryFipsMapping := make(map[string]int64, 0) var count int64 = 0 for { record, err := csvReader.Read() if err == io.EOF { break } if err != nil { return nil, err } name := strings.TrimSpace(record[54]) if len(name) > 0 { countryId, err := getCountryId(tx, name) if err != nil { return nil, err } countryFipsMapping[record[7]] = countryId count++ } } log.Infof("read %d countries", count) return countryFipsMapping, nil } func getLocationId(tx *sql.Tx, name string, countryId, regionId int64) (int64, error) { var ( row *sql.Row result sql.Result locationId int64 err error ) row = tx.QueryRow("SELECT id FROM locations WHERE name=? AND ccid=? AND regid=?", name, countryId, regionId) if err := row.Scan(&locationId); err != nil { if err != sql.ErrNoRows { return 0, err } } else { return locationId, nil } result, err = tx.Exec("INSERT INTO locations (regid, ccid, name, acount) VALUES (?, ?, ?, ?)", regionId, countryId, name, 0) if err != nil { return 0, err } locationId, err = result.LastInsertId() if err != nil { return 0, err } return locationId, nil } func getRegionId(tx *sql.Tx, name string, countryId int64) (int64, error) { var ( row *sql.Row result sql.Result regionId int64 err error ) encoder := charmap.ISO8859_1.NewEncoder() cutDownName, err := encoding.HTMLEscapeUnsupported(encoder).String(name) if err != nil { return 0, err } if len(cutDownName) > 50 { cutDownName = cutDownName[:50] } row = tx.QueryRow("SELECT id FROM regions WHERE name=? AND ccid=?", cutDownName, countryId) if err := row.Scan(®ionId); err != nil { if err != sql.ErrNoRows { return 0, err } } else { return regionId, nil } result, err = tx.Exec("INSERT INTO regions (ccid, name, acount) VALUES (?, ?, ?)", countryId, cutDownName, 0) if err != nil { return 0, err } regionId, err = result.LastInsertId() if err != nil { return 0, err } return regionId, nil } func getCountryId(tx *sql.Tx, name string) (int64, error) { var ( row *sql.Row result sql.Result countryId int64 err error ) row = tx.QueryRow("SELECT id FROM countries WHERE name=?", name) if err := row.Scan(&countryId); err != nil { if err != sql.ErrNoRows { return 0, err } } else { return countryId, nil } result, err = tx.Exec("INSERT INTO countries (countries.name, countries.acount) VALUES (?, ?)", name, 0) if err != nil { return 0, err } countryId, err = result.LastInsertId() if err != nil { return 0, err } return countryId, nil } func Down20201214193523(tx *sql.Tx) error { // This code is executed when the migration is rolled back. return nil }