download-publications/main.go

package main;

import (
	       "io";
	       "os";
	       "fmt";
	       "time";
	       "strings";
	       "reflect";
	       "net/url";
	       "net/http";
	       "encoding/json";
	       "github.com/xaionaro/gorm";
	     _ "github.com/mattn/go-sqlite3";
	models "devel.mephi.ru/ut/download-publications-models"
);

type Config struct {
	DBPath			string;
	PublicationsURL		string;
	AuthorshipsURL		string;
};

func stripChars(str, chr string) string {	// The function is from here: https://www.rosettacode.org/wiki/Strip_a_set_of_characters_from_a_string#Go
	return strings.Map(func(r rune) rune {
	if strings.IndexRune(chr, r) < 0 {
		return r
	}
		return -1
	}, str)
}

func getJson(urlStr string) (r []map[string]interface{}, err error) {
	var reader io.Reader;

	u, e := url.Parse(urlStr);
	if (e == nil && u.Scheme != "") {
		resp, err := http.Get(urlStr);
		if (err != nil) {
			return r, err;
		}
		defer resp.Body.Close();
		reader = resp.Body;
	} else {
		file, err := os.Open(urlStr);
		if (err != nil) {
			return r, err;
		}
		defer file.Close();
		reader = file;
	}

	decoder := json.NewDecoder(reader);
	decoder.UseNumber();
	decoder.Decode(&r);

	return r, nil;
}

func getConfig() (config Config) {
	file, err := os.Open("config.json");
	if (err != nil) {
		fmt.Errorf("Config file error: %v", err.Error());
		os.Exit(-1);
	}
	defer file.Close();

	decoder := json.NewDecoder(file);
	decoder.Decode(&config);

	//fmt.Printf("config: %v\n", config);
	return config;
}

func parseValue(in interface{}) (out interface{}) {
	switch v := in.(type) {
		case json.Number:
			i,_ := v.Int64();
			return int(i);
		case string:
			t,e := time.Parse("2006-01-02T15:04:05.000-07:00", v); //????-??-??T??:??:??.???+??:??
			if (e == nil) {
				return t;
			}
			break;
	}

	return in;
}

// TODO: make this function recursive
func parse(datasValueMap []map[string]interface{}, dataSample interface{}) (interface{}) {
	dataFieldMap := make(map[string]string);
	dataValue    := reflect.ValueOf(dataSample).Elem();
	dataType     := dataValue.Type();

	//datas        := reflect.MakeSlice(reflect.SliceOf(dataType), 0, 0)
	var datas []interface{}

	// Getting fields names and making a map of ToLower(Name) -> Name
	fields       := dataValue.NumField();
	for i:=0; i<fields; i++ {
		field := dataType.Field(i);
		dataFieldMap[strings.ToLower(field.Name)] = field.Name;
		if (field.Type.Kind() == reflect.Struct) {
			fieldValue := dataValue.Field(i);
			subFields  := fieldValue.NumField();
			for i:=0; i<subFields; i++ {
				subField := field.Type.Field(i);
				dataFieldMap[strings.ToLower(subField.Name)] = subField.Name;
			}
		}
	}

	// Setting values
	dataPtr := reflect.New(dataType);
	data    := dataPtr.Elem();

	for _,row := range datasValueMap {
		data.Set(reflect.Zero(dataType));

		for k,v := range row {
			sk := strings.ToLower(stripChars(k, "_"));
			field := data.FieldByName(dataFieldMap[ sk ]);
			switch vt := v.(type) {
				case map[string]interface{}:
					for k,v := range vt {
						sk := strings.ToLower(stripChars(k, "_"));
						field.FieldByName(dataFieldMap[ sk ]).Set(reflect.ValueOf(parseValue( v )));
					}
					continue;
			}
			field.Set(reflect.ValueOf(parseValue( v )));
		}

		datas = append(datas, data.Interface());
	}

	return datas
}

func getDB(config Config) (db gorm.DB, err error) {
	connectionString := "file:"+config.DBPath+"?cache=shared&mode=rwc"

	//fmt.Printf("connectionString: %v\n", connectionString);

	db, err = gorm.Open("sqlite3", connectionString);
	if (err != nil) {
		return db, err;
	}

	db.DB().SetMaxIdleConns(1);
	db.SingularTable(true);

	return db, err;
}

func main() {
	config := getConfig();

	db,err := getDB(config);
	if (err != nil) {
		fmt.Errorf("DB error: %v", err.Error());
		os.Exit(-1);
	}

	publicationsValueMap, err := getJson(config.PublicationsURL);
	if (err != nil) {
		fmt.Errorf("Error: %v", err.Error());
		os.Exit(-1);
	}
	authorshipsValueMap,  err := getJson(config.AuthorshipsURL);
	if (err != nil) {
		fmt.Errorf("Error: %v", err.Error());
		os.Exit(-1);
	}
	//os.Exit(-1);

	publications := parse(publicationsValueMap, &models.Publication{}).([]interface{});
	authorships  := parse(authorshipsValueMap,  &models.Authorship{} ).([]interface{});

	//fmt.Printf("%v", publications);
	//fmt.Printf("%v", authorships);

	db.CreateTable(&models.Publication{});
	db.CreateTable(&models.Authorship{}).Model(&models.Authorship{}).AddUniqueIndex("authorship_uniidx",  "article_id",  "termit_id");

	transaction := db.Begin();

	for _,publication := range publications {
		row := publication.(models.Publication);
		transaction.Create(&row);
	}
	for _,authorship := range authorships {
		row := authorship.(models.Authorship);
		transaction.Create(&row);
	}
	transaction.Commit();

	os.Exit(0);
}