diff options
Diffstat (limited to 'cleaner.go')
| -rw-r--r-- | cleaner.go | 36 |
1 files changed, 19 insertions, 17 deletions
@@ -9,9 +9,9 @@ import ( "time" ) -func cleanData(conf Config, db_user, db_pass, db_conn, db_name string) (rowsLeft int, err error) { +func cleanData(cfg *Config) (rowsLeft int, err error) { - db, err := sql.Open("mysql", db_user+":"+db_pass+"@"+db_conn+"/"+db_name) + db, err := sql.Open("mysql", cfg.DBUser+":"+cfg.DBPass+"@"+cfg.DBConn+"/"+cfg.DBName) if err != nil { log.Println("Failed to connect to db") return @@ -19,25 +19,25 @@ func cleanData(conf Config, db_user, db_pass, db_conn, db_name string) (rowsLeft defer db.Close() //Remove the processed mark on entries older than 6 hours - err = reprocess(db, time.Now().Add(-1*time.Hour)) + err = reprocess(db, cfg, time.Now().Add(-1*time.Hour)) if err != nil { return } - interval, err := conf.getInterval() + interval, err := cfg.getInterval() if err != nil { return } cleanLimit := time.Now().Add(-2 * interval) //Fetch data that should be cleaned - rDat, err := fetchRawData(db, cleanLimit, conf.Limit) + rDat, err := fetchRawData(db, cfg, cleanLimit) if err != nil { log.Println("Faild to fetch raw data") return } - cDat, err := clean(rDat, conf) + cDat, err := clean(rDat, cfg) if err != nil { log.Println("Failed to clean data") return @@ -51,7 +51,7 @@ func cleanData(conf Config, db_user, db_pass, db_conn, db_name string) (rowsLeft } //save cleaned data - err = insertCleanData(tx, cDat) + err = insertCleanData(tx, cfg, cDat) if err != nil { tx.Rollback() log.Println("Failed to save cleaned data") @@ -59,13 +59,13 @@ func cleanData(conf Config, db_user, db_pass, db_conn, db_name string) (rowsLeft } //remove old data - err = purgeRawData(tx, rDat) + err = purgeRawData(tx, cfg, rDat) if err != nil { tx.Rollback() log.Println("Failed to remove old data") return } - rowsLeft, err = availableRows(tx, cleanLimit) + rowsLeft, err = availableRows(tx, cfg, cleanLimit) if err != nil { tx.Rollback() log.Println("Failed to fetch available rows") @@ -76,42 +76,42 @@ func cleanData(conf Config, db_user, db_pass, db_conn, db_name string) (rowsLeft return } -func getTimespan(t time.Time, conf Config) (span time.Time, err error) { +func getTimespan(t time.Time, cfg *Config) (span time.Time, err error) { loc, err := time.LoadLocation(TIMEZONE) if err != nil { return } switch { - case conf.Interval == "5min": //Round the date into 5 minutes + case cfg.Interval == "5min": //Round the date into 5 minutes y, m, d := t.Date() h := t.Hour() min := t.Minute() min = (min / 5) * 5 span = time.Date(y, m, d, h, min, 0, 0, loc) - case conf.Interval == "10min": //Round the date into 10 minutes + case cfg.Interval == "10min": //Round the date into 10 minutes y, m, d := t.Date() h := t.Hour() min := t.Minute() min = (min / 10) * 10 span = time.Date(y, m, d, h, min, 0, 0, loc) - case conf.Interval == "hour": //Round the date into hour + case cfg.Interval == "hour": //Round the date into hour y, m, d := t.Date() h := t.Hour() span = time.Date(y, m, d, h, 0, 0, 0, loc) - case conf.Interval == "day": //Round the date into day + case cfg.Interval == "day": //Round the date into day y, m, d := t.Date() span = time.Date(y, m, d, 0, 0, 0, 0, loc) default: - err = errors.New(fmt.Sprintf("Bad interval in config %s", conf.Interval)) + err = errors.New(fmt.Sprintf("Bad interval in config %s", cfg.Interval)) return } return } -func clean(rDat []rawData, conf Config) (cDat []cleanedData, err error) { +func clean(rDat []rawData, cfg *Config) (cDat []cleanedData, err error) { // collect all ips so we can query for their ip blocks ips := make(map[string]struct{}) for _, rd := range rDat { @@ -124,14 +124,16 @@ func clean(rDat []rawData, conf Config) (cDat []cleanedData, err error) { iplist = append(iplist, ip) } + log.Println("Querying for ip-blocks...") pairs, err := findIPBlock(iplist...) if err != nil { return } + log.Println("ip-blocks returned") for _, rd := range rDat { var tim time.Time - tim, err = getTimespan(rd.time, conf) + tim, err = getTimespan(rd.time, cfg) if err != nil { return } |