diff options
| -rw-r--r-- | cleaner.go | 42 |
1 files changed, 28 insertions, 14 deletions
@@ -3,6 +3,7 @@ package main import ( "database/sql" "errors" + "fmt" _ "github.com/go-sql-driver/mysql" "log" "time" @@ -13,11 +14,10 @@ const ( DATABASE_PASS = "nil" DATABASE_CONNECTION = "" //e.g. "tcp(localhost:55555) DATABASE_NAME = "netflow" - TIMESPAN = "day" EPSILON = 1000 ) -func cleanData() (err error) { +func cleanData(conf Config) (err error) { db, err := sql.Open("mysql", DATABASE_USER+":"+DATABASE_PASS+"@/"+DATABASE_NAME) if err != nil { @@ -26,20 +26,34 @@ func cleanData() (err error) { } defer db.Close() - //Remove the processed mark on entries older than 1 day - err = reprocess(db, time.Now().Add(-24*1*time.Hour)) + //Remove the processed mark on entries older than 6 hours + err = reprocess(db, time.Now().Add(-6*time.Hour)) if err != nil { return } + var interval time.Duration + switch conf.Interval { + case "5min": + interval = time.Minute * 5 + case "10min": + interval = time.Minute * 10 + case "hour": + interval = time.Hour + case "day": + interval = time.Hour * 24 + default: + err = errors.New(fmt.Sprintf("Invalid interval: %s", conf.Interval)) + return + } //Fetch data that should be cleaned - rDat, err := fetchRawData(db, time.Now().Add(-1*time.Hour)) + rDat, err := fetchRawData(db, time.Now().Add(-2*interval)) if err != nil { log.Println("Faild to fetch raw data") return } - cDat, err := clean(rDat) + cDat, err := clean(rDat, conf) if err != nil { log.Println("Failed to clean data") return @@ -79,42 +93,42 @@ func cleanData() (err error) { return } -func getTimespan(t time.Time) (span time.Time, err error) { +func getTimespan(t time.Time, conf Config) (span time.Time, err error) { loc, err := time.LoadLocation(TIMEZONE) if err != nil { return } switch { - case TIMESPAN == "5min": //Round the date into 5 minutes + case conf.Interval == "5min": //Round the date into 5 minutes y, m, d := t.Date() h := t.Hour() min := t.Minute() min = (min / 5) * 5 span = time.Date(y, m, d, h, min, 0, 0, loc) - case TIMESPAN == "10min": //Round the date into 10 minutes + case conf.Interval == "10min": //Round the date into 10 minutes y, m, d := t.Date() h := t.Hour() min := t.Minute() min = (min / 10) * 10 span = time.Date(y, m, d, h, min, 0, 0, loc) - case TIMESPAN == "hour": //Round the date into hour + case conf.Interval == "hour": //Round the date into hour y, m, d := t.Date() h := t.Hour() span = time.Date(y, m, d, h, 0, 0, 0, loc) - case TIMESPAN == "day": //Round the date into day + case conf.Interval == "day": //Round the date into day y, m, d := t.Date() span = time.Date(y, m, d, 0, 0, 0, 0, loc) default: - err = errors.New("Bad timespan") + err = errors.New(fmt.Sprintf("Bad interval in config %s", conf.Interval)) return } return } -func clean(rDat []RawData) (cDat []CleanData, err error) { +func clean(rDat []RawData, conf Config) (cDat []CleanData, err error) { // collect all ips so we can query for their ip blocks ips := make(map[string]*asnipPair) for _, rd := range rDat { @@ -142,7 +156,7 @@ func clean(rDat []RawData) (cDat []CleanData, err error) { return } var tim time.Time - tim, err = getTimespan(rd.time) + tim, err = getTimespan(rd.time, conf) if err != nil { return } |