diff options
| -rw-r--r-- | cleaner.go | 2 | ||||
| -rw-r--r-- | config.go | 1 | ||||
| -rw-r--r-- | config.json | 3 | ||||
| -rw-r--r-- | sqlQueries.go | 19 |
4 files changed, 20 insertions, 5 deletions
@@ -39,7 +39,7 @@ func cleanData(conf Config, db_user, db_pass, db_conn, db_name string) (err erro return } //Fetch data that should be cleaned - rDat, err := fetchRawData(db, time.Now().Add(-2*interval)) + rDat, err := fetchRawData(db, time.Now().Add(-2*interval), conf.Limit) if err != nil { log.Println("Faild to fetch raw data") return @@ -8,6 +8,7 @@ import ( type Config struct { Volumes []VolumeInfo `json:volumes` + Limit int `json:limit` Interval string `json:interval` Epsilon float64 `json:epsilon` } diff --git a/config.json b/config.json index 4447a1b..2742756 100644 --- a/config.json +++ b/config.json @@ -2,6 +2,9 @@ "comment Interval": "Interval is how large time intervals that the data should be grouped into", "interval": "day", + "comment Limit": "Limit is how many rows that is processed at a time for the program. If limit <= 0, there will be no limit", + "limit": 100, + "comment Epsilon": "Epsilon is the epsilon value for differential privacy. epsilon < 1 high privacy, 10 < epsilon low privacy. If epsilon is set to 0, differential privacy will not be used.", "epsilon": 0, diff --git a/sqlQueries.go b/sqlQueries.go index 99a35d2..805248c 100644 --- a/sqlQueries.go +++ b/sqlQueries.go @@ -11,16 +11,27 @@ const ( TIMEZONE = "UTC" ) -//Retrieves all rawdata older than tim -func fetchRawData(db *sql.DB, tim time.Time) (rDat []RawData, err error) { +//Retrieves limit rawdata entries that are older than tim +//limit <= 0 returns all entries that are older than tim +func fetchRawData(db *sql.DB, tim time.Time, limit int) (rDat []RawData, err error) { - prepSel, err := db.Prepare("SELECT ip_src,ip_dst,time,port,packet_size FROM raw_data WHERE time < ?") + var prepSel *sql.Stmt + if limit > 0 { + prepSel, err = db.Prepare("SELECT ip_src,ip_dst,time,port,packet_size FROM raw_data WHERE time < ? LIMIT ?") + } else { + prepSel, err = db.Prepare("SELECT ip_src,ip_dst,time,port,packet_size FROM raw_data WHERE time < ?") + } if err != nil { log.Println("Failed to prepare select") return } - rows, err := prepSel.Query(tim) + var rows *sql.Rows + if limit > 0 { + rows, err = prepSel.Query(tim, limit) + } else { + rows, err = prepSel.Query(tim) + } if err != nil { log.Println("Failed to query prepared selection") return |