Added support for how many entries that should be processed at a time.

4 files changed, 20 insertions, 5 deletions

diff --git a/cleaner.go b/cleaner.go
index 45811b8..fa93aa4 100644
--- a/cleaner.go
+++ b/cleaner.go
@@ -39,7 +39,7 @@ func cleanData(conf Config, db_user, db_pass, db_conn, db_name string) (err erro
 		return
 	}
 	//Fetch data that should be cleaned
-	rDat, err := fetchRawData(db, time.Now().Add(-2*interval))
+	rDat, err := fetchRawData(db, time.Now().Add(-2*interval), conf.Limit)
 	if err != nil {
 		log.Println("Faild to fetch raw data")
 		return
diff --git a/config.go b/config.go
index af84071..31a98d2 100644
--- a/config.go
+++ b/config.go
@@ -8,6 +8,7 @@ import (
 
 type Config struct {
 	Volumes  []VolumeInfo `json:volumes`
+	Limit    int          `json:limit`
 	Interval string       `json:interval`
 	Epsilon  float64      `json:epsilon`
 }
diff --git a/config.json b/config.json
index 4447a1b..2742756 100644
--- a/config.json
+++ b/config.json
@@ -2,6 +2,9 @@
 	"comment Interval": "Interval is how large time intervals that the data should be grouped into",
 	"interval": "day",
 
+	"comment Limit": "Limit is how many rows that is processed at a time for the program. If limit <= 0, there will be no limit",
+	"limit": 100,
+
 	"comment Epsilon": "Epsilon is the epsilon value for differential privacy. epsilon < 1 high privacy, 10 < epsilon low privacy. If epsilon is set to 0, differential privacy will not be used.",
 	"epsilon": 0,
 
diff --git a/sqlQueries.go b/sqlQueries.go
index 99a35d2..805248c 100644
--- a/sqlQueries.go
+++ b/sqlQueries.go
@@ -11,16 +11,27 @@ const (
 	TIMEZONE = "UTC"
 )
 
-//Retrieves all rawdata older than tim
-func fetchRawData(db *sql.DB, tim time.Time) (rDat []RawData, err error) {
+//Retrieves limit rawdata entries that are older than tim
+//limit <= 0 returns all entries that are older than tim
+func fetchRawData(db *sql.DB, tim time.Time, limit int) (rDat []RawData, err error) {
 
-	prepSel, err := db.Prepare("SELECT ip_src,ip_dst,time,port,packet_size FROM raw_data WHERE time < ?")
+	var prepSel *sql.Stmt
+	if limit > 0 {
+		prepSel, err = db.Prepare("SELECT ip_src,ip_dst,time,port,packet_size FROM raw_data WHERE time < ? LIMIT ?")
+	} else {
+		prepSel, err = db.Prepare("SELECT ip_src,ip_dst,time,port,packet_size FROM raw_data WHERE time < ?")
+	}
 	if err != nil {
 		log.Println("Failed to prepare select")
 		return
 	}
 
-	rows, err := prepSel.Query(tim)
+	var rows *sql.Rows
+	if limit > 0 {
+		rows, err = prepSel.Query(tim, limit)
+	} else {
+		rows, err = prepSel.Query(tim)
+	}
 	if err != nil {
 		log.Println("Failed to query prepared selection")
 		return