updated tests so you just run 'go test' and also so it tests db as well as from stdin

3 files changed, 185 insertions, 1 deletions

diff --git a/flow-cleaner_test.go b/flow-cleaner_test.go
index a8f9f55..51283a7 100644
--- a/flow-cleaner_test.go
+++ b/flow-cleaner_test.go
@@ -6,7 +6,9 @@ import (
 	"fmt"
 	_ "github.com/go-sql-driver/mysql"
 	"io"
+	"io/ioutil"
 	"os"
+	"strings"
 	"testing"
 	"time"
 )
@@ -25,9 +27,58 @@ func TestCleaningFromJSON(t *testing.T) {
 		DBPass:     "nil",
 	}
 
+	fmt.Println("== Testing to process from stdin ==")
+	prepareDB(t, cfg)
 	testProcessFromStdin(t, cfg)
 	time.Sleep(15 * time.Second)
 	controlDB(t, cfg)
+	fmt.Println("== Finished testing to process from stdin ==")
+}
+
+func TestCleaningFromDB(t *testing.T) {
+	cfg := &Config{
+		Limit:      0,
+		Interval:   "5min",
+		Epsilon:    0,
+		DataSource: "mysql",
+
+		DBConn:     "",
+		DBName:     "test",
+		RawTable:   "test_raw",
+		CleanTable: "test_clean",
+		DBUser:     "flowcleaner",
+		DBPass:     "nil",
+	}
+
+	fmt.Println("== Testing to process from DB ==")
+	prepareDB(t, cfg)
+	processFromDB(cfg)
+	controlDB(t, cfg)
+	fmt.Println("== Finished testing to process from DB ==")
+}
+
+func prepareDB(t *testing.T, cfg *Config) {
+	db, err := sql.Open("mysql", cfg.DBUser+":"+cfg.DBPass+"@"+cfg.DBConn+"/"+cfg.DBName)
+	if err != nil {
+		t.Fatal("Failed to connect to db:", err)
+	}
+	defer db.Close()
+
+	file, err := ioutil.ReadFile("testdata/dbTestSetup.mysql")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, query := range strings.Split(string(file), ";") {
+		query = strings.TrimSpace(query)
+		if len(query) > 0 {
+			_, err = db.Exec(query + ";")
+			if err != nil {
+				fmt.Println("QUERY:", query)
+				t.Fatal(err)
+			}
+		}
+	}
 }
 
 func testProcessFromStdin(t *testing.T, cfg *Config) {
diff --git a/test.sh b/test.sh
deleted file mode 100644
index 5fe4612..0000000
--- a/test.sh
+++ /dev/null
@@ -1 +0,0 @@
-mysql -u root -p < dbSetup/dbTestData.mysql && go test
diff --git a/testdata/dbTestSetup.mysql b/testdata/dbTestSetup.mysql
new file mode 100644
index 0000000..a4bd59c
--- /dev/null
+++ b/testdata/dbTestSetup.mysql
@@ -0,0 +1,134 @@
+drop database if exists test;
+create database test;
+
+USE test;
+
+drop table if exists test_raw;
+
+create table test_raw (
+	ip_src CHAR(39) NOT NULL,
+	ip_dst CHAR(39) NOT NULL,
+	as_src INT(4) UNSIGNED NOT NULL,
+	as_dst INT(4) UNSIGNED NOT NULL,
+	port_src INT(2) UNSIGNED NOT NULL,
+	port_dst INT(2) UNSIGNED NOT NULL,
+	packets INT UNSIGNED NOT NULL,
+	bytes INT UNSIGNED NOT NULL,
+	pkt_len_distrib CHAR(10) NOT NULL,
+	stamp_inserted DATETIME NOT NULL,
+	stamp_updated DATETIME,
+	stamp_processed DATETIME,
+	PRIMARY KEY (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, pkt_len_distrib, stamp_inserted)
+);
+
+DROP TABLE IF EXISTS test_clean;
+
+CREATE TABLE test_clean (
+	ipb_src CHAR(39) NOT NULL,
+	ipb_dst CHAR(39) NOT NULL,
+	as_src INT(4) UNSIGNED NOT NULL,
+	as_dst INT(4) UNSIGNED NOT NULL,
+	port_src INT(2) UNSIGNED NOT NULL,
+	port_dst INT(2) UNSIGNED NOT NULL,
+	occurences INT UNSIGNED NOT NULL,
+	volume CHAR(10) NOT NULL,
+	time_added DATETIME NOT NULL,
+	time_privatized DATETIME,
+	UNIQUE idx (ipb_src, ipb_dst, as_src, as_dst, port_src, port_dst, volume, time_added)
+);
+
+grant all privileges on test.* to flowcleaner@localhost identified by 'nil' with grant option;
+-- New data that should not be processed in case there is more that should be aggregated together with it.
+-- Should give wrong number of rows if it is processed
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("123.123.123.124", "12.12.12.12", 123, 321, 80, 80, 3, 300, "0-200", NOW());
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("123.123.123.124", "73.12.12.12", 123, 321, 80, 80, 3, 300, "0-200", NOW());
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("123.123.123.124", "12.12.12.12", 123, 319, 77, 78, 3, 301, "0-200", NOW());
+
+-- Old data that should be processed
+
+-- First set of data
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("1.1.1.1", "2.2.2.2", 1, 2, 5, 10, 5, 3, "0-199", "2008-12-01 12:23:32");
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("1.1.1.1", "2.2.2.2", 1, 2, 5, 10, 6, 3, "0-199", "2008-12-01 12:23:33");
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("1.1.1.1", "2.2.2.2", 1, 2, 5, 10, 7, 3, "0-199", "2008-12-01 12:23:34");
+
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("1.1.1.1", "2.2.2.2", 1, 2, 5, 10, 9, 3, "200-299", "2008-12-01 12:23:32");
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("1.1.1.1", "2.2.2.2", 1, 2, 5, 10, 1, 3, "200-299", "2008-12-01 12:23:33");
+
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("1.1.1.1", "2.2.2.2", 2, 3, 5, 10, 7, 3, "200-299", "2008-12-01 12:23:32");
+
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("2.2.2.2", "1.1.1.1", 2, 3, 5, 10, 6, 3, "200-299", "2008-12-01 12:23:32");
+
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("2.2.2.2", "1.1.1.1", 2, 3, 6, 10, 1, 3, "200-299", "2008-12-01 12:23:32");
+
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("2.2.2.2", "1.1.1.1", 2, 3, 6, 11, 0, 3, "200-299", "2008-12-01 12:23:32");
+
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("2.2.2.2", "1.1.1.1", 2, 4, 6, 11, 4, 3, "200-299", "2008-12-01 12:23:32");
+
+-- Second set of data
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("1.1.1.1", "2.2.2.2", 1, 2, 5, 10, 5, 3, "0-199", "2008-12-01 12:28:13");
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("1.1.1.1", "2.2.2.2", 1, 2, 5, 10, 6, 3, "0-199", "2008-12-01 12:28:14");
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("1.1.1.1", "2.2.2.2", 1, 2, 5, 10, 7, 3, "0-199", "2008-12-01 12:28:15");
+
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("1.1.1.1", "2.2.2.2", 1, 2, 5, 10, 9, 3, "200-299", "2008-12-01 12:28:13");
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("1.1.1.1", "2.2.2.2", 1, 2, 5, 10, 1, 3, "200-299", "2008-12-01 12:28:14");
+
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("1.1.1.1", "2.2.2.2", 2, 3, 5, 10, 7, 3, "200-299", "2008-12-01 12:28:13");
+
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("2.2.2.2", "1.1.1.1", 2, 3, 5, 10, 6, 3, "200-299", "2008-12-01 12:28:13");
+
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("2.2.2.2", "1.1.1.1", 2, 3, 6, 10, 1, 3, "200-299", "2008-12-01 12:28:13");
+
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("2.2.2.2", "1.1.1.1", 2, 3, 6, 11, 0, 3, "200-299", "2008-12-01 12:28:13");
+
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("2.2.2.2", "1.1.1.1", 2, 4, 6, 11, 4, 3, "200-299", "2008-12-01 12:28:13");
+
+-- Third set of data
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("1.1.1.1", "2.2.2.2", 1, 2, 5, 10, 5, 3, "0-199", "2008-12-01 12:29:59");
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("1.1.1.1", "2.2.2.2", 1, 2, 5, 10, 6, 3, "0-199", "2008-12-01 12:29:58");
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("1.1.1.1", "2.2.2.2", 1, 2, 5, 10, 7, 3, "0-199", "2008-12-01 12:29:57");
+
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("1.1.1.1", "2.2.2.2", 1, 2, 5, 10, 9, 3, "200-299", "2008-12-01 12:29:59");
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("1.1.1.1", "2.2.2.2", 1, 2, 5, 10, 1, 3, "200-299", "2008-12-01 12:29:58");
+
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("1.1.1.1", "2.2.2.2", 2, 3, 5, 10, 7, 3, "200-299", "2008-12-01 12:29:59");
+
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("2.2.2.2", "1.1.1.1", 2, 3, 5, 10, 6, 3, "200-299", "2008-12-01 12:29:59");
+
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("2.2.2.2", "1.1.1.1", 2, 3, 6, 10, 1, 3, "200-299", "2008-12-01 12:29:59");
+
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("2.2.2.2", "1.1.1.1", 2, 3, 6, 11, 0, 3, "200-299", "2008-12-01 12:29:59");
+
+INSERT INTO test_raw (ip_src, ip_dst, as_src, as_dst, port_src, port_dst, packets, bytes, pkt_len_distrib, stamp_inserted)
+	VALUES ("2.2.2.2", "1.1.1.1", 2, 4, 6, 11, 4, 3, "200-299", "2008-12-01 12:29:59");