From e044a4670963c95db54eb396d6455008d7c07ebd Mon Sep 17 00:00:00 2001 From: Daniel Langesten Date: Mon, 2 Mar 2015 11:48:59 +0100 Subject: renamed files --- cleaner.go | 151 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ dbsetup.sql | 28 +++++++++++ mysql.go | 151 ---------------------------------------------------------- mysqltest.sql | 28 ----------- 4 files changed, 179 insertions(+), 179 deletions(-) create mode 100644 cleaner.go create mode 100644 dbsetup.sql delete mode 100644 mysql.go delete mode 100644 mysqltest.sql diff --git a/cleaner.go b/cleaner.go new file mode 100644 index 0000000..012b308 --- /dev/null +++ b/cleaner.go @@ -0,0 +1,151 @@ +package main + +import ( + "database/sql" + _ "github.com/go-sql-driver/mysql" + "time" +) + +const ( + DATABASE_USER = "root" + DATABASE_PASS = "pass" + DATABASE_CONNECTION = "" //e.g. "tcp(localhost:55555) + DATABASE_NAME = "netflow" + PROCESS_BATCH = 3 +) + +type RawData struct { + ipSrc string + ipDst string + time time.Time + port int + packetSize int +} + +func derp() (err error) { + db, err := sql.Open("mysql", DATABASE_USER+":"+DATABASE_PASS+"@/"+DATABASE_NAME) + if err != nil { + return + } + defer db.Close() + + rDat, err := fetchRawData(db) + if err != nil { + return + } + + //Begin transaction + tx, err := db.Begin() + if err != nil { + return + } + + //TODO insertCleanedData + + //remove old data + err = purgeRawData(tx, rDat) + if err != nil { + tx.Rollback() + return + } + + tx.Commit() + return +} + +func insertCleanedData(tx *sql.DB, rDat []RawData) (err error) { + return +} + +func fetchRawData(db *sql.DB) (rDat []RawData, err error) { + prepStmt, err := db.Prepare("SELECT * FROM raw_data LIMIT ? ") + if err != nil { + return + } + + rows, err := prepStmt.Query(PROCESS_BATCH) + if err != nil { + return + } + + for rows.Next() { + var r RawData + err = rows.Scan(&r.ipSrc, &r.ipDst, &r.time, &r.port, &r.packetSize) + if err != nil { + return + } + rDat = append(rDat, r) + } + return +} +func purgeRawData(tx *sql.Tx, rDat []RawData) (err error) { + for _, r := range rDat { + prepStmt, err := tx.Prepare("DELETE FROM raw_data WHERE ip_src = ? AND ip_dst = ? AND time = ? AND port = ? AND packet_size = ?") + if err != nil { + return + } + _, err = prepStmt.Exec(r.ipSrc, r.ipDst, r.time, r.port, r.packetSize) + if err != nil { + return + } + } + return +} + +func insertCleanData(db *sql.DB, ipbSrc, ipbDst, volume string, time time.Time, port, occurences int) error { + prepStmt, err := db.Prepare("INSERT INTO clean_data (ipb_src, ipb_dst, time, port, volume, occurenaces) VALUES ( ? , ? , ? , ? , ? , ? , ?) ON DUPLICATE KEY UPDATE occurences = occurences + ?") + if err != nil { + return err + } + + _, err = prepStmt.Exec(ipbSrc, ipbDst, time, port, volume, occurences, occurences) + if err != nil { + return err + } + + return nil +} + +func insertASNIP(db *sql.DB, asn int, ipBlock string) error { + prepCheck, err := db.Prepare("SELECT asn FROM asnip WHERE ip_block = ?") + if err != nil { + return err + } + defer prepCheck.Close() + + rows, err := prepCheck.Exec(ipBlock) + if err != nil { + return err + } + if rows != nil { + return nil + } + + prepIns, err := db.Prepare("INSERT INTO asnip VALUES ( ? , ? )") + if err != nil { + return err + } + defer prepIns.Close() + + _, err = prepIns.Exec(asn, ipBlock) + if err != nil { + return err + } + + return nil +} + +func removeASNIP(db *sql.DB, asn int, ipBlock string) error { + prepStmt, err := db.Prepare("DELETE FROM anip WHERE asn = ? AND ip_block = ?") + if err != nil { + return err + } + defer prepStmt.Close() + + _, err = prepStmt.Exec(asn, ipBlock) + if err != nil { + return err + } + + return nil +} diff --git a/dbsetup.sql b/dbsetup.sql new file mode 100644 index 0000000..aad5b9a --- /dev/null +++ b/dbsetup.sql @@ -0,0 +1,28 @@ +DROP TABLE IF EXISTS asnip; + +CREATE TABLE asnip ( + asn INT, + ip_block CHAR(18) -- Since a ip block consist of a maximum of 18 chars "255.255.255.255/32" +); + +DROP TABLE IF EXISTS clean_data; + +CREATE TABLE clean_data ( + ipb_src CHAR(18), + ipb_dst CHAR(18), + time DATETIME, + port INT, + volume CHAR(10), + occurences INT, + UNIQUE idx (ipb_src, ipb_dst, time, port, packet_size) +); + +DROP TABLE IF EXISTS raw_data; + +CREATE TABLE raw_data ( + ip_src CHAR(15), -- Since a ip consist of a maximum of 15 chars "255.255.255.255" + ip_dst CHAR(15), + time DATETIME, + port INT, + packet_size INT +); diff --git a/mysql.go b/mysql.go deleted file mode 100644 index 012b308..0000000 --- a/mysql.go +++ /dev/null @@ -1,151 +0,0 @@ -package main - -import ( - "database/sql" - _ "github.com/go-sql-driver/mysql" - "time" -) - -const ( - DATABASE_USER = "root" - DATABASE_PASS = "pass" - DATABASE_CONNECTION = "" //e.g. "tcp(localhost:55555) - DATABASE_NAME = "netflow" - PROCESS_BATCH = 3 -) - -type RawData struct { - ipSrc string - ipDst string - time time.Time - port int - packetSize int -} - -func derp() (err error) { - db, err := sql.Open("mysql", DATABASE_USER+":"+DATABASE_PASS+"@/"+DATABASE_NAME) - if err != nil { - return - } - defer db.Close() - - rDat, err := fetchRawData(db) - if err != nil { - return - } - - //Begin transaction - tx, err := db.Begin() - if err != nil { - return - } - - //TODO insertCleanedData - - //remove old data - err = purgeRawData(tx, rDat) - if err != nil { - tx.Rollback() - return - } - - tx.Commit() - return -} - -func insertCleanedData(tx *sql.DB, rDat []RawData) (err error) { - return -} - -func fetchRawData(db *sql.DB) (rDat []RawData, err error) { - prepStmt, err := db.Prepare("SELECT * FROM raw_data LIMIT ? ") - if err != nil { - return - } - - rows, err := prepStmt.Query(PROCESS_BATCH) - if err != nil { - return - } - - for rows.Next() { - var r RawData - err = rows.Scan(&r.ipSrc, &r.ipDst, &r.time, &r.port, &r.packetSize) - if err != nil { - return - } - rDat = append(rDat, r) - } - return -} -func purgeRawData(tx *sql.Tx, rDat []RawData) (err error) { - for _, r := range rDat { - prepStmt, err := tx.Prepare("DELETE FROM raw_data WHERE ip_src = ? AND ip_dst = ? AND time = ? AND port = ? AND packet_size = ?") - if err != nil { - return - } - _, err = prepStmt.Exec(r.ipSrc, r.ipDst, r.time, r.port, r.packetSize) - if err != nil { - return - } - } - return -} - -func insertCleanData(db *sql.DB, ipbSrc, ipbDst, volume string, time time.Time, port, occurences int) error { - prepStmt, err := db.Prepare("INSERT INTO clean_data (ipb_src, ipb_dst, time, port, volume, occurenaces) VALUES ( ? , ? , ? , ? , ? , ? , ?) ON DUPLICATE KEY UPDATE occurences = occurences + ?") - if err != nil { - return err - } - - _, err = prepStmt.Exec(ipbSrc, ipbDst, time, port, volume, occurences, occurences) - if err != nil { - return err - } - - return nil -} - -func insertASNIP(db *sql.DB, asn int, ipBlock string) error { - prepCheck, err := db.Prepare("SELECT asn FROM asnip WHERE ip_block = ?") - if err != nil { - return err - } - defer prepCheck.Close() - - rows, err := prepCheck.Exec(ipBlock) - if err != nil { - return err - } - if rows != nil { - return nil - } - - prepIns, err := db.Prepare("INSERT INTO asnip VALUES ( ? , ? )") - if err != nil { - return err - } - defer prepIns.Close() - - _, err = prepIns.Exec(asn, ipBlock) - if err != nil { - return err - } - - return nil -} - -func removeASNIP(db *sql.DB, asn int, ipBlock string) error { - prepStmt, err := db.Prepare("DELETE FROM anip WHERE asn = ? AND ip_block = ?") - if err != nil { - return err - } - defer prepStmt.Close() - - _, err = prepStmt.Exec(asn, ipBlock) - if err != nil { - return err - } - - return nil -} diff --git a/mysqltest.sql b/mysqltest.sql deleted file mode 100644 index aad5b9a..0000000 --- a/mysqltest.sql +++ /dev/null @@ -1,28 +0,0 @@ -DROP TABLE IF EXISTS asnip; - -CREATE TABLE asnip ( - asn INT, - ip_block CHAR(18) -- Since a ip block consist of a maximum of 18 chars "255.255.255.255/32" -); - -DROP TABLE IF EXISTS clean_data; - -CREATE TABLE clean_data ( - ipb_src CHAR(18), - ipb_dst CHAR(18), - time DATETIME, - port INT, - volume CHAR(10), - occurences INT, - UNIQUE idx (ipb_src, ipb_dst, time, port, packet_size) -); - -DROP TABLE IF EXISTS raw_data; - -CREATE TABLE raw_data ( - ip_src CHAR(15), -- Since a ip consist of a maximum of 15 chars "255.255.255.255" - ip_dst CHAR(15), - time DATETIME, - port INT, - packet_size INT -); -- cgit v1.1