new direction

7 files changed, 272 insertions, 0 deletions

diff --git a/data/collector_container/Dockerfile b/data/collector_container/Dockerfile
new file mode 100644
index 0000000..e02a5d2
--- /dev/null
+++ b/data/collector_container/Dockerfile
@@ -0,0 +1,36 @@
+FROM debian:bullseye-20221024-slim@sha256:76cdda8fe5eb597ef5e712e4c9a9f5f1fb119e69f353daaa7bd6d0f6e66e541d
+
+EXPOSE 8000
+
+COPY ./requirements.txt /app/requirements.txt
+
+RUN apt-get update \
+    && apt-get install -y python3 python3-pip \
+    && pip3 install -r /app/requirements.txt \
+    && apt-get remove -y \
+    gcc \
+    curl \
+    wget \
+    python3-pip \
+    python3-dev \
+    && apt-get autoremove -y \
+    && apt-get clean
+
+
+# Remove setuid and setgid
+RUN find / -xdev -perm /6000 -type f -exec chmod a-s {} \; || true
+
+# Add user
+RUN useradd collector -u 1500 -s /usr/sbin/nologin
+
+COPY ./src /app/src
+
+WORKDIR /app/
+
+USER collector
+
+ENTRYPOINT ["uvicorn", "src.collector.main:app", "--host", "0.0.0.0", "--workers", "1", "--header", "server:collector"]
+
+
+
+
diff --git a/data/example_data_1.json b/data/example_data_1.json
new file mode 100644
index 0000000..69f5d85
--- /dev/null
+++ b/data/example_data_1.json
@@ -0,0 +1,57 @@
+{
+    "document_version": 1,
+    "ip": "192.0.2.10",
+    "port": 443,
+    "whois_description": "SOMENET",
+    "asn": "AS65001",
+    "asn_country_code": "SE",
+    "ptr": "host10.test.soc.sunet.se",
+    "abuse_mail": "abuse@test.soc.sunet.se",
+    "domain": "sunet.se",
+    "timestamp": "2021-06-21T14:06:00Z",
+    "display_name": "Apache 2.1.3",
+    "description": "The Apache HTTP Server is a free and open-source cross-platform web server software, released under the terms of Apache License 2.0.",
+    "custom_data": {
+        "subject_cn": {
+            "data": "Apache",
+            "display_name": "Subject Common Name"
+        },
+        "end_of_general_support": {
+            "data": false,
+            "display_name": "End of general support",
+            "description": "Is the software currently supported?"
+        }
+    },
+    "result": {
+        "cve_2015_0049": {
+            "display_name": "CVE-2015-0049",
+            "vulnerable": false,
+            "description": "Allows remote attackers to execute arbitrary code or cause a denial of service (memory corruption)."
+        },
+        "cve_2015_0050": {
+            "display_name": "CVE-2015-0050",
+            "vulnerable": false
+        },
+        "cve_2015_0060": {
+            "display_name": "CVE-2015-0060",
+            "vulnerable": true,
+            "reliability": 2
+        },
+        "cve_2015_0063": {
+            "display_name": "CVE-2015-0063",
+            "vulnerable": false
+        },
+        "insecure_cryptography": {
+            "display_name": "Insecure cryptography",
+            "vulnerable": true,
+            "reliability": 5,
+            "description": "Uses RSA instead of elliptic curve."
+        },
+        "possible_webshell": {
+            "display_name": "Webshells (PST)",
+            "investigation_needed": true,
+            "reliability": 1,
+            "description": "A webshell of type PST was confirmed at /test/webshell.php"
+        }
+    }
+}
diff --git a/data/example_data_3.json b/data/example_data_3.json
new file mode 100644
index 0000000..44d483b
--- /dev/null
+++ b/data/example_data_3.json
@@ -0,0 +1,51 @@
+{
+    "document_version": 1,
+    "ip": "192.0.2.28",
+    "port": 111,
+    "whois_description": "SOMENET",
+    "asn": "AS65001",
+    "asn_country_code": "SE",
+    "ptr": "host111.test.soc.sunet.se",
+    "abuse_mail": "abuse@test.soc.sunet.se",
+    "domain": "sunet.se",
+    "timestamp": "2021-06-30T15:00:00Z",
+    "display_name": "VMware ESXi 6.7.0 build-17700523",
+    "description": "VMware ESXi is an enterprise-class, type-1 hypervisor developed by VMware for deploying and serving virtual computers. As a type-1 hypervisor, ESXi is not a software application that is installed on an operating system; instead, it includes and integrates vital OS components, such as a kernel.",
+    "custom_data": {
+        "subject_cn": {
+            "data": "VMware ESXi",
+            "display_name": "Subject Common Name"
+        },
+        "end_of_general_support": {
+            "data": true,
+            "display_name": "End of general support",
+            "description": "Is the software currently supported?"
+        }
+    },
+    "result": {
+        "cve_2019_0001": {
+            "display_name": "CVE-2019-0001",
+            "vulnerable": false
+        },
+        "cve_2015_0002": {
+            "display_name": "CVE-2015-0002",
+            "vulnerable": false,
+            "description": "There is a use of insufficiently random values vulnerability. An unauthenticated, remote attacker can guess information by a large number of attempts. Successful exploitation may cause information leak."
+        },
+        "cve_2015_0003": {
+            "display_name": "CVE-2015-0003",
+            "vulnerable": true,
+            "reliability": 2,
+            "description": "A carefully crafted request body can cause a read to a random memory area which could cause the process to crash."
+        },
+        "cve_2015_0004": {
+            "display_name": "CVE-2015-0004",
+            "vulnerable": false
+        },
+        "cve_2015_0005": {
+            "display_name": "CVE-2015-0005",
+            "vulnerable": true,
+            "reliability": 4
+        }
+    }
+}
diff --git a/data/example_data_3_replace_test.json b/data/example_data_3_replace_test.json
new file mode 100644
index 0000000..31cc64d
--- /dev/null
+++ b/data/example_data_3_replace_test.json
@@ -0,0 +1,52 @@
+{
+    "_id": "6370498050845fac09e0fc01",
+    "document_version": 2,
+    "ip": "192.0.2.28",
+    "port": 112,
+    "whois_description": "SOMENET",
+    "asn": "AS65001",
+    "asn_country_code": "SE",
+    "ptr": "host111.test.soc.sunet.se",
+    "abuse_mail": "abuse@test.soc.sunet.se",
+    "domain": "sunet.se",
+    "timestamp": "2021-06-30T15:00:00Z",
+    "display_name": "VMware ESXi 6.7.0 build-17700523",
+    "description": "VMware ESXi is an enterprise-class, type-1 hypervisor developed by VMware for deploying and serving virtual computers. As a type-1 hypervisor, ESXi is not a software application that is installed on an operating system; instead, it includes and integrates vital OS components, such as a kernel.",
+    "custom_data": {
+        "subject_cn": {
+            "data": "VMware ESXi",
+            "display_name": "Subject Common Name"
+        },
+        "end_of_general_support": {
+            "data": true,
+            "display_name": "End of general support",
+            "description": "Is the software currently supported?"
+        }
+    },
+    "result": {
+        "cve_2019_0001": {
+            "display_name": "CVE-2019-0001",
+            "vulnerable": false
+        },
+        "cve_2015_0002": {
+            "display_name": "CVE-2015-0002",
+            "vulnerable": false,
+            "description": "There is a use of insufficiently random values vulnerability. An unauthenticated, remote attacker can guess information by a large number of attempts. Successful exploitation may cause information leak."
+        },
+        "cve_2015_0003": {
+            "display_name": "CVE-2015-0003",
+            "vulnerable": true,
+            "reliability": 2,
+            "description": "A carefully crafted request body can cause a read to a random memory area which could cause the process to crash."
+        },
+        "cve_2015_0004": {
+            "display_name": "CVE-2015-0004",
+            "vulnerable": false
+        },
+        "cve_2015_0005": {
+            "display_name": "CVE-2015-0005",
+            "vulnerable": true,
+            "reliability": 4
+        }
+    }
+}
diff --git a/data/init-mongodb.js b/data/init-mongodb.js
new file mode 100644
index 0000000..4b64674
--- /dev/null
+++ b/data/init-mongodb.js
@@ -0,0 +1,30 @@
+
+// To help improve our products, anonymous usage data is collected and sent to MongoDB periodically (https://www.mongodb.com/legal/privacy-policy).
+// You can opt-out by running the disableTelemetry() command.
+disableTelemetry()
+
+// Create the DB by inserting some data
+db.v0.insertOne({init_key: "init_data"})
+
+// Create user
+db.createUser(
+    {
+        user: "REPLACE_USERNAME",
+        pwd: "REPLACE_PASSWORD",
+        roles: [
+            {
+                role: "readWrite",
+                db: "production"
+            }
+        ]
+    }
+)
+
+// Delete the init data
+db.v0.deleteOne({init_key: "init_data"})
+
+// Disable the ad about monitoring
+db.disableFreeMonitoring()
+
+// Restart server now
+db.shutdownServer()
diff --git a/data/mongodb_container/Dockerfile b/data/mongodb_container/Dockerfile
new file mode 100644
index 0000000..32ee43b
--- /dev/null
+++ b/data/mongodb_container/Dockerfile
@@ -0,0 +1,27 @@
+FROM debian:bullseye-20221024-slim@sha256:76cdda8fe5eb597ef5e712e4c9a9f5f1fb119e69f353daaa7bd6d0f6e66e541d
+
+EXPOSE 27017
+
+RUN apt-get update && apt-get install curl -y \
+  && curl -fsSL https://pgp.mongodb.com/server-6.0.pub | tee /usr/share/keyrings/mongodb-archive-keyring.gpg > /dev/null \
+  && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/mongodb-archive-keyring.gpg] http://repo.mongodb.org/apt/debian bullseye/mongodb-org/6.0 main" | tee /etc/apt/sources.list.d/mongodb-org-6.0.list \
+  && apt-get update \
+  && apt-get install \
+  mongodb-org -y \
+  && apt-get remove -y \
+  wget \
+  curl \
+  && apt-get autoremove -y \
+  && apt-get clean
+
+# Remove setuid and setgid
+RUN find / -xdev -perm /6000 -type f -exec chmod a-s {} \; || true
+
+COPY ./data/mongodb_entrypoint.sh /mongodb_entrypoint.sh
+COPY ./data/init-mongodb.js /init-mongodb.js
+
+USER mongodb
+
+WORKDIR /data/db
+
+ENTRYPOINT ["bash", "/mongodb_entrypoint.sh"]
diff --git a/data/mongodb_entrypoint.sh b/data/mongodb_entrypoint.sh
new file mode 100755
index 0000000..3db507a
--- /dev/null
+++ b/data/mongodb_entrypoint.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+if [ ! -f /data/db/user_exist ]
+then
+  # Another port to prevent 'address already in use'
+  /usr/bin/mongod --port 27015 --nounixsocket &
+  sleep 1
+  cp /init-mongodb.js /data/db/init-mongodb.js
+  sed -i "s/REPLACE_USERNAME/$MONGODB_USERNAME/g" /data/db/init-mongodb.js
+  sed -i "s/REPLACE_PASSWORD/$MONGODB_PASSWORD/g" /data/db/init-mongodb.js
+
+  /usr/bin/mongosh localhost:27015/production /data/db/init-mongodb.js
+  sleep 1 # Allow DB to shutdown
+  /usr/bin/touch /data/db/user_exist
+  rm /data/db/init-mongodb.js
+fi
+
+# Startup normally now with our user
+exec /usr/bin/mongod --nounixsocket --bind_ip_all --auth