Use FastAPI routers and split things to multiple files.

14 files changed, 206 insertions, 120 deletions

diff --git a/src/couch/__init__.py b/src/db/couch/__init__.py
index a7537bc..b099235 100644
--- a/src/couch/__init__.py
+++ b/src/db/couch/__init__.py
@@ -8,4 +8,4 @@ __email__ = "rinat.sabitov@gmail.com"
 __status__ = "Development"
 
 
-from couch.client import Server  # noqa: F401
+from db.couch.client import Server  # noqa: F401
diff --git a/src/couch/client.py b/src/db/couch/client.py
index 188e0de..73d85a1 100644
--- a/src/couch/client.py
+++ b/src/db/couch/client.py
@@ -1,18 +1,16 @@
 # -*- coding: utf-8 -*-
 # Based on py-couchdb (https://github.com/histrio/py-couchdb)
 
-import os
-import json
-import uuid
 import copy
+import json
 import mimetypes
+import os
+import uuid
 import warnings
 
-from couch import utils
-from couch import feedreader
-from couch import exceptions as exp
-from couch.resource import Resource
-
+from db.couch import exceptions as exp
+from db.couch import feedreader, utils
+from db.couch.resource import Resource
 
 DEFAULT_BASE_URL = os.environ.get('COUCHDB_URL', 'http://localhost:5984/')
 
diff --git a/src/couch/exceptions.py b/src/db/couch/exceptions.py
index d7e037b..d7e037b 100644
--- a/src/couch/exceptions.py
+++ b/src/db/couch/exceptions.py
diff --git a/src/couch/feedreader.py b/src/db/couch/feedreader.py
index e293932..e293932 100644
--- a/src/couch/feedreader.py
+++ b/src/db/couch/feedreader.py
diff --git a/src/couch/resource.py b/src/db/couch/resource.py
index da1e0dd..8ff883b 100644
--- a/src/couch/resource.py
+++ b/src/db/couch/resource.py
@@ -5,10 +5,9 @@
 from __future__ import unicode_literals
 
 import json
-import requests
 
-from couch import utils
-from couch import exceptions
+import requests
+from db.couch import exceptions, utils
 
 
 class Resource(object):
diff --git a/src/couch/utils.py b/src/db/couch/utils.py
index 1cd21d8..1cd21d8 100644
--- a/src/couch/utils.py
+++ b/src/db/couch/utils.py
diff --git a/src/db.py b/src/db/db.py
index 012dfac..511748c 100755
--- a/src/db.py
+++ b/src/db/db.py
@@ -10,9 +10,9 @@
 import os
 import sys
 import time
-import couch
 
-from index import CouchIindex
+from db import couch
+from db.index import CouchIindex
 
 
 class DictDB():
diff --git a/src/index.py b/src/db/index.py
index 3541ec7..3541ec7 100644
--- a/src/index.py
+++ b/src/db/index.py
diff --git a/src/db/scanner.py b/src/db/scanner.py
new file mode 100644
index 0000000..714551f
--- /dev/null
+++ b/src/db/scanner.py
@@ -0,0 +1,46 @@
+import enum
+from datetime import datetime
+
+from sqlalchemy import (Boolean, Column, DateTime, Integer, Unicode,
+                        UniqueConstraint, create_engine)
+from sqlalchemy.ext.declarative import declarative_base
+
+from db import SqlDB, get_conn_str
+
+Base = declarative_base()
+engine = create_engine(get_conn_str())
+
+
+class Scanner(Base):
+    __tablename__ = 'scanners'
+    __table_args__ = (
+        None,
+        UniqueConstraint('id'),
+    )
+
+    id = Column(Integer, autoincrement=True, primary_key=True)
+    uuid = Column(Unicode(37), nullable=False)
+    enabled = Column(Boolean, nullable=False)
+    first_seen = Column(DateTime, default=datetime.utcnow, nullable=False)
+    last_seen = Column(DateTime, default=datetime.utcnow,
+                       onupdate=datetime.utcnow, nullable=False)
+    comment = Column(Unicode(255), nullable=True)
+    scanners = Column(Unicode(2048), nullable=False)
+    target = Column(Unicode(255), nullable=True)
+
+    def as_dict(self):
+        """Return JSON serializable dict."""
+        d = {}
+        for col in self.__table__.columns:
+            value = getattr(self, col.name)
+            if issubclass(value.__class__, enum.Enum):
+                value = value.value
+            elif issubclass(value.__class__, Base):
+                continue
+            elif issubclass(value.__class__, datetime):
+                value = str(value)
+            d[col.name] = value
+        return d
+
+
+Base.metadata.create_all(engine)
diff --git a/src/log.py b/src/log.py
new file mode 100644
index 0000000..de0a6ea
--- /dev/null
+++ b/src/log.py
@@ -0,0 +1,16 @@
+import logging
+
+
+def get_logger():
+    logger = logging.getLogger('soc-collector')
+
+    if not logger.handlers:
+        formatter = logging.Formatter('%(levelname)s:     %(message)s')
+
+        handler = logging.StreamHandler()
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+
+    logger.setLevel(logging.DEBUG)
+
+    return logger
diff --git a/src/main.py b/src/main.py
index f95a09c..aa3b133 100755
--- a/src/main.py
+++ b/src/main.py
@@ -1,20 +1,20 @@
 import os
 import sys
-import uvicorn
 
-from fastapi import FastAPI, Depends, Request
+import uvicorn
+from fastapi import Depends, FastAPI, Request
+from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse
 from fastapi_jwt_auth import AuthJWT
 from fastapi_jwt_auth.exceptions import AuthJWTException
-from fastapi.middleware.cors import CORSMiddleware
 from pydantic import BaseModel
-from index import CouchIindex
-import time
-from db import DictDB
-import requests
+
+import routers
 
 app = FastAPI()
 
+app.include_router(routers.router, prefix='/sc/v0')
+
 app.add_middleware(
     CORSMiddleware,
     allow_origins=["http://localhost:8001"],
@@ -24,28 +24,12 @@ app.add_middleware(
     expose_headers=["X-Total-Count"],
 )
 
-# TODO: X-Total-Count
-
-
 @app.middleware("http")
 async def mock_x_total_count_header(request: Request, call_next):
     response = await call_next(request)
     response.headers["X-Total-Count"] = "100"
     return response
 
-for i in range(10):
-    try:
-        db = DictDB()
-    except requests.exceptions.ConnectionError:
-        print(f'Database not responding, will try again soon.' +
-              'Attempt {i + 1} of 10.')
-    else:
-        break
-    time.sleep(10)
-else:
-    print('Database did not respond after 10 attempts, quitting.')
-    sys.exit(-1)
-
 
 def get_pubkey():
     try:
@@ -63,27 +47,6 @@ def get_pubkey():
     return pubkey
 
 
-def get_data(key=None, limit=25, skip=0, ip=None,
-             port=None, asn=None, domain=None):
-    if key:
-        return db.get(key)
-
-    selectors = dict()
-    indexes = CouchIindex().dict()
-    selectors['domain'] = domain
-
-    if ip and 'ip' in indexes:
-        selectors['ip'] = ip
-    if port and 'port' in indexes:
-        selectors['port'] = port
-    if asn and 'asn' in indexes:
-        selectors['asn'] = asn
-
-    data = db.search(**selectors, limit=limit, skip=skip)
-
-    return data
-
-
 class JWTConfig(BaseModel):
     authjwt_algorithm: str = "ES256"
     authjwt_public_key: str = get_pubkey()
@@ -107,68 +70,6 @@ def app_exception_handler(request: Request, exc: RuntimeError):
                         status_code=400)
 
 
-@app.get('/sc/v0/get')
-async def get(key=None, limit=25, skip=0, ip=None, port=None,
-              asn=None, Authorize: AuthJWT = Depends()):
-
-    Authorize.jwt_required()
-
-    data = []
-    raw_jwt = Authorize.get_raw_jwt()
-
-    if 'domains' not in raw_jwt:
-        return JSONResponse(content={"status": "error",
-                                     "message": "Could not find domains" +
-                                     "claim in JWT token"},
-                            status_code=400)
-    else:
-        domains = raw_jwt['domains']
-
-    for domain in domains:
-        data.extend(get_data(key, limit, skip, ip, port, asn, domain))
-
-    return JSONResponse(content={"status": "success", "docs": data})
-
-
-@app.get('/sc/v0/get/{key}')
-async def get_key(key=None, Authorize: AuthJWT = Depends()):
-
-    Authorize.jwt_required()
-
-    # TODO: Use JWT authz and check e.g. domain here
-
-    data = get_data(key)
-
-    return JSONResponse(content={"status": "success", "docs": data})
-
-
-@app.post('/sc/v0/add')
-async def add(data: Request, Authorize: AuthJWT = Depends()):
-
-    # Maybe we should protect this enpoint too and let the scanner use
-    # a JWT token as well.
-    # Authorize.jwt_required()
-
-    json_data = await data.json()
-
-    key = db.add(json_data)
-
-    return JSONResponse(content={"status": "success", "docs": key})
-
-
-@app.delete('/sc/v0/delete/{key}')
-async def delete(key, Authorize: AuthJWT = Depends()):
-
-    Authorize.jwt_required()
-
-    if db.delete(key) is None:
-        return JSONResponse(content={"status": "error",
-                                     "message": "Document not found"},
-                            status_code=400)
-
-    return JSONResponse(content={"status": "success", "docs": {}})
-
-
 def main(standalone=False):
     if not standalone:
         return app
diff --git a/src/routers/__init__.py b/src/routers/__init__.py
new file mode 100644
index 0000000..300ed3a
--- /dev/null
+++ b/src/routers/__init__.py
@@ -0,0 +1,9 @@
+from fastapi import APIRouter
+
+from .collector import router as collector_router
+from .scanner import router as scanner_router
+
+router = APIRouter()
+
+router.include_router(collector_router, tags=['collector'])
+router.include_router(scanner_router, tags=['scanner'])
diff --git a/src/routers/collector.py b/src/routers/collector.py
new file mode 100644
index 0000000..3cda23a
--- /dev/null
+++ b/src/routers/collector.py
@@ -0,0 +1,101 @@
+import sys
+import time
+
+import requests
+from db.db import DictDB
+from db.index import CouchIindex
+from fastapi import APIRouter, Depends, Request
+from fastapi.responses import JSONResponse
+from fastapi_jwt_auth import AuthJWT
+
+router = APIRouter()
+
+for i in range(10):
+    try:
+        db = DictDB()
+    except requests.exceptions.ConnectionError:
+        print('Database not responding, will try again soon.' +
+              f'Attempt {i + 1} of 10.')
+    else:
+        break
+    time.sleep(10)
+else:
+    print('Database did not respond after 10 attempts, quitting.')
+    sys.exit(-1)
+
+
+def get_data(key=None, limit=25, skip=0, ip=None,
+             port=None, asn=None, domain=None):
+    if key:
+        return db.get(key)
+
+    selectors = dict()
+    indexes = CouchIindex().dict()
+    selectors['domain'] = domain
+
+    if ip and 'ip' in indexes:
+        selectors['ip'] = ip
+    if port and 'port' in indexes:
+        selectors['port'] = port
+    if asn and 'asn' in indexes:
+        selectors['asn'] = asn
+
+    data = db.search(**selectors, limit=limit, skip=skip)
+
+    return data
+
+
+@router.get('/get')
+async def get(key=None, limit=25, skip=0, ip=None, port=None,
+              asn=None, Authorize: AuthJWT = Depends()):
+    Authorize.jwt_required()
+
+    data = []
+    raw_jwt = Authorize.get_raw_jwt()
+
+    if 'domains' not in raw_jwt:
+        return JSONResponse(content={"status": "error",
+                                     "message": "Could not find domains" +
+                                     "claim in JWT token"},
+                            status_code=400)
+    else:
+        domains = raw_jwt['domains']
+
+    for domain in domains:
+        data.extend(get_data(key, limit, skip, ip, port, asn, domain))
+
+    return JSONResponse(content={"status": "success", "docs": data})
+
+
+@router.get('/get/{key}')
+async def get_key(key=None, Authorize: AuthJWT = Depends()):
+    Authorize.jwt_required()
+
+    # TODO: Use JWT authz and check e.g. domain here
+
+    data = get_data(key)
+
+    return JSONResponse(content={"status": "success", "docs": data})
+
+
+@router.post('/add')
+async def add(data: Request, Authorize: AuthJWT = Depends()):
+    Authorize.jwt_required()
+
+    json_data = await data.json()
+
+    key = db.add(json_data)
+
+    return JSONResponse(content={"status": "success", "docs": key})
+
+
+@router.delete('/delete/{key}')
+async def delete(key, Authorize: AuthJWT = Depends()):
+    Authorize.jwt_required()
+
+    if db.delete(key) is None:
+        return JSONResponse(content={"status": "error",
+                                     "message": "Document not found"},
+                            status_code=400)
+
+    return JSONResponse(content={"status": "success", "docs": {}})
diff --git a/src/routers/scanner.py b/src/routers/scanner.py
new file mode 100644
index 0000000..956153b
--- /dev/null
+++ b/src/routers/scanner.py
@@ -0,0 +1,16 @@
+from fastapi import APIRouter, Depends, Request
+from fastapi.responses import JSONResponse
+from fastapi_jwt_auth import AuthJWT
+
+router = APIRouter()
+
+
+@router.get('/callhome')
+async def callhome(data: Request, Authorize: AuthJWT = Depends()):
+    Authorize.jwt_required()
+
+    json_data = await data.json()
+
+    if 'uuid' not in json_data:
+        return JSONResponse(content={"status": "error",
+                                     "message": "UUID missing"})