diff options
| -rw-r--r-- | README.md | 9 | ||||
| -rwxr-xr-x | docker/collector/setup.sh | 2 | ||||
| -rwxr-xr-x | src/db.py | 33 | ||||
| -rwxr-xr-x | src/wsgi.py | 89 |
4 files changed, 86 insertions, 47 deletions
@@ -59,6 +59,7 @@ Start CouchDB and the collector. Make sure to give it a username and password: export COUCHDB_USER=couchdb export COUCHDB_PASSWORD=insecure export COUCHDB_NAME=test + export COUCHDB_HOSTNAME=couchdb docker-compose up Now the database and the API server should be running, now we can try @@ -92,3 +93,11 @@ Try retreiving all observations for a user with read access to 'sunet.se': We might also filter the data: curl -s -u user1:pw1 http://localhost:80/sc/v0/get?port=111 | json_pp -json_opt utf8,pretty + +Believe it or not, but we can also get a single observation by looking up its key: + + curl -s -u user1:pw1 http://localhost:80/sc/v0/get/1633633714355 | json_pp -json_opt utf8,pretty + +We can also limit the number of results and skip N results forward with the parameters limit and skip: + + curl -s -u user1:pw1 http://localhost:80/sc/v0/get?limit=5&skip=2 | json_pp -json_opt utf8,pretty diff --git a/docker/collector/setup.sh b/docker/collector/setup.sh index 2289487..77627d4 100755 --- a/docker/collector/setup.sh +++ b/docker/collector/setup.sh @@ -2,7 +2,7 @@ pip3 install uwsgi cd /opt/collector/ -git checkout khn.nginx_wsgi +git checkout main virtualenv venv . venv/bin/activate pip3 install -r requirements.txt @@ -1,7 +1,7 @@ -# A database storing dictionaries, keyed on a timestamp. -# value = A dict which will be stored as a JSON object encoded in -# UTF-8. Note that dict keys of type integer or float will become -# strings while values will keep their type. +# A database storing dictionaries, keyed on a timestamp. value = A +# dict which will be stored as a JSON object encoded in UTF-8. Note +# that dict keys of type integer or float will become strings while +# values will keep their type. # Note that there's a (slim) chance that you'd stomp on the previous # value if you're too quick with generating the timestamps, ie @@ -76,26 +76,35 @@ class DictDB(): def slice(self, key_from=None, key_to=None): pass - def search(self, **kwargs): + def search(self, limit=25, skip=0, **kwargs): """ - Execute a Mango query, ideally we should have an index matching the, + Execute a Mango query, ideally we should have an index matching the query otherwise things will be slow. """ data = list() selector = dict() + try: + limit = int(limit) + skip = int(skip) + except ValueError: + limit = 25 + skip = 0 + if kwargs: - selector = {"selector": {}} + selector = { + "limit": limit, + "skip": skip, + "selector": {} + } for key in kwargs: - if kwargs[key] is None: - continue - if kwargs[key].isnumeric(): + if kwargs[key] and kwargs[key].isnumeric(): kwargs[key] = int(kwargs[key]) selector['selector'][key] = {'$eq': kwargs[key]} - print(selector) - for doc in self.couchdb.find(selector): + + for doc in self.couchdb.find(selector, wrapper=None, limit=5): data.append(doc) return data diff --git a/src/wsgi.py b/src/wsgi.py index 701f77d..0aff8f4 100755 --- a/src/wsgi.py +++ b/src/wsgi.py @@ -34,9 +34,11 @@ class CollectorResource(): def user_auth(self, auth_header, authfun): if not auth_header: return None, None # Fail. + BAlit, b64 = auth_header.split() if BAlit != "Basic": return None, None # Fail + userbytes, pwbytes = b64decode(b64).split(b':') try: user = userbytes.decode('utf-8') @@ -47,36 +49,55 @@ class CollectorResource(): class EPGet(CollectorResource): - def on_get(self, req, resp): + def on_get(self, req, resp, key=None): out = list() selectors = dict() - resp.status = falcon.HTTP_200 - resp.content_type = falcon.MEDIA_JSON - print(req.context) - if 'domains' in req.context['user']: - orgs = req.context['user']['domains'] +<< << << < HEAD + resp.status = falcon.HTTP_200 + resp.content_type = falcon.MEDIA_JSON - if not orgs: - resp.status = falcon.HTTP_401 - resp.text = json.dumps( - {'status': 'error', 'message': 'Invalid username or password\n'}) - return + print(req.context) + if 'domains' in req.context['user']: + orgs = req.context['user']['domains'] +== == == = + limit = 25 + skip = 0 - for param in req.params: - for i in index.indexes: - for j in i['index']['fields']: - if j == param: - selectors[param] = req.params[param] + orgs = self.user_auth(req.auth, self._users.read_perms) +>>>>>> > main - for org in orgs: - selectors['domain'] = org - data = self._db.search(**selectors) - if data: - out.append(data) + if not orgs: + resp.status = falcon.HTTP_401 + resp.text = json.dumps({ + 'status': 'error', + 'message': 'Invalid username or password\n' + }) + return + if key: + out = self._db.get(key) resp.text = json.dumps({'status': 'success', 'data': out}) + return + + for param in req.params: + if param == 'limit': + limit = req.params['limit'] + elif param == 'skip': + skip = req.params['skip'] + for i in index.indexes: + for j in i['index']['fields']: + if j == param: + selectors[param] = req.params[param] + + for org in orgs: + selectors['domain'] = org + data = self._db.search(**selectors, limit=limit, skip=skip) + if data: + out += data + + resp.text = json.dumps({'status': 'success', 'data': out}) class EPAdd(CollectorResource): @@ -102,6 +123,7 @@ class EPAdd(CollectorResource): # NOTE: Reading the whole body in one go instead of streaming # it nicely. rawin = req.bounded_stream.read() + try: decodedin = rawin.decode('UTF-8') except Exception: @@ -133,25 +155,24 @@ class EPAdd(CollectorResource): def main(port=8000, wsgi_helper=False): db = DictDB(database, hostname, username, password) users = authn.UserDB('wsgi_demo_users.yaml') - resources_map = [ - ('/sc/v0/add', EPAdd(db, users)), - ('/sc/v0/get', EPGet(db, users)) - ] app = falcon.App(cors_enable=True, middleware=middleware_jwt) + app.add_route('/sc/v0/add', EPAdd(db, users)) + app.add_route('/sc/v0/get', EPGet(db, users)) + app.add_route('/sc/v0/get/{key}', EPGet(db, users)) - for url, res in resources_map: - app.add_route(url, res) - - if not wsgi_helper: - print('Serving on port 8000...') - httpd = make_server('', port, app) - httpd.serve_forever() + if wsgi_helper: + return app - return app + print('Serving on port 8000...') + httpd = make_server('', port, app) + httpd.serve_forever() if __name__ == '__main__': - sys.exit(main()) + try: + sys.exit(main()) + except KeyboardInterrupt: + print('\nBye!') else: app = main(port=8000, wsgi_helper=True) |