diff options
58 files changed, 735 insertions, 3617 deletions
diff --git a/.jenkins b/.jenkins deleted file mode 100644 index f6ddc6c..0000000 --- a/.jenkins +++ /dev/null @@ -1,31 +0,0 @@ -disabled: false - -git: - branch: main - extensions: - shallow_clone: true - -extra_jobs: - - name: collector - git: - branch: main - builders: - - docker - docker_name: soc-collector/collector - docker_context_dir: docker/collector - - - name: nginx - git: - branch: main - builders: - - docker - docker_name: soc-collector/nginx - docker_context_dir: docker/nginx - - - name: couchdb - git: - branch: main - builders: - - docker - docker_name: soc-collector/couchdb - docker_context_dir: docker/couchdb diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..582803d --- /dev/null +++ b/.pylintrc @@ -0,0 +1,2 @@ +[MASTER] +extension-pkg-whitelist=pydantic diff --git a/README.md b/README.md deleted file mode 100644 index c2f1325..0000000 --- a/README.md +++ /dev/null @@ -1,99 +0,0 @@ -# soc_collector -- Gathering vulnerability information and presenting it - -## The oneliner - -The collector answers the fundamental question constantly posed by all -SOC staff ever: Can we have lunch now? - -## The elevator pitch - -You're working as a Security Operations Center engineer and your job -is to, one, know when any part of your infrastructure is vulnerable -and, two, if it is, do something smart about it. - -The collector compiles data from vulnerability scanners and stores the -data in a database. You query the collector for the current -vulnerability status of your network infrastructure. - -Without a summary of your vulnerability status and the ability to quickly -deepen your knowledge of a given system, your chances of ever eating -lunch with a clear conscience are slim. - -## The user interface - -TODO - -## The gory^Wtechnical details - -TODO - -## The name - -The "soc" part means Security Operations Center. - -The "collector" part is correct but misleading since `soc_collector` -also processes and presents. - -## The license - -This code is licensed under the 2-Clause BSD License, see LICENSE for -the full text. - -## How to test it out - -The collector has been tested on Debian 11 (Bullseye). Other Unix -systems should also be capable of running a collector. - -Clone the repository. - - git clone https://git.sunet.se/soc_collector.git - -Install dependencies (Debian). - - sudo apt install docker.io docker-compose jq curl apache2-utils - -Start the collector and JWT server, and generate certificates for JWT signing: - - ./quickstart.sh - -Now the database and the API server should be running, now we can try -adding some observations. First, get a JWT for the default user `usr`: - - JWT=$(curl http://localhost:8000/api/v1.0/auth -X POST -p -u usr:pwd | jq -r .access_token) - -Then we use the JWT to add an observation (defined in `example_data.json`): - - curl -s --data-binary @example_data.json -H "Authorization: Bearer $JWT" https://localhost:1443/sc/v0/add - -Try retreiving all observations permitted by our JWT: - - curl -s -H "Authorization: Bearer $JWT" https://localhost:1443/sc/v0/get | json_pp -json_opt utf8,pretty - -We might also filter the data: - - curl -s -H "Authorization: Bearer $JWT" https://localhost:1443/sc/v0/get?port=111 | json_pp -json_opt utf8,pretty - -Believe it or not, but we can also get a single observation by looking up its key (_id): - - curl -s -H "Authorization: Bearer $JWT" https://localhost:1443/sc/v0/get/1633633714355 | json_pp -json_opt utf8,pretty - -We can also limit the number of results and skip N results forward with the parameters limit and skip: - - curl -s -H "Authorization: Bearer $JWT" 'https://localhost:1443/sc/v0/get?limit=5&skip=2' | json_pp -json_opt utf8,pretty - -## Tips and tricks - -There is a convenience script `do-as` which simplifies performing actions as a particular user. - -You can decode a JWT using jq by piping to `jq -r '.access_token | split(".") | .[0],.[1] | @base64d' | jq`. Full example: - - curl http://localhost:8000/api/v1.0/auth -X POST -p -u user1:pwd | jq -r '.access_token | split(".") | .[0],.[1] | @base64d' | jq - -## Development - -There are two docker-compose files used for development: - -- `docker/docker-compose-dev.yaml` for the collector, and -- `auth-server-poc/docker-compose.yml` for the JWT server. - -To apply changes, `source env-vars.sh` and do e.g. `docker-compose -f docker/docker-compose-dev.yaml up -d --build collector`. diff --git a/auth-server-poc/.gitignore b/auth-server-poc/.gitignore deleted file mode 100644 index 894a44c..0000000 --- a/auth-server-poc/.gitignore +++ /dev/null @@ -1,104 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ diff --git a/auth-server-poc/Dockerfile b/auth-server-poc/Dockerfile deleted file mode 100644 index b3344d2..0000000 --- a/auth-server-poc/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -FROM debian:bullseye-20221024-slim@sha256:76cdda8fe5eb597ef5e712e4c9a9f5f1fb119e69f353daaa7bd6d0f6e66e541d -# FROM debian:buster - -RUN mkdir /opt/auth-server-poc -COPY requirements.txt setup.sh /opt/auth-server-poc/ -RUN /opt/auth-server-poc/setup.sh - -# Prepare for supervisord, uwsgi, ngninx -COPY config/uwsgi.ini /opt/auth-server-poc/ -#COPY config/.htpasswd /opt/auth-server-poc/.htpasswd -COPY config/supervisord_app.conf /etc/supervisor/supervisord.conf -COPY config/nginx_app.conf /etc/nginx/sites-available/ -COPY config/nginx.conf /etc/nginx/ - -# Give nginx some special treatment -RUN unlink /etc/nginx/sites-enabled/default -RUN ln -s /etc/nginx/sites-available/nginx_app.conf /etc/nginx/sites-enabled/default -RUN chown www-data:www-data /var/log/nginx -RUN chown -R www-data:www-data /var/log/nginx/ -RUN chown -R www-data:www-data /var/lib/nginx -RUN chown www-data:www-data /var/lib/nginx/ -RUN chown www-data:www-data /opt/auth-server-poc - -# Expose HTTP -EXPOSE 80 - -COPY ./ /opt/auth-server-poc/ - -ENTRYPOINT supervisord -c /etc/supervisor/supervisord.conf diff --git a/auth-server-poc/LICENSE b/auth-server-poc/LICENSE deleted file mode 100644 index 8aad690..0000000 --- a/auth-server-poc/LICENSE +++ /dev/null @@ -1,25 +0,0 @@ -BSD 2-Clause License - -Copyright (c) 2019, SUNET -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/auth-server-poc/README.md b/auth-server-poc/README.md deleted file mode 100644 index 37029c3..0000000 --- a/auth-server-poc/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# auth-server-poc - -This is a modified version of [SUNET/auth-server-poc](https://github.com/SUNET/auth-server-poc). More detailed information is available in the original README at this URL. - -``` -Start container: -$ docker-compose up - -Generate JWT cert: -$ docker exec auth-server-poc /opt/auth-server-poc/gen-jwt-cert.sh - -Create user accounts: -(note that the -c flag is used to create the .htpasswd file and should only be used the first time) -$ docker exec auth-server-poc htpasswd -c /opt/auth-server-poc/userdb/.htpasswd indy -$ docker exec auth-server-poc htpasswd /opt/auth-server-poc/userdb/.htpasswd bob - -Get a token: -$ curl http://localhost:8000/api/v1.0/auth -X POST -p -u indy -``` diff --git a/auth-server-poc/config/nginx.conf b/auth-server-poc/config/nginx.conf deleted file mode 100644 index 6b17bd0..0000000 --- a/auth-server-poc/config/nginx.conf +++ /dev/null @@ -1,23 +0,0 @@ -user www-data; -worker_processes auto; -pid /tmp/nginx.pid; -include /etc/nginx/modules-enabled/*.conf; - -events { - worker_connections 768; -} - -http { - sendfile on; - tcp_nopush on; - tcp_nodelay on; - keepalive_timeout 65; - types_hash_max_size 2048; - include /etc/nginx/mime.types; - default_type application/octet-stream; - access_log /var/log/nginx/access.log; - error_log /var/log/nginx/error.log; - gzip on; - include /etc/nginx/conf.d/*.conf; - include /etc/nginx/sites-enabled/*; -} diff --git a/auth-server-poc/config/nginx_app.conf b/auth-server-poc/config/nginx_app.conf deleted file mode 100644 index 7b1e6f9..0000000 --- a/auth-server-poc/config/nginx_app.conf +++ /dev/null @@ -1,17 +0,0 @@ -server { - listen 80; - server_name auth-server-poc; - client_max_body_size 200M; - - location / { - limit_except OPTIONS { - auth_basic "auth-server-poc static auth"; - auth_basic_user_file "/opt/auth-server-poc/userdb/.htpasswd"; - } - uwsgi_pass unix:///tmp/uwsgi.sock; - default_type application/json; - include uwsgi_params; - uwsgi_param REMOTE_USER $remote_user; - uwsgi_param AUTH_TYPE Basic; - } -} diff --git a/auth-server-poc/config/supervisord_app.conf b/auth-server-poc/config/supervisord_app.conf deleted file mode 100644 index 3a03f32..0000000 --- a/auth-server-poc/config/supervisord_app.conf +++ /dev/null @@ -1,15 +0,0 @@ -[supervisord] -nodaemon=true -user=www-data -logfile=/tmp/supervisord.log -loglevel=debug -pidfile=/tmp/supervisord.pid -childlogdir=/tmp - -[program:uwsgi] -command = /usr/local/bin/uwsgi --ini /opt/auth-server-poc/uwsgi.ini -autorestart=true - -[program:nginx] -command=/usr/sbin/nginx -g "daemon off;" -autorestart=true diff --git a/auth-server-poc/config/uwsgi.ini b/auth-server-poc/config/uwsgi.ini deleted file mode 100644 index 492b30c..0000000 --- a/auth-server-poc/config/uwsgi.ini +++ /dev/null @@ -1,13 +0,0 @@ -[uwsgi] -uid=www-data -gid=www-data -chdir = /opt/auth-server-poc/src/ -callable = app -module = app -socket = /tmp/uwsgi.sock -master = true -# uwsgi websockets only allow max 1 process? -processes = 1 -chmod-socket = 666 -enable-threads = true -virtualenv = /opt/auth-server-poc diff --git a/auth-server-poc/docker-compose.yml b/auth-server-poc/docker-compose.yml deleted file mode 100644 index b73532c..0000000 --- a/auth-server-poc/docker-compose.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -version: '3.7' -services: - auth-server-poc: - build: . - ports: - - 8000:80 - volumes: - - ${DOCKER_JWT_PUBKEY_PATH}:/opt/auth-server-poc/cert/ - - ${DOCKER_JWT_HTPASSWD_PATH}:/opt/auth-server-poc/userdb/ - container_name: auth-server-poc diff --git a/auth-server-poc/gen-jwt-cert.sh b/auth-server-poc/gen-jwt-cert.sh deleted file mode 100755 index 8b23990..0000000 --- a/auth-server-poc/gen-jwt-cert.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -cd /opt/auth-server-poc/cert/ -openssl ecparam -genkey -name prime256v1 -noout -out private.pem -openssl ec -in private.pem -pubout -out public.pem -chgrp www-data private.pem -chmod g+r private.pem -killall uwsgi diff --git a/auth-server-poc/requirements.txt b/auth-server-poc/requirements.txt deleted file mode 100644 index fc8fc53..0000000 --- a/auth-server-poc/requirements.txt +++ /dev/null @@ -1,284 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.7 -# To update, run: -# -# pip-compile --generate-hashes requirements.txt -# -aniso8601==9.0.1 \ - --hash=sha256:1d2b7ef82963909e93c4f24ce48d4de9e66009a21bf1c1e1c85bdd0812fe412f \ - --hash=sha256:72e3117667eedf66951bb2d93f4296a56b94b078a8a95905a052611fb3f1b973 - # via - # -r requirements.txt - # flask-restful -cffi==1.15.0 \ - --hash=sha256:00c878c90cb53ccfaae6b8bc18ad05d2036553e6d9d1d9dbcf323bbe83854ca3 \ - --hash=sha256:0104fb5ae2391d46a4cb082abdd5c69ea4eab79d8d44eaaf79f1b1fd806ee4c2 \ - --hash=sha256:06c48159c1abed75c2e721b1715c379fa3200c7784271b3c46df01383b593636 \ - --hash=sha256:0808014eb713677ec1292301ea4c81ad277b6cdf2fdd90fd540af98c0b101d20 \ - --hash=sha256:10dffb601ccfb65262a27233ac273d552ddc4d8ae1bf93b21c94b8511bffe728 \ - --hash=sha256:14cd121ea63ecdae71efa69c15c5543a4b5fbcd0bbe2aad864baca0063cecf27 \ - --hash=sha256:17771976e82e9f94976180f76468546834d22a7cc404b17c22df2a2c81db0c66 \ - --hash=sha256:181dee03b1170ff1969489acf1c26533710231c58f95534e3edac87fff06c443 \ - --hash=sha256:23cfe892bd5dd8941608f93348c0737e369e51c100d03718f108bf1add7bd6d0 \ - --hash=sha256:263cc3d821c4ab2213cbe8cd8b355a7f72a8324577dc865ef98487c1aeee2bc7 \ - --hash=sha256:2756c88cbb94231c7a147402476be2c4df2f6078099a6f4a480d239a8817ae39 \ - --hash=sha256:27c219baf94952ae9d50ec19651a687b826792055353d07648a5695413e0c605 \ - --hash=sha256:2a23af14f408d53d5e6cd4e3d9a24ff9e05906ad574822a10563efcef137979a \ - --hash=sha256:31fb708d9d7c3f49a60f04cf5b119aeefe5644daba1cd2a0fe389b674fd1de37 \ - --hash=sha256:3415c89f9204ee60cd09b235810be700e993e343a408693e80ce7f6a40108029 \ - --hash=sha256:3773c4d81e6e818df2efbc7dd77325ca0dcb688116050fb2b3011218eda36139 \ - --hash=sha256:3b96a311ac60a3f6be21d2572e46ce67f09abcf4d09344c49274eb9e0bf345fc \ - --hash=sha256:3f7d084648d77af029acb79a0ff49a0ad7e9d09057a9bf46596dac9514dc07df \ - --hash=sha256:41d45de54cd277a7878919867c0f08b0cf817605e4eb94093e7516505d3c8d14 \ - --hash=sha256:4238e6dab5d6a8ba812de994bbb0a79bddbdf80994e4ce802b6f6f3142fcc880 \ - --hash=sha256:45db3a33139e9c8f7c09234b5784a5e33d31fd6907800b316decad50af323ff2 \ - --hash=sha256:45e8636704eacc432a206ac7345a5d3d2c62d95a507ec70d62f23cd91770482a \ - --hash=sha256:4958391dbd6249d7ad855b9ca88fae690783a6be9e86df65865058ed81fc860e \ - --hash=sha256:4a306fa632e8f0928956a41fa8e1d6243c71e7eb59ffbd165fc0b41e316b2474 \ - --hash=sha256:57e9ac9ccc3101fac9d6014fba037473e4358ef4e89f8e181f8951a2c0162024 \ - --hash=sha256:59888172256cac5629e60e72e86598027aca6bf01fa2465bdb676d37636573e8 \ - --hash=sha256:5e069f72d497312b24fcc02073d70cb989045d1c91cbd53979366077959933e0 \ - --hash=sha256:64d4ec9f448dfe041705426000cc13e34e6e5bb13736e9fd62e34a0b0c41566e \ - --hash=sha256:6dc2737a3674b3e344847c8686cf29e500584ccad76204efea14f451d4cc669a \ - --hash=sha256:74fdfdbfdc48d3f47148976f49fab3251e550a8720bebc99bf1483f5bfb5db3e \ - --hash=sha256:75e4024375654472cc27e91cbe9eaa08567f7fbdf822638be2814ce059f58032 \ - --hash=sha256:786902fb9ba7433aae840e0ed609f45c7bcd4e225ebb9c753aa39725bb3e6ad6 \ - --hash=sha256:8b6c2ea03845c9f501ed1313e78de148cd3f6cad741a75d43a29b43da27f2e1e \ - --hash=sha256:91d77d2a782be4274da750752bb1650a97bfd8f291022b379bb8e01c66b4e96b \ - --hash=sha256:91ec59c33514b7c7559a6acda53bbfe1b283949c34fe7440bcf917f96ac0723e \ - --hash=sha256:920f0d66a896c2d99f0adbb391f990a84091179542c205fa53ce5787aff87954 \ - --hash=sha256:a5263e363c27b653a90078143adb3d076c1a748ec9ecc78ea2fb916f9b861962 \ - --hash=sha256:abb9a20a72ac4e0fdb50dae135ba5e77880518e742077ced47eb1499e29a443c \ - --hash=sha256:c2051981a968d7de9dd2d7b87bcb9c939c74a34626a6e2f8181455dd49ed69e4 \ - --hash=sha256:c21c9e3896c23007803a875460fb786118f0cdd4434359577ea25eb556e34c55 \ - --hash=sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962 \ - --hash=sha256:d4d692a89c5cf08a8557fdeb329b82e7bf609aadfaed6c0d79f5a449a3c7c023 \ - --hash=sha256:da5db4e883f1ce37f55c667e5c0de439df76ac4cb55964655906306918e7363c \ - --hash=sha256:e7022a66d9b55e93e1a845d8c9eba2a1bebd4966cd8bfc25d9cd07d515b33fa6 \ - --hash=sha256:ef1f279350da2c586a69d32fc8733092fd32cc8ac95139a00377841f59a3f8d8 \ - --hash=sha256:f54a64f8b0c8ff0b64d18aa76675262e1700f3995182267998c31ae974fbc382 \ - --hash=sha256:f5c7150ad32ba43a07c4479f40241756145a1f03b43480e058cfd862bf5041c7 \ - --hash=sha256:f6f824dc3bce0edab5f427efcfb1d63ee75b6fcb7282900ccaf925be84efb0fc \ - --hash=sha256:fd8a250edc26254fe5b33be00402e6d287f562b6a5b2152dec302fa15bb3e997 \ - --hash=sha256:ffaa5c925128e29efbde7301d8ecaf35c8c60ffbcd6a1ffd3a552177c8e5e796 - # via - # -r requirements.txt - # cryptography -click==8.0.3 \ - --hash=sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3 \ - --hash=sha256:410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b - # via - # -r requirements.txt - # flask -cryptography==35.0.0 \ - --hash=sha256:07bb7fbfb5de0980590ddfc7f13081520def06dc9ed214000ad4372fb4e3c7f6 \ - --hash=sha256:18d90f4711bf63e2fb21e8c8e51ed8189438e6b35a6d996201ebd98a26abbbe6 \ - --hash=sha256:1ed82abf16df40a60942a8c211251ae72858b25b7421ce2497c2eb7a1cee817c \ - --hash=sha256:22a38e96118a4ce3b97509443feace1d1011d0571fae81fc3ad35f25ba3ea999 \ - --hash=sha256:2d69645f535f4b2c722cfb07a8eab916265545b3475fdb34e0be2f4ee8b0b15e \ - --hash=sha256:4a2d0e0acc20ede0f06ef7aa58546eee96d2592c00f450c9acb89c5879b61992 \ - --hash=sha256:54b2605e5475944e2213258e0ab8696f4f357a31371e538ef21e8d61c843c28d \ - --hash=sha256:7075b304cd567694dc692ffc9747f3e9cb393cc4aa4fb7b9f3abd6f5c4e43588 \ - --hash=sha256:7b7ceeff114c31f285528ba8b390d3e9cfa2da17b56f11d366769a807f17cbaa \ - --hash=sha256:7eba2cebca600a7806b893cb1d541a6e910afa87e97acf2021a22b32da1df52d \ - --hash=sha256:928185a6d1ccdb816e883f56ebe92e975a262d31cc536429041921f8cb5a62fd \ - --hash=sha256:9933f28f70d0517686bd7de36166dda42094eac49415459d9bdf5e7df3e0086d \ - --hash=sha256:a688ebcd08250eab5bb5bca318cc05a8c66de5e4171a65ca51db6bd753ff8953 \ - --hash=sha256:abb5a361d2585bb95012a19ed9b2c8f412c5d723a9836418fab7aaa0243e67d2 \ - --hash=sha256:c10c797ac89c746e488d2ee92bd4abd593615694ee17b2500578b63cad6b93a8 \ - --hash=sha256:ced40344e811d6abba00295ced98c01aecf0c2de39481792d87af4fa58b7b4d6 \ - --hash=sha256:d57e0cdc1b44b6cdf8af1d01807db06886f10177469312fbde8f44ccbb284bc9 \ - --hash=sha256:d99915d6ab265c22873f1b4d6ea5ef462ef797b4140be4c9d8b179915e0985c6 \ - --hash=sha256:eb80e8a1f91e4b7ef8b33041591e6d89b2b8e122d787e87eeb2b08da71bb16ad \ - --hash=sha256:ebeddd119f526bcf323a89f853afb12e225902a24d29b55fe18dd6fcb2838a76 - # via -r requirements.txt -flask==2.0.2 \ - --hash=sha256:7b2fb8e934ddd50731893bdcdb00fc8c0315916f9fcd50d22c7cc1a95ab634e2 \ - --hash=sha256:cb90f62f1d8e4dc4621f52106613488b5ba826b2e1e10a33eac92f723093ab6a - # via - # -r requirements.txt - # flask-cors - # flask-jwt-extended - # flask-restful -flask-cors==3.0.10 \ - --hash=sha256:74efc975af1194fc7891ff5cd85b0f7478be4f7f59fe158102e91abb72bb4438 \ - --hash=sha256:b60839393f3b84a0f3746f6cdca56c1ad7426aa738b70d6c61375857823181de - # via -r requirements.txt -flask-jwt-extended==4.3.1 \ - --hash=sha256:ad6977b07c54e51c13b5981afc246868b9901a46715d9b9827898bfd916aae88 \ - --hash=sha256:c82c9e505bc96f4a5186de31c05262dbcde6fa10581e9aa46df8f99ca04be2c3 - # via -r requirements.txt -flask-restful==0.3.9 \ - --hash=sha256:4970c49b6488e46c520b325f54833374dc2b98e211f1b272bd4b0c516232afe2 \ - --hash=sha256:ccec650b835d48192138c85329ae03735e6ced58e9b2d9c2146d6c84c06fa53e - # via -r requirements.txt -importlib-metadata==4.8.2 \ - --hash=sha256:53ccfd5c134223e497627b9815d5030edf77d2ed573922f7a0b8f8bb81a1c100 \ - --hash=sha256:75bdec14c397f528724c1bfd9709d660b33a4d2e77387a3358f20b848bb5e5fb - # via - # -r requirements.txt - # click -itsdangerous==2.0.1 \ - --hash=sha256:5174094b9637652bdb841a3029700391451bd092ba3db90600dea710ba28e97c \ - --hash=sha256:9e724d68fc22902a1435351f84c3fb8623f303fffcc566a4cb952df8c572cff0 - # via - # -r requirements.txt - # flask -jinja2==3.0.3 \ - --hash=sha256:077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8 \ - --hash=sha256:611bb273cd68f3b993fabdc4064fc858c5b47a973cb5aa7999ec1ba405c87cd7 - # via - # -r requirements.txt - # flask -markupsafe==2.0.1 \ - --hash=sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298 \ - --hash=sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64 \ - --hash=sha256:0446679737af14f45767963a1a9ef7620189912317d095f2d9ffa183a4d25d2b \ - --hash=sha256:04635854b943835a6ea959e948d19dcd311762c5c0c6e1f0e16ee57022669194 \ - --hash=sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567 \ - --hash=sha256:0955295dd5eec6cb6cc2fe1698f4c6d84af2e92de33fbcac4111913cd100a6ff \ - --hash=sha256:0d4b31cc67ab36e3392bbf3862cfbadac3db12bdd8b02a2731f509ed5b829724 \ - --hash=sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74 \ - --hash=sha256:168cd0a3642de83558a5153c8bd34f175a9a6e7f6dc6384b9655d2697312a646 \ - --hash=sha256:1d609f577dc6e1aa17d746f8bd3c31aa4d258f4070d61b2aa5c4166c1539de35 \ - --hash=sha256:1f2ade76b9903f39aa442b4aadd2177decb66525062db244b35d71d0ee8599b6 \ - --hash=sha256:20dca64a3ef2d6e4d5d615a3fd418ad3bde77a47ec8a23d984a12b5b4c74491a \ - --hash=sha256:2a7d351cbd8cfeb19ca00de495e224dea7e7d919659c2841bbb7f420ad03e2d6 \ - --hash=sha256:2d7d807855b419fc2ed3e631034685db6079889a1f01d5d9dac950f764da3dad \ - --hash=sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26 \ - --hash=sha256:36bc903cbb393720fad60fc28c10de6acf10dc6cc883f3e24ee4012371399a38 \ - --hash=sha256:37205cac2a79194e3750b0af2a5720d95f786a55ce7df90c3af697bfa100eaac \ - --hash=sha256:3c112550557578c26af18a1ccc9e090bfe03832ae994343cfdacd287db6a6ae7 \ - --hash=sha256:3dd007d54ee88b46be476e293f48c85048603f5f516008bee124ddd891398ed6 \ - --hash=sha256:4296f2b1ce8c86a6aea78613c34bb1a672ea0e3de9c6ba08a960efe0b0a09047 \ - --hash=sha256:47ab1e7b91c098ab893b828deafa1203de86d0bc6ab587b160f78fe6c4011f75 \ - --hash=sha256:49e3ceeabbfb9d66c3aef5af3a60cc43b85c33df25ce03d0031a608b0a8b2e3f \ - --hash=sha256:4dc8f9fb58f7364b63fd9f85013b780ef83c11857ae79f2feda41e270468dd9b \ - --hash=sha256:4efca8f86c54b22348a5467704e3fec767b2db12fc39c6d963168ab1d3fc9135 \ - --hash=sha256:53edb4da6925ad13c07b6d26c2a852bd81e364f95301c66e930ab2aef5b5ddd8 \ - --hash=sha256:5855f8438a7d1d458206a2466bf82b0f104a3724bf96a1c781ab731e4201731a \ - --hash=sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a \ - --hash=sha256:5b6d930f030f8ed98e3e6c98ffa0652bdb82601e7a016ec2ab5d7ff23baa78d1 \ - --hash=sha256:5bb28c636d87e840583ee3adeb78172efc47c8b26127267f54a9c0ec251d41a9 \ - --hash=sha256:60bf42e36abfaf9aff1f50f52644b336d4f0a3fd6d8a60ca0d054ac9f713a864 \ - --hash=sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914 \ - --hash=sha256:6300b8454aa6930a24b9618fbb54b5a68135092bc666f7b06901f897fa5c2fee \ - --hash=sha256:63f3268ba69ace99cab4e3e3b5840b03340efed0948ab8f78d2fd87ee5442a4f \ - --hash=sha256:6557b31b5e2c9ddf0de32a691f2312a32f77cd7681d8af66c2692efdbef84c18 \ - --hash=sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8 \ - --hash=sha256:6a7fae0dd14cf60ad5ff42baa2e95727c3d81ded453457771d02b7d2b3f9c0c2 \ - --hash=sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d \ - --hash=sha256:6fcf051089389abe060c9cd7caa212c707e58153afa2c649f00346ce6d260f1b \ - --hash=sha256:7d91275b0245b1da4d4cfa07e0faedd5b0812efc15b702576d103293e252af1b \ - --hash=sha256:89c687013cb1cd489a0f0ac24febe8c7a666e6e221b783e53ac50ebf68e45d86 \ - --hash=sha256:8d206346619592c6200148b01a2142798c989edcb9c896f9ac9722a99d4e77e6 \ - --hash=sha256:905fec760bd2fa1388bb5b489ee8ee5f7291d692638ea5f67982d968366bef9f \ - --hash=sha256:97383d78eb34da7e1fa37dd273c20ad4320929af65d156e35a5e2d89566d9dfb \ - --hash=sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833 \ - --hash=sha256:99df47edb6bda1249d3e80fdabb1dab8c08ef3975f69aed437cb69d0a5de1e28 \ - --hash=sha256:9f02365d4e99430a12647f09b6cc8bab61a6564363f313126f775eb4f6ef798e \ - --hash=sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415 \ - --hash=sha256:ab3ef638ace319fa26553db0624c4699e31a28bb2a835c5faca8f8acf6a5a902 \ - --hash=sha256:aca6377c0cb8a8253e493c6b451565ac77e98c2951c45f913e0b52facdcff83f \ - --hash=sha256:add36cb2dbb8b736611303cd3bfcee00afd96471b09cda130da3581cbdc56a6d \ - --hash=sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9 \ - --hash=sha256:b7f2d075102dc8c794cbde1947378051c4e5180d52d276987b8d28a3bd58c17d \ - --hash=sha256:baa1a4e8f868845af802979fcdbf0bb11f94f1cb7ced4c4b8a351bb60d108145 \ - --hash=sha256:be98f628055368795d818ebf93da628541e10b75b41c559fdf36d104c5787066 \ - --hash=sha256:bf5d821ffabf0ef3533c39c518f3357b171a1651c1ff6827325e4489b0e46c3c \ - --hash=sha256:c47adbc92fc1bb2b3274c4b3a43ae0e4573d9fbff4f54cd484555edbf030baf1 \ - --hash=sha256:cdfba22ea2f0029c9261a4bd07e830a8da012291fbe44dc794e488b6c9bb353a \ - --hash=sha256:d6c7ebd4e944c85e2c3421e612a7057a2f48d478d79e61800d81468a8d842207 \ - --hash=sha256:d7f9850398e85aba693bb640262d3611788b1f29a79f0c93c565694658f4071f \ - --hash=sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53 \ - --hash=sha256:deb993cacb280823246a026e3b2d81c493c53de6acfd5e6bfe31ab3402bb37dd \ - --hash=sha256:e0f138900af21926a02425cf736db95be9f4af72ba1bb21453432a07f6082134 \ - --hash=sha256:e9936f0b261d4df76ad22f8fee3ae83b60d7c3e871292cd42f40b81b70afae85 \ - --hash=sha256:f0567c4dc99f264f49fe27da5f735f414c4e7e7dd850cfd8e69f0862d7c74ea9 \ - --hash=sha256:f5653a225f31e113b152e56f154ccbe59eeb1c7487b39b9d9f9cdb58e6c79dc5 \ - --hash=sha256:f826e31d18b516f653fe296d967d700fddad5901ae07c622bb3705955e1faa94 \ - --hash=sha256:f8ba0e8349a38d3001fae7eadded3f6606f0da5d748ee53cc1dab1d6527b9509 \ - --hash=sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51 \ - --hash=sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872 - # via - # -r requirements.txt - # jinja2 -pycparser==2.21 \ - --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ - --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 - # via - # -r requirements.txt - # cffi -pyjwt==2.3.0 \ - --hash=sha256:b888b4d56f06f6dcd777210c334e69c737be74755d3e5e9ee3fe67dc18a0ee41 \ - --hash=sha256:e0c4bb8d9f0af0c7f5b1ec4c5036309617d03d56932877f2f7a0beeb5318322f - # via - # -r requirements.txt - # flask-jwt-extended -pytz==2021.3 \ - --hash=sha256:3672058bc3453457b622aab7a1c3bfd5ab0bdae451512f6cf25f64ed37f5b87c \ - --hash=sha256:acad2d8b20a1af07d4e4c9d2e9285c5ed9104354062f275f3fcd88dcef4f1326 - # via - # -r requirements.txt - # flask-restful -six==1.16.0 \ - --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ - --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 - # via - # -r requirements.txt - # flask-cors - # flask-restful -typing-extensions==3.10.0.2 \ - --hash=sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e \ - --hash=sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7 \ - --hash=sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34 - # via - # -r requirements.txt - # importlib-metadata -werkzeug==2.0.2 \ - --hash=sha256:63d3dc1cf60e7b7e35e97fa9861f7397283b75d765afcaefd993d6046899de8f \ - --hash=sha256:aa2bb6fc8dee8d6c504c0ac1e7f5f7dc5810a9903e793b6f715a9f015bdadb9a - # via - # -r requirements.txt - # flask - # flask-jwt-extended -zipp==3.6.0 \ - --hash=sha256:71c644c5369f4a6e07636f0aa966270449561fcea2e3d6747b8d23efaa9d7832 \ - --hash=sha256:9fe5ea21568a0a70e50f273397638d39b03353731e6cbbb3fd8502a33fec40bc - # via - # -r requirements.txt - # importlib-metadata -pyyaml==6.0 \ - --hash=sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293 \ - --hash=sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b \ - --hash=sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57 \ - --hash=sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b \ - --hash=sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4 \ - --hash=sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07 \ - --hash=sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba \ - --hash=sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9 \ - --hash=sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287 \ - --hash=sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513 \ - --hash=sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0 \ - --hash=sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0 \ - --hash=sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92 \ - --hash=sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f \ - --hash=sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2 \ - --hash=sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc \ - --hash=sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c \ - --hash=sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86 \ - --hash=sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4 \ - --hash=sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c \ - --hash=sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34 \ - --hash=sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b \ - --hash=sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c \ - --hash=sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb \ - --hash=sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737 \ - --hash=sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3 \ - --hash=sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d \ - --hash=sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53 \ - --hash=sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78 \ - --hash=sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803 \ - --hash=sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a \ - --hash=sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174 \ - --hash=sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5 diff --git a/auth-server-poc/setup.sh b/auth-server-poc/setup.sh deleted file mode 100755 index 77aee9a..0000000 --- a/auth-server-poc/setup.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -set -e -set -x - -export DEBIAN_FRONTEND noninteractive - -# /bin/sed -i s/deb.debian.org/ftp.se.debian.org/g /etc/apt/sources.list - -apt-get update && \ - apt-get -y dist-upgrade && \ - apt-get install -y \ - git \ - python3-venv \ - python3-pip \ - python3-yaml \ - iputils-ping \ - procps \ - bind9-host \ - netcat-openbsd \ - net-tools \ - curl \ - netcat \ - nginx \ - supervisor \ - libssl-dev \ - apache2-utils \ - && apt-get clean - -pip3 install uwsgi - -# Start venv -python3 -m venv /opt/auth-server-poc -cd /opt/auth-server-poc -source bin/activate - -/opt/auth-server-poc/bin/pip install -U pip - -python3 -m pip install -r requirements.txt - -# Temporary for testing new branch -#cd /opt/cnaas/venv/cnaas-nms/ -#git remote update -#git fetch -#git checkout --track origin/feature.websocket -#python3 -m pip install -r requirements.txt - -#rm -rf /var/lib/apt/lists/* - - diff --git a/auth-server-poc/src/app.py b/auth-server-poc/src/app.py deleted file mode 100644 index 37a7030..0000000 --- a/auth-server-poc/src/app.py +++ /dev/null @@ -1,54 +0,0 @@ -from flask import Flask, request -from flask_restful import Api, Resource -from flask_jwt_extended import create_access_token, JWTManager -from flask_cors import CORS - -import authn - -app = Flask(__name__) -cors = CORS( - app, - resources={r"/api/*": {"origins": "*"}}, - expose_headers=["Content-Type", "Authorization", "X-Total-Count"], -) -api = Api(app, prefix="/api/v1.0") -jwt = JWTManager(app) - -PEM_PRIVATE = "/opt/auth-server-poc/cert/private.pem" -PEM_PUBLIC = "/opt/auth-server-poc/cert/public.pem" -USERDB_YAML = "/opt/auth-server-poc/userdb/userdb.yaml" - -app.config["JWT_PRIVATE_KEY"] = open(PEM_PRIVATE).read() -app.config["JWT_PUBLIC_KEY"] = open(PEM_PUBLIC).read() -app.config["JWT_ALGORITHM"] = "ES256" -app.config["JWT_IDENTITY_CLAIM"] = "sub" -app.config["JWT_ACCESS_TOKEN_EXPIRES"] = False - - -class AuthApi(Resource): - def post(self): - - identity = request.environ.get("REMOTE_USER") - db = authn.UserDB(USERDB_YAML) - additional_claims = { - "type": "access", - "read": db.read_perms(identity), - "write": db.write_perms(identity), - } - - access_token = create_access_token( - identity=identity, - additional_claims=additional_claims, - ) - - return {"access_token": access_token}, 200 - - -@app.route("/") -def index(): - return "<p>Username: {}</p><p>Auth type: {}</p>".format( - request.environ.get("REMOTE_USER"), request.environ.get("AUTH_TYPE") - ) - - -api.add_resource(AuthApi, "/auth") diff --git a/auth-server-poc/src/authn.py b/auth-server-poc/src/authn.py deleted file mode 100755 index 8b32cdc..0000000 --- a/auth-server-poc/src/authn.py +++ /dev/null @@ -1,97 +0,0 @@ -#! /usr/bin/env python3 - -import yaml - - -class Authz: - def __init__(self, org, perms): - self._org = org - self._perms = perms - - def dump(self): - return "{}: {}".format(self._org, self._perms) - - def read_p(self): - return "r" in self._perms - - def write_p(self): - return "w" in self._perms - - -class User: - def __init__(self, username, authz): - self._username = username - self._authz = {} - for org, perms in authz.items(): - self._authz[org] = Authz(org, perms) - - def dump(self): - return [ - "{}: {}".format(self._username, auth.dump()) - for auth in self._authz.values() - ] - - def orgnames(self): - return [x for x in self._authz.keys()] - - def read_perms(self): - acc = [] - for k, v in self._authz.items(): - if v.read_p(): - acc.append(k) - return acc - - def write_perms(self): - acc = [] - for k, v in self._authz.items(): - if v.write_p(): - acc.append(k) - return acc - - -class UserDB: - def __init__(self, yamlfile): - self._users = {} - for u, d in yaml.safe_load(open(yamlfile)).items(): - self._users[u] = User(u, d["authz"]) - - def dump(self): - return [u.dump() for u in self._users.values()] - - def orgs_for_user(self, username): - return self._users.get(username).orgnames() - - def read_perms(self, username): - user = self._users.get(username) - if not user: - return None - return user.read_perms() - - def write_perms(self, username): - user = self._users.get(username) - if not user: - return None - return user.write_perms() - - -def self_test(): - db = UserDB("userdb.yaml") - print(db.dump()) - - orgs = db.orgs_for_user("user3") - assert "sunet.se" in orgs - assert "su.se" in orgs - assert len(orgs) == 2 - - rp = db.read_perms("user3", "pw3") - assert len(rp) == 2 - assert "sunet.se" in rp - assert "su.se" in rp - - wp = db.write_perms("user3", "pw3") - assert len(wp) == 1 - assert "sunet.se" in wp - - -if __name__ == "__main__": - self_test() diff --git a/auth-server-poc/userdb.yaml b/auth-server-poc/userdb.yaml deleted file mode 100644 index 937328c..0000000 --- a/auth-server-poc/userdb.yaml +++ /dev/null @@ -1,29 +0,0 @@ -usr: - authz: - sunet.se: rw - su.se: rw - kth.se: rw - -user1: - authz: - sunet.se: r - su.se: r - kth.se: r - -user2: - authz: - sunet.se: w - su.se: w - kth.se: w - -user3: - authz: - sunet.se: rw - su.se: rw - kth.se: rw - -user4: - authz: - sunet.se: rw - su.se: r - kth.se: w diff --git a/docker/collector/Dockerfile b/data/collector_container/Dockerfile index 099bc0a..e02a5d2 100644 --- a/docker/collector/Dockerfile +++ b/data/collector_container/Dockerfile @@ -1,15 +1,12 @@ FROM debian:bullseye-20221024-slim@sha256:76cdda8fe5eb597ef5e712e4c9a9f5f1fb119e69f353daaa7bd6d0f6e66e541d -# FROM debian:bullseye -# ENV DEBIAN_FRONTEND noninteractive -# RUN apt-get update -# RUN apt-get install -y git supervisor emacs-nox virtualenv procps +EXPOSE 8000 -COPY ./requirements.txt /opt/collector/requirements.txt +COPY ./requirements.txt /app/requirements.txt RUN apt-get update \ && apt-get install -y python3 python3-pip \ - && pip3 install -r /opt/collector/requirements.txt \ + && pip3 install -r /app/requirements.txt \ && apt-get remove -y \ gcc \ curl \ @@ -26,23 +23,13 @@ RUN find / -xdev -perm /6000 -type f -exec chmod a-s {} \; || true # Add user RUN useradd collector -u 1500 -s /usr/sbin/nologin -COPY ./src /opt/collector/src +COPY ./src /app/src -WORKDIR /opt/collector/ +WORKDIR /app/ USER collector ENTRYPOINT ["uvicorn", "src.collector.main:app", "--host", "0.0.0.0", "--workers", "1", "--header", "server:collector"] -# ENTRYPOINT ["sleep", "300"] - -# RUN git clone https://git.sunet.se/soc_collector.git /opt/collector -# WORKDIR /opt/collector/ -# COPY setup.sh /opt/collector/ -# COPY supervisord.conf /etc/supervisor/ - -# RUN /opt/collector/setup.sh -# ENTRYPOINT supervisord -c /etc/supervisor/supervisord.conf - diff --git a/example_data_1.json b/data/example_data_1.json index 69f5d85..69f5d85 100644 --- a/example_data_1.json +++ b/data/example_data_1.json diff --git a/example_data_2.json b/data/example_data_3.json index 11c7e61..44d483b 100644 --- a/example_data_2.json +++ b/data/example_data_3.json @@ -1,14 +1,14 @@ { "document_version": 1, - "ip": "192.0.2.20", - "port": 80, + "ip": "192.0.2.28", + "port": 111, "whois_description": "SOMENET", "asn": "AS65001", "asn_country_code": "SE", - "ptr": "host11.test.soc.sunet.se", + "ptr": "host111.test.soc.sunet.se", "abuse_mail": "abuse@test.soc.sunet.se", "domain": "sunet.se", - "timestamp": "2021-06-30T10:00:00Z", + "timestamp": "2021-06-30T15:00:00Z", "display_name": "VMware ESXi 6.7.0 build-17700523", "description": "VMware ESXi is an enterprise-class, type-1 hypervisor developed by VMware for deploying and serving virtual computers. As a type-1 hypervisor, ESXi is not a software application that is installed on an operating system; instead, it includes and integrates vital OS components, such as a kernel.", "custom_data": { diff --git a/data/example_data_3_replace_test.json b/data/example_data_3_replace_test.json new file mode 100644 index 0000000..31cc64d --- /dev/null +++ b/data/example_data_3_replace_test.json @@ -0,0 +1,52 @@ +{ + "_id": "6370498050845fac09e0fc01", + "document_version": 2, + "ip": "192.0.2.28", + "port": 112, + "whois_description": "SOMENET", + "asn": "AS65001", + "asn_country_code": "SE", + "ptr": "host111.test.soc.sunet.se", + "abuse_mail": "abuse@test.soc.sunet.se", + "domain": "sunet.se", + "timestamp": "2021-06-30T15:00:00Z", + "display_name": "VMware ESXi 6.7.0 build-17700523", + "description": "VMware ESXi is an enterprise-class, type-1 hypervisor developed by VMware for deploying and serving virtual computers. As a type-1 hypervisor, ESXi is not a software application that is installed on an operating system; instead, it includes and integrates vital OS components, such as a kernel.", + "custom_data": { + "subject_cn": { + "data": "VMware ESXi", + "display_name": "Subject Common Name" + }, + "end_of_general_support": { + "data": true, + "display_name": "End of general support", + "description": "Is the software currently supported?" + } + }, + "result": { + "cve_2019_0001": { + "display_name": "CVE-2019-0001", + "vulnerable": false + }, + "cve_2015_0002": { + "display_name": "CVE-2015-0002", + "vulnerable": false, + "description": "There is a use of insufficiently random values vulnerability. An unauthenticated, remote attacker can guess information by a large number of attempts. Successful exploitation may cause information leak." + }, + "cve_2015_0003": { + "display_name": "CVE-2015-0003", + "vulnerable": true, + "reliability": 2, + "description": "A carefully crafted request body can cause a read to a random memory area which could cause the process to crash." + }, + "cve_2015_0004": { + "display_name": "CVE-2015-0004", + "vulnerable": false + }, + "cve_2015_0005": { + "display_name": "CVE-2015-0005", + "vulnerable": true, + "reliability": 4 + } + } +} diff --git a/data/init-mongodb.js b/data/init-mongodb.js new file mode 100644 index 0000000..4b64674 --- /dev/null +++ b/data/init-mongodb.js @@ -0,0 +1,30 @@ + +// To help improve our products, anonymous usage data is collected and sent to MongoDB periodically (https://www.mongodb.com/legal/privacy-policy). +// You can opt-out by running the disableTelemetry() command. +disableTelemetry() + +// Create the DB by inserting some data +db.v0.insertOne({init_key: "init_data"}) + +// Create user +db.createUser( + { + user: "REPLACE_USERNAME", + pwd: "REPLACE_PASSWORD", + roles: [ + { + role: "readWrite", + db: "production" + } + ] + } +) + +// Delete the init data +db.v0.deleteOne({init_key: "init_data"}) + +// Disable the ad about monitoring +db.disableFreeMonitoring() + +// Restart server now +db.shutdownServer() diff --git a/data/mongodb_container/Dockerfile b/data/mongodb_container/Dockerfile new file mode 100644 index 0000000..32ee43b --- /dev/null +++ b/data/mongodb_container/Dockerfile @@ -0,0 +1,27 @@ +FROM debian:bullseye-20221024-slim@sha256:76cdda8fe5eb597ef5e712e4c9a9f5f1fb119e69f353daaa7bd6d0f6e66e541d + +EXPOSE 27017 + +RUN apt-get update && apt-get install curl -y \ + && curl -fsSL https://pgp.mongodb.com/server-6.0.pub | tee /usr/share/keyrings/mongodb-archive-keyring.gpg > /dev/null \ + && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/mongodb-archive-keyring.gpg] http://repo.mongodb.org/apt/debian bullseye/mongodb-org/6.0 main" | tee /etc/apt/sources.list.d/mongodb-org-6.0.list \ + && apt-get update \ + && apt-get install \ + mongodb-org -y \ + && apt-get remove -y \ + wget \ + curl \ + && apt-get autoremove -y \ + && apt-get clean + +# Remove setuid and setgid +RUN find / -xdev -perm /6000 -type f -exec chmod a-s {} \; || true + +COPY ./data/mongodb_entrypoint.sh /mongodb_entrypoint.sh +COPY ./data/init-mongodb.js /init-mongodb.js + +USER mongodb + +WORKDIR /data/db + +ENTRYPOINT ["bash", "/mongodb_entrypoint.sh"] diff --git a/data/mongodb_entrypoint.sh b/data/mongodb_entrypoint.sh new file mode 100755 index 0000000..3db507a --- /dev/null +++ b/data/mongodb_entrypoint.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +if [ ! -f /data/db/user_exist ] +then + # Another port to prevent 'address already in use' + /usr/bin/mongod --port 27015 --nounixsocket & + sleep 1 + cp /init-mongodb.js /data/db/init-mongodb.js + sed -i "s/REPLACE_USERNAME/$MONGODB_USERNAME/g" /data/db/init-mongodb.js + sed -i "s/REPLACE_PASSWORD/$MONGODB_PASSWORD/g" /data/db/init-mongodb.js + + /usr/bin/mongosh localhost:27015/production /data/db/init-mongodb.js + sleep 1 # Allow DB to shutdown + /usr/bin/touch /data/db/user_exist + rm /data/db/init-mongodb.js +fi + +# Startup normally now with our user +exec /usr/bin/mongod --nounixsocket --bind_ip_all --auth @@ -1,20 +1,74 @@ #!/bin/bash - echo "Checking package" -mypy --strict --namespace-packages --ignore-missing-imports --cache-dir=/dev/null src/collector/*.py # || exit 1 +mypy --strict --namespace-packages --ignore-missing-imports --cache-dir=/tmp/ src/collector/*.py # || exit 1 black --line-length 120 src/collector/*.py # || exit 1 pylint --max-line-length 120 src/collector/*.py # || exit 1 +sudo chown -R $USER data/mongodb_data +docker-compose -f docker-compose.yml build + +sudo chown -R 101 data/mongodb_data +docker-compose -f docker-compose.yml up -d + +sleep 2 + + +echo +echo +curl --data-binary @data/example_data_3.json http://localhost:8000/sc/v0 +echo +echo + +curl -X DELETE http://localhost:8000/sc/v0/63702570e004d2b0b2254d27 +echo +echo +curl -X DELETE http://localhost:8000/sc/v0/63702570e004d2b0b2254d27 +echo +echo + +curl -d '{"search": {"port": {"$lt": 4}}}' -H 'Content-Type: application/json' http://localhost:8000/sc/v0/search +echo +echo +curl -d '{"search": {"port": 112}}' -H 'Content-Type: application/json' http://localhost:8000/sc/v0/search +echo +echo +curl -d '{"search": {"port": {"$gt": 4}}}' -H 'Content-Type: application/json' http://localhost:8000/sc/v0/search +echo +echo +curl -d '{"search": {"port": 111}}' -H 'Content-Type: application/json' http://localhost:8000/sc/v0/search +echo +echo +curl -d '{"search": {"port": {"sdfsf": 7}}}' -H 'Content-Type: application/json' http://localhost:8000/sc/v0/search +echo +echo +curl -d '{"search": {"port": {"$sdfsf": 7}}}' -H 'Content-Type: application/json' http://localhost:8000/sc/v0/search +echo +echo +curl -d '{"search": {"portfdv": {"$asa": 7}}}' -H 'Content-Type: application/json' http://localhost:8000/sc/v0/search +echo +echo + +echo +echo +curl -X PUT --data-binary @data/example_data_3_replace_test.json http://localhost:8000/sc/v0 + + +# bash quickstart.sh -b || exit 1 +# sleep 3 +# JWT=$(curl -k http://localhost:8000/api/v1.0/auth -X POST -p -u usr:pwd | jq -r .access_token) || exit 1 +# curl -k --data-binary @example_data_1.json -H "Authorization: Bearer $JWT" https://localhost:1443/sc/v0/add || exit 1 +# curl -k --data-binary @example_data_3.json -H "Authorization: Bearer $JWT" https://localhost:1443/sc/v0/add || exit 1 +# sleep 1 +# curl -k -H "Authorization: Bearer $JWT" https://localhost:1443/sc/v0/get | json_pp -json_opt utf8,pretty || exit 1 + +# curl -k -H "Authorization: Bearer $JWT" https://localhost:1443/sc/v0/get?port=111 || exit 1 -bash quickstart.sh -b || exit 1 -sleep 3 -JWT=$(curl -k http://localhost:8000/api/v1.0/auth -X POST -p -u usr:pwd | jq -r .access_token) || exit 1 -curl -k --data-binary @example_data_1.json -H "Authorization: Bearer $JWT" https://localhost:1443/sc/v0/add || exit 1 -exit 0 +# echo "OK" +# exit 0 -echo "Checking tests" +#echo "Checking tests" #mypy --strict --namespace-packages --ignore-missing-imports --cache-dir=/dev/null tests/*.py || exit 1 #black --line-length 120 tests/*.py || exit 1 #pylint --max-line-length 120 tests/*.py || exit 1 @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -# Usage: -# ./do-as <username> <last part of URL> [<any additional curl args>] -# Example: -# ./do-as user1 get -# ./do-as user3 delete/1642091653617 -X DELETE - -set -e - -USER=$1 -CMD=$2 -shift -shift - -JWT=$(curl http://localhost:8000/api/v1.0/auth -X POST -p -u "$USER:pwd" | jq -r .access_token) -curl -H "Authorization: Bearer $JWT" https://localhost:1443/sc/v0/$CMD "$@" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..729a1ec --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,29 @@ +version: '3.9' +services: + collector: + build: + context: . + dockerfile: ./data/collector_container/Dockerfile + ports: + - "8000:8000" + environment: + - MONGODB_USERNAME + - MONGODB_PASSWORD + - MONGODB_COLLECTION + depends_on: + - mongodb + read_only: true + + mongodb: + build: + context: . + dockerfile: ./data/mongodb_container/Dockerfile + ports: + - "27017:27017" + environment: + - MONGODB_USERNAME + - MONGODB_PASSWORD + volumes: + - ./data/mongodb_data:/data/db + read_only: true + diff --git a/docker/collector/_dev_dockerfile_dev b/docker/collector/_dev_dockerfile_dev deleted file mode 100644 index 15a6ebe..0000000 --- a/docker/collector/_dev_dockerfile_dev +++ /dev/null @@ -1,22 +0,0 @@ -FROM debian:bullseye-20221024-slim@sha256:76cdda8fe5eb597ef5e712e4c9a9f5f1fb119e69f353daaa7bd6d0f6e66e541d - -# FROM debian:bullseye - -ENV DEBIAN_FRONTEND noninteractive - -RUN apt update -RUN apt install -y git supervisor emacs-nox virtualenv procps -RUN apt clean - -WORKDIR /opt/ - -COPY . /opt/collector - -WORKDIR /opt/collector/ - -COPY docker/collector/setup.sh /opt/collector/ -COPY docker/collector/supervisord.conf /etc/supervisor/ - -RUN /opt/collector/setup.sh - -ENTRYPOINT supervisord -c /etc/supervisor/supervisord.conf diff --git a/docker/collector/setup.sh b/docker/collector/setup.sh deleted file mode 100755 index fce6b42..0000000 --- a/docker/collector/setup.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh - -if [ ! -d /opt/certs/ ]; then - echo "Creating certs/ directory." - mkdir /opt/certs/ -fi - -cd /opt/collector/ - -if [ -d /opt/collector/venv/ ]; then - rm -rf /opt/collector/venv/ -fi - -virtualenv venv -. venv/bin/activate -pip3 install -r requirements.txt diff --git a/docker/collector/supervisord.conf b/docker/collector/supervisord.conf deleted file mode 100644 index 2a2f5ca..0000000 --- a/docker/collector/supervisord.conf +++ /dev/null @@ -1,10 +0,0 @@ -[supervisord] -nodaemon=true - -[program:uvicorn] -directory = /opt/collector/src/ -command = /opt/collector/venv/bin/uvicorn --log-level debug --proxy-headers --host 0.0.0.0 --port 8000 main:app -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes=0 diff --git a/docker/couchdb/10-single-node.ini b/docker/couchdb/10-single-node.ini deleted file mode 100644 index c85b081..0000000 --- a/docker/couchdb/10-single-node.ini +++ /dev/null @@ -1,2 +0,0 @@ -[couchdb] -single_node=true
\ No newline at end of file diff --git a/docker/couchdb/Dockerfile b/docker/couchdb/Dockerfile deleted file mode 100644 index ce3d5b2..0000000 --- a/docker/couchdb/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM couchdb:latest - -COPY 10-single-node.ini /opt/couchdb/etc/local.d/10-single-node.ini - -EXPOSE 5984 diff --git a/docker/docker-compose-dev.yaml b/docker/docker-compose-dev.yaml deleted file mode 100644 index ead32e0..0000000 --- a/docker/docker-compose-dev.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Compose file for local development -version: '3.7' -services: - nginx: - build: ./nginx - ports: - - 1443:443 - depends_on: - - collector - volumes: - - certs:/etc/ssl/collector/ - - collector: - build: - context: .. - dockerfile: docker/collector/Dockerfile-dev - environment: - - COUCHDB_USER - - COUCHDB_PASSWORD - - COUCHDB_NAME - - COUCHDB_HOSTNAME - - JWT_PUBKEY_PATH - volumes: - - ${DOCKER_JWT_PUBKEY_PATH}:/opt/certs/:ro - depends_on: - - couchdb - - couchdb: - build: ./couchdb/ - ports: - - "5984:5984" - environment: - - COUCHDB_USER - - COUCHDB_PASSWORD - volumes: - - type: volume - source: couchdb-data - target: /opt/couchdb/data - -volumes: - couchdb-data: - certs: diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml deleted file mode 100644 index 119d3a9..0000000 --- a/docker/docker-compose.yaml +++ /dev/null @@ -1,42 +0,0 @@ -version: '3.7' -services: - nginx: - build: ./nginx - ports: - - 1443:443 - depends_on: - - collector - volumes: - - certs:/etc/ssl/collector/ - - collector: - build: - context: .. - dockerfile: docker/collector/Dockerfile - environment: - - COUCHDB_USER - - COUCHDB_PASSWORD - - COUCHDB_NAME - - COUCHDB_HOSTNAME - - JWT_PUBKEY_PATH - volumes: - - ${DOCKER_JWT_PUBKEY_PATH}:/opt/certs/:ro - depends_on: - - couchdb - - couchdb: - build: ./couchdb/ - ports: - - "5984:5984" - environment: - - COUCHDB_USER - - COUCHDB_PASSWORD - volumes: - - type: volume - source: couchdb-data - target: /opt/couchdb/data - -volumes: - couchdb-data: - external: false - certs: diff --git a/docker/nginx/Dockerfile b/docker/nginx/Dockerfile deleted file mode 100644 index c44c3cf..0000000 --- a/docker/nginx/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM nginx - -COPY certs/* /etc/ssl/collector/ -COPY default.conf /etc/nginx/conf.d/default.conf diff --git a/docker/nginx/certs/soc_collector-key.pem b/docker/nginx/certs/soc_collector-key.pem deleted file mode 100644 index 91b47df..0000000 --- a/docker/nginx/certs/soc_collector-key.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCnvrRD65+Ht4yi -hyA4gVZ/fAPycpkE3gV4j/hOPKQ2HQ4/Hgu4gLvYbXWVSbjoVT0VkioEtpYQMA4E -ZRwznV28pe6cQS1BdG1TkJpT4jTBa0JYRWD0Sk6uADtWtduEYIN2Zn/XRwc358Y7 -ios32prAJzer6OaILhFf4zEdC8K7z/9pWxH2nTsgAzFccReN+62AkaVfWmSCHk5I -PpsoU076t18jHFTsfvl1r+u2fTQgokC9qFJ7zk62m3NhJh6D4n3J/KzQi4ZvdU7Z -f5XHScdfcq2RoaYN55wS6zN5aSvcdksJMsFj/gOS0Wy5sMWS9MUf25merlqkwyZn -iQqd4DwlAgMBAAECggEBAItSOSawbK+sI4JZ+Mj1o4H+3shfpfgE70ZTTUit1TWc -rFh/+/wh7+KyBxcXRByzryhbXcrMW5bWUs4TrHiyG0i0CJp/B7d1m9fsIkHJf0em -blQiS4Sasv6R8rbyFIC+KHIOb9+9fqFQP7T0PO0HA8DU0chWiCJTINsjN4eIXRTf -NAzjL5xOnW2bIJTz1ZOWxZtVgSjbecKSw3PEy04RVmjEe2jcY0AqaG+IOvjNHgwZ -PR7JVAeU8ReyeEaNHOnJTCtwwTvlxtU3dhwcCDJPlZjjzZ0tbovzFs051o5yJEp7 -/q0he1Oj9TN4UzqfZoq8ApaFRHCBaXPps97bQiaKo4ECgYEA2OfUfDwlVYLbNphb -g8NKjTImhSrBGFXREOOCIE9HirpABR4FCXon0qKDaq3IkMenfuF9Wb7qb4cOA+g6 -xwYNxZAtXAzQ+AqjFiZOzJlyaYhToqKE5Pxific+x1VSNSk8DDkbS1IrTQU+hN0q -y++UDiRm5GQnx3wzJ1UgziD4P+MCgYEAxfqRlnu344ZQssBgU35UmdMupDOPpzwL -l/auo2TgQEqS35nSwqibSZ5rqh2WR7mkLTQuCyt3GFDyWHqMqUFzi0eFDzzZGEZt -omjCJfhNUjZEoxWp3iVg1WDVpD0IIGeGWunTi8rw7AVO2SyAZrPrtr6d9/K1taxP -ZTiQHSzuQlcCgYAJUDsBcpuvxkSfXX9gcvw6f9LDmQhdgSHO9dXaiUzrGgAxuA5D -T+lx0+SMqhWYkdoRSqFVPytVypjBdjE/5nSk7QHll+9JFzvVcaOkiVouSdo9e3j1 -VBjujcTWTkWPXsvjQnh3jXmqfvUmQ1DZHNpgmROJ7vr+R8jygWc9MpE2PQKBgQC6 -GcjjIRcBvI99CBNESPtzwa0VUxaVSH9JcOxG0ZtvM+qOX8VbkCyw7kccmUVb1Oi8 -SVGC1G/WHrlAbKpurATWBvvQEA2uoP2L7leaY6cjQboRZ1rPZGl7CtSo12odM9N7 -AFQIE1OXOYId6ZQldrl7hgFuQuuGhBv59Uqa8lJhbwKBgEjjbYo+lSH1KeyK6goG -lA944pJq8foYHRI23DPyl3E2ARRQgI2j5maJ80PV/1ECS3EaIQOqDaoMaMSTW2P2 -ZEHD8GKd+hoLot2iMjDXEKX0A3GsFR6380qSamJ0sv0dz5sANMB1KSNjdLZ/ZooN -aEaBRM8DuT+ttKylj4B+uNDn ------END PRIVATE KEY----- diff --git a/docker/nginx/certs/soc_collector.pem b/docker/nginx/certs/soc_collector.pem deleted file mode 100644 index 4c9afc8..0000000 --- a/docker/nginx/certs/soc_collector.pem +++ /dev/null @@ -1,26 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIEWzCCAsOgAwIBAgIQFFGAJROQSltNZhThNIJnLzANBgkqhkiG9w0BAQsFADCB -hzEeMBwGA1UEChMVbWtjZXJ0IGRldmVsb3BtZW50IENBMS4wLAYDVQQLDCVraG5A -S3Jpc3RvZmVycy1NQlAgKEtyaXN0b2ZlciBIYWxsaW4pMTUwMwYDVQQDDCxta2Nl -cnQga2huQEtyaXN0b2ZlcnMtTUJQIChLcmlzdG9mZXIgSGFsbGluKTAeFw0yMjAy -MDQwODIyMjNaFw0yNDA1MDQwNzIyMjNaMGcxJzAlBgNVBAoTHm1rY2VydCBkZXZl -bG9wbWVudCBjZXJ0aWZpY2F0ZTE8MDoGA1UECwwza2huQEtyaXN0b2ZlcnMtTWFj -Qm9vay1Qcm8ubG9jYWwgKEtyaXN0b2ZlciBIYWxsaW4pMIIBIjANBgkqhkiG9w0B -AQEFAAOCAQ8AMIIBCgKCAQEAp760Q+ufh7eMoocgOIFWf3wD8nKZBN4FeI/4Tjyk -Nh0OPx4LuIC72G11lUm46FU9FZIqBLaWEDAOBGUcM51dvKXunEEtQXRtU5CaU+I0 -wWtCWEVg9EpOrgA7VrXbhGCDdmZ/10cHN+fGO4qLN9qawCc3q+jmiC4RX+MxHQvC -u8//aVsR9p07IAMxXHEXjfutgJGlX1pkgh5OSD6bKFNO+rdfIxxU7H75da/rtn00 -IKJAvahSe85OtptzYSYeg+J9yfys0IuGb3VO2X+Vx0nHX3KtkaGmDeecEuszeWkr -3HZLCTLBY/4DktFsubDFkvTFH9uZnq5apMMmZ4kKneA8JQIDAQABo2IwYDAOBgNV -HQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwHwYDVR0jBBgwFoAUzcpu -HwNfeGSOBcv5b4qtfePeuDMwGAYDVR0RBBEwD4INc29jX2NvbGxlY3RvcjANBgkq -hkiG9w0BAQsFAAOCAYEAx4b3jIymleUw+Bt/zUnKllmSOA581MrWT9tdLu/uC7hj -KM2YidbzqmI7vnSCIp9YZGNZIK+7/lWopFWsIsqsd4BOaVuodZ/WK3e22+7h/yzd -imde2bv0mSNHxFeaU4x++MNpjKStziC/UwD2PqNqoACXOIcXMqFL/esluKW1APKZ -xTSxBcHmesNeFQL8wkK3HTYm7TYuKx/gcVnySvxnaPqJCRgF6jVtua83RTQXpS1b -i8NW1sfF5La1/Biy0rnCIbOvQwWwPcou2hRQPMIS68dTFu0fJkhrrI/BVQj9doJp -8WMCM1805uymn7OMEkJU4n3g7iicNWBnFr8C0UXvSVEkh35AG4J2CxMyixm83NaY -1w0hDGcan0XPVTElRhXjmPO1aWQFwfUmfQm94GfJD4biGdAAhSU+ejuiGxCIS79e -G4Av3Ax5ixqqnGboW4QSK04brqfBASLcg10kv60QEDp/Rj+VCMPnwa21FU0V1hBT -8jMbIUkUrNlHLwGLZLPW ------END CERTIFICATE----- diff --git a/docker/nginx/default.conf b/docker/nginx/default.conf deleted file mode 100644 index 35eb2a2..0000000 --- a/docker/nginx/default.conf +++ /dev/null @@ -1,14 +0,0 @@ -server { - listen 443 ssl; - server_name _; - - ssl_certificate /etc/ssl/collector/soc_collector.pem; - ssl_certificate_key /etc/ssl/collector/soc_collector-key.pem; - - access_log /var/log/nginx/access.log; - error_log /var/log/nginx/error.log warn; - - location / { - proxy_pass http://collector:8000; - } -}
\ No newline at end of file diff --git a/env-vars.sh b/env-vars.sh deleted file mode 100644 index 8361a2f..0000000 --- a/env-vars.sh +++ /dev/null @@ -1,6 +0,0 @@ -export COUCHDB_USER=test -export COUCHDB_PASSWORD=test -export COUCHDB_NAME=test -export COUCHDB_HOSTNAME=couchdb -export DOCKER_JWT_PUBKEY_PATH=/tmp/soc_collector/ -export DOCKER_JWT_HTPASSWD_PATH=/tmp/soc_collector_htpasswd/ diff --git a/quickstart.sh b/quickstart.sh deleted file mode 100755 index 56cc77a..0000000 --- a/quickstart.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/sh - -source env-vars.sh -build="" - -while getopts "b" flag -do - case "$flag" in - b) build="--build";; - esac -done - -# Create a directory to store the certificates in. -if [ ! -d ${DOCKER_JWT_PUBKEY_PATH} ]; then - mkdir ${DOCKER_JWT_PUBKEY_PATH} -fi - -# And for the htpasswd file. -if [ ! -d ${DOCKER_JWT_HTPASSWD_PATH} ]; then - mkdir ${DOCKER_JWT_HTPASSWD_PATH} -fi - - -# Generate new certificates to use for JWT. -if [ ! -f ${DOCKER_JWT_PUBKEY_PATH}/private.pem ] && [ ! -f ${DOCKER_JWT_PUBKEY_PATH}/public.pem ]; then - openssl ecparam -genkey -name prime256v1 -noout -out ${DOCKER_JWT_PUBKEY_PATH}/private.pem - chmod 644 ${DOCKER_JWT_PUBKEY_PATH}/private.pem - openssl ec -in ${DOCKER_JWT_PUBKEY_PATH}/private.pem -pubout -out ${DOCKER_JWT_PUBKEY_PATH}/public.pem -fi - -# Generate a default htpasswd file with a user "usr:pwd". -if [ ! -f ${DOCKER_JWT_HTPASSWD_PATH}/.htpasswd ]; then - htpasswd -b -c ${DOCKER_JWT_HTPASSWD_PATH}/.htpasswd usr pwd - htpasswd -b ${DOCKER_JWT_HTPASSWD_PATH}/.htpasswd user1 pwd - htpasswd -b ${DOCKER_JWT_HTPASSWD_PATH}/.htpasswd user2 pwd - htpasswd -b ${DOCKER_JWT_HTPASSWD_PATH}/.htpasswd user3 pwd - htpasswd -b ${DOCKER_JWT_HTPASSWD_PATH}/.htpasswd user4 pwd -fi - -if [ ! -f ${DOCKER_JWT_HTPASSWD_PATH}/userdb.yaml ]; then - cp auth-server-poc/userdb.yaml ${DOCKER_JWT_HTPASSWD_PATH}/userdb.yaml -fi - -# Launch the containers. -docker-compose -f docker/docker-compose.yaml up -d $build -docker-compose -f auth-server-poc/docker-compose.yml up -d $build diff --git a/quickstart_test.sh b/quickstart_test.sh deleted file mode 100755 index 9254271..0000000 --- a/quickstart_test.sh +++ /dev/null @@ -1,67 +0,0 @@ -# Usage: ./quickstart_test.sh [-v] [-c] [-- <args to pytest>] - -export COUCHDB_NAME=unittest -export COUCHDB_HOSTNAME=localhost -export COUCHDB_USER=test -export COUCHDB_PASSWORD=test - -export DOCKER_JWT_PUBKEY_PATH="`pwd`/test/unittest_cert/" -export JWT_PUBKEY_PATH="`pwd`/test/unittest_cert/public.pem" - -virtualenv=no -couchdb=no - -while getopts ":vc" flag -do - case "$flag" in - v) virtualenv=yes;; - c) couchdb=yes;; - esac -done - -if [ -d test/unittest_cert ]; then - rm -r test/unittest_cert -fi - -if [ $virtualenv == "yes" ]; then - shift - if [ -d test/unittest_venv ]; then - rm -r test/unittest_venv - fi - - virtualenv test/unittest_venv - source test/unittest_venv/bin/activate - pip3 install -r ../requirements.txt -fi - -if [ $couchdb == "yes" ]; then - shift - docker run -it -p 6123:5984 --rm -d --name unittest_couchdb -e COUCHDB_USER=$COUCHDB_USER -e COUCHDB_PASSWORD=$COUCHDB_PASSWORD couchdb - - docker inspect unittest_couchdb > /dev/null - - if (( $? != 0 )); then - echo "Failed to start CouchDB container." - exit - fi - - export COUCHDB_PORT=6123 -fi - -mkdir test/unittest_cert - -cat <<EOF > test/unittest_cert/public.pem ------BEGIN PUBLIC KEY----- -MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEGHX8ipqVWtr49TXyX0f/L4GPhEpg -N0Erzy7hHkXVrkgKpnHSRLYWgbW4rscLoJAJeEv7Be5iH0TM8l09w8Q3wQ== ------END PUBLIC KEY----- -EOF - -shift -pytest --capture=tee-sys "$@" - -rm -r test/unittest_cert - -if [ $couchdb == "yes" ]; then - docker kill unittest_couchdb -fi diff --git a/requirements.in b/requirements.in index baafb83..1c1574a 100644 --- a/requirements.in +++ b/requirements.in @@ -1,15 +1,7 @@ -cryptography +# cryptography fastapi -fastapi-jwt-auth -py -pycparser -PyJWT -pyparsing -pytest requests rfc3339-validator -toml -typing-extensions -urllib3 -uvicorn +uvicorn[standard] jsonschema +motor diff --git a/requirements.txt b/requirements.txt index 9c587ab..664ce71 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,83 +7,17 @@ anyio==3.6.2 \ --hash=sha256:25ea0d673ae30af41a0c442f81cf3b38c7e79fdc7b60335a4c14e05eb0947421 \ --hash=sha256:fbbe32bd270d2a2ef3ed1c5d45041250284e31fc0a4df4a5a6071842051a51e3 - # via starlette + # via + # starlette + # watchfiles attrs==22.1.0 \ --hash=sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6 \ --hash=sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c - # via - # jsonschema - # pytest + # via jsonschema certifi==2022.9.24 \ --hash=sha256:0d9c601124e5a6ba9712dbc60d9c53c21e34f5f641fe83002317394311bdce14 \ --hash=sha256:90c1a32f1d68f940488354e36370f6cca89f0f106db09518524c88d6ed83f382 # via requests -cffi==1.15.1 \ - --hash=sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5 \ - --hash=sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef \ - --hash=sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104 \ - --hash=sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426 \ - --hash=sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405 \ - --hash=sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375 \ - --hash=sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a \ - --hash=sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e \ - --hash=sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc \ - --hash=sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf \ - --hash=sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185 \ - --hash=sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497 \ - --hash=sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3 \ - --hash=sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35 \ - --hash=sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c \ - --hash=sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83 \ - --hash=sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21 \ - --hash=sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca \ - --hash=sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984 \ - --hash=sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac \ - --hash=sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd \ - --hash=sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee \ - --hash=sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a \ - --hash=sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2 \ - --hash=sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192 \ - --hash=sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7 \ - --hash=sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585 \ - --hash=sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f \ - --hash=sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e \ - --hash=sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27 \ - --hash=sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b \ - --hash=sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e \ - --hash=sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e \ - --hash=sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d \ - --hash=sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c \ - --hash=sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415 \ - --hash=sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82 \ - --hash=sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02 \ - --hash=sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314 \ - --hash=sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325 \ - --hash=sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c \ - --hash=sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3 \ - --hash=sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914 \ - --hash=sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045 \ - --hash=sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d \ - --hash=sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9 \ - --hash=sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5 \ - --hash=sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2 \ - --hash=sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c \ - --hash=sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3 \ - --hash=sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2 \ - --hash=sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8 \ - --hash=sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d \ - --hash=sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d \ - --hash=sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9 \ - --hash=sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162 \ - --hash=sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76 \ - --hash=sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4 \ - --hash=sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e \ - --hash=sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9 \ - --hash=sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6 \ - --hash=sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b \ - --hash=sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01 \ - --hash=sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0 - # via cryptography charset-normalizer==2.1.1 \ --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 \ --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f @@ -92,84 +26,75 @@ click==8.1.3 \ --hash=sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e \ --hash=sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48 # via uvicorn -cryptography==38.0.1 \ - --hash=sha256:0297ffc478bdd237f5ca3a7dc96fc0d315670bfa099c04dc3a4a2172008a405a \ - --hash=sha256:10d1f29d6292fc95acb597bacefd5b9e812099d75a6469004fd38ba5471a977f \ - --hash=sha256:16fa61e7481f4b77ef53991075de29fc5bacb582a1244046d2e8b4bb72ef66d0 \ - --hash=sha256:194044c6b89a2f9f169df475cc167f6157eb9151cc69af8a2a163481d45cc407 \ - --hash=sha256:1db3d807a14931fa317f96435695d9ec386be7b84b618cc61cfa5d08b0ae33d7 \ - --hash=sha256:3261725c0ef84e7592597606f6583385fed2a5ec3909f43bc475ade9729a41d6 \ - --hash=sha256:3b72c360427889b40f36dc214630e688c2fe03e16c162ef0aa41da7ab1455153 \ - --hash=sha256:3e3a2599e640927089f932295a9a247fc40a5bdf69b0484532f530471a382750 \ - --hash=sha256:3fc26e22840b77326a764ceb5f02ca2d342305fba08f002a8c1f139540cdfaad \ - --hash=sha256:5067ee7f2bce36b11d0e334abcd1ccf8c541fc0bbdaf57cdd511fdee53e879b6 \ - --hash=sha256:52e7bee800ec869b4031093875279f1ff2ed12c1e2f74923e8f49c916afd1d3b \ - --hash=sha256:64760ba5331e3f1794d0bcaabc0d0c39e8c60bf67d09c93dc0e54189dfd7cfe5 \ - --hash=sha256:765fa194a0f3372d83005ab83ab35d7c5526c4e22951e46059b8ac678b44fa5a \ - --hash=sha256:79473cf8a5cbc471979bd9378c9f425384980fcf2ab6534b18ed7d0d9843987d \ - --hash=sha256:896dd3a66959d3a5ddcfc140a53391f69ff1e8f25d93f0e2e7830c6de90ceb9d \ - --hash=sha256:89ed49784ba88c221756ff4d4755dbc03b3c8d2c5103f6d6b4f83a0fb1e85294 \ - --hash=sha256:ac7e48f7e7261207d750fa7e55eac2d45f720027d5703cd9007e9b37bbb59ac0 \ - --hash=sha256:ad7353f6ddf285aeadfaf79e5a6829110106ff8189391704c1d8801aa0bae45a \ - --hash=sha256:b0163a849b6f315bf52815e238bc2b2346604413fa7c1601eea84bcddb5fb9ac \ - --hash=sha256:b6c9b706316d7b5a137c35e14f4103e2115b088c412140fdbd5f87c73284df61 \ - --hash=sha256:c2e5856248a416767322c8668ef1845ad46ee62629266f84a8f007a317141013 \ - --hash=sha256:ca9f6784ea96b55ff41708b92c3f6aeaebde4c560308e5fbbd3173fbc466e94e \ - --hash=sha256:d1a5bd52d684e49a36582193e0b89ff267704cd4025abefb9e26803adeb3e5fb \ - --hash=sha256:d3971e2749a723e9084dd507584e2a2761f78ad2c638aa31e80bc7a15c9db4f9 \ - --hash=sha256:d4ef6cc305394ed669d4d9eebf10d3a101059bdcf2669c366ec1d14e4fb227bd \ - --hash=sha256:d9e69ae01f99abe6ad646947bba8941e896cb3aa805be2597a0400e0764b5818 - # via -r requirements.in -exceptiongroup==1.0.0 \ - --hash=sha256:2ac84b496be68464a2da60da518af3785fff8b7ec0d090a581604bc870bdee41 \ - --hash=sha256:affbabf13fb6e98988c38d9c5650e701569fe3c1de3233cfb61c5f33774690ad - # via pytest -fastapi==0.85.2 \ - --hash=sha256:3e10ea0992c700e0b17b6de8c2092d7b9cd763ce92c49ee8d4be10fee3b2f367 \ - --hash=sha256:6292db0edd4a11f0d938d6033ccec5f706e9d476958bf33b119e8ddb4e524bde - # via - # -r requirements.in - # fastapi-jwt-auth -fastapi-jwt-auth==0.5.0 \ - --hash=sha256:43110a227e36f93b99257a1c79e66df8ed3d4c893291db8f9db2686192011f17 \ - --hash=sha256:d98068a9e828fe5909a1f338efabc3bb53149f11a9d2d395ab9d90fbb486b375 +dnspython==2.2.1 \ + --hash=sha256:0f7569a4a6ff151958b64304071d370daa3243d15941a7beedf0c9fe5105603e \ + --hash=sha256:a851e51367fb93e9e1361732c1d60dab63eff98712e503ea7d92e6eccb109b4f + # via pymongo +fastapi==0.86.0 \ + --hash=sha256:1020d7ca205d8b95813881fb3282e9c3656e47993531af3aa4ae11065b61dd2c \ + --hash=sha256:cdcaff84ecf7ae939b9579f0c98b0a0989ee3dd855710a32bc985260d92612f6 # via -r requirements.in h11==0.14.0 \ --hash=sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d \ --hash=sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761 # via uvicorn +httptools==0.5.0 \ + --hash=sha256:0297822cea9f90a38df29f48e40b42ac3d48a28637368f3ec6d15eebefd182f9 \ + --hash=sha256:1af91b3650ce518d226466f30bbba5b6376dbd3ddb1b2be8b0658c6799dd450b \ + --hash=sha256:1f90cd6fd97c9a1b7fe9215e60c3bd97336742a0857f00a4cb31547bc22560c2 \ + --hash=sha256:24bb4bb8ac3882f90aa95403a1cb48465de877e2d5298ad6ddcfdebec060787d \ + --hash=sha256:295874861c173f9101960bba332429bb77ed4dcd8cdf5cee9922eb00e4f6bc09 \ + --hash=sha256:3625a55886257755cb15194efbf209584754e31d336e09e2ffe0685a76cb4b60 \ + --hash=sha256:3a47a34f6015dd52c9eb629c0f5a8a5193e47bf2a12d9a3194d231eaf1bc451a \ + --hash=sha256:3cb8acf8f951363b617a8420768a9f249099b92e703c052f9a51b66342eea89b \ + --hash=sha256:4b098e4bb1174096a93f48f6193e7d9aa7071506a5877da09a783509ca5fff42 \ + --hash=sha256:4d9ebac23d2de960726ce45f49d70eb5466725c0087a078866043dad115f850f \ + --hash=sha256:50d4613025f15f4b11f1c54bbed4761c0020f7f921b95143ad6d58c151198142 \ + --hash=sha256:5230a99e724a1bdbbf236a1b58d6e8504b912b0552721c7c6b8570925ee0ccde \ + --hash=sha256:54465401dbbec9a6a42cf737627fb0f014d50dc7365a6b6cd57753f151a86ff0 \ + --hash=sha256:550059885dc9c19a072ca6d6735739d879be3b5959ec218ba3e013fd2255a11b \ + --hash=sha256:557be7fbf2bfa4a2ec65192c254e151684545ebab45eca5d50477d562c40f986 \ + --hash=sha256:5b65be160adcd9de7a7e6413a4966665756e263f0d5ddeffde277ffeee0576a5 \ + --hash=sha256:64eba6f168803a7469866a9c9b5263a7463fa8b7a25b35e547492aa7322036b6 \ + --hash=sha256:72ad589ba5e4a87e1d404cc1cb1b5780bfcb16e2aec957b88ce15fe879cc08ca \ + --hash=sha256:7d0c1044bce274ec6711f0770fd2d5544fe392591d204c68328e60a46f88843b \ + --hash=sha256:7e5eefc58d20e4c2da82c78d91b2906f1a947ef42bd668db05f4ab4201a99f49 \ + --hash=sha256:850fec36c48df5a790aa735417dca8ce7d4b48d59b3ebd6f83e88a8125cde324 \ + --hash=sha256:85b392aba273566c3d5596a0a490978c085b79700814fb22bfd537d381dd230c \ + --hash=sha256:8c2a56b6aad7cc8f5551d8e04ff5a319d203f9d870398b94702300de50190f63 \ + --hash=sha256:8f470c79061599a126d74385623ff4744c4e0f4a0997a353a44923c0b561ee51 \ + --hash=sha256:8ffce9d81c825ac1deaa13bc9694c0562e2840a48ba21cfc9f3b4c922c16f372 \ + --hash=sha256:9423a2de923820c7e82e18980b937893f4aa8251c43684fa1772e341f6e06887 \ + --hash=sha256:9b571b281a19762adb3f48a7731f6842f920fa71108aff9be49888320ac3e24d \ + --hash=sha256:a04fe458a4597aa559b79c7f48fe3dceabef0f69f562daf5c5e926b153817281 \ + --hash=sha256:aa47ffcf70ba6f7848349b8a6f9b481ee0f7637931d91a9860a1838bfc586901 \ + --hash=sha256:bede7ee075e54b9a5bde695b4fc8f569f30185891796b2e4e09e2226801d09bd \ + --hash=sha256:c1d2357f791b12d86faced7b5736dea9ef4f5ecdc6c3f253e445ee82da579449 \ + --hash=sha256:c6eeefd4435055a8ebb6c5cc36111b8591c192c56a95b45fe2af22d9881eee25 \ + --hash=sha256:ca1b7becf7d9d3ccdbb2f038f665c0f4857e08e1d8481cbcc1a86a0afcfb62b2 \ + --hash=sha256:e67d4f8734f8054d2c4858570cc4b233bf753f56e85217de4dfb2495904cf02e \ + --hash=sha256:e8a34e4c0ab7b1ca17b8763613783e2458e77938092c18ac919420ab8655c8c1 \ + --hash=sha256:e90491a4d77d0cb82e0e7a9cb35d86284c677402e4ce7ba6b448ccc7325c5421 \ + --hash=sha256:ef1616b3ba965cd68e6f759eeb5d34fbf596a79e84215eeceebf34ba3f61fdc7 \ + --hash=sha256:f222e1e9d3f13b68ff8a835574eda02e67277d51631d69d7cf7f8e07df678c86 \ + --hash=sha256:f5e3088f4ed33947e16fd865b8200f9cfae1144f41b64a8cf19b599508e096bc \ + --hash=sha256:f659d7a48401158c59933904040085c200b4be631cb5f23a7d561fbae593ec1f \ + --hash=sha256:fe9c766a0c35b7e3d6b6939393c8dfdd5da3ac5dec7f971ec9134f284c6c36d6 + # via uvicorn idna==3.4 \ --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 \ --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2 # via # anyio # requests -iniconfig==1.1.1 \ - --hash=sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3 \ - --hash=sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32 - # via pytest -jsonschema==4.16.0 \ - --hash=sha256:165059f076eff6971bae5b742fc029a7b4ef3f9bcf04c14e4776a7605de14b23 \ - --hash=sha256:9e74b8f9738d6a946d70705dc692b74b5429cd0960d58e79ffecfc43b2221eb9 +jsonschema==4.17.0 \ + --hash=sha256:5bfcf2bca16a087ade17e02b282d34af7ccd749ef76241e7f9bd7c0cb8a9424d \ + --hash=sha256:f660066c3966db7d6daeaea8a75e0b68237a48e51cf49882087757bb59916248 # via -r requirements.in -packaging==21.3 \ - --hash=sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb \ - --hash=sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522 - # via pytest -pluggy==1.0.0 \ - --hash=sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159 \ - --hash=sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3 - # via pytest -py==1.11.0 \ - --hash=sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719 \ - --hash=sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378 +motor==3.1.1 \ + --hash=sha256:01d93d7c512810dcd85f4d634a7244ba42ff6be7340c869791fe793561e734da \ + --hash=sha256:a4bdadf8a08ebb186ba16e557ba432aa867f689a42b80f2e9f8b24bbb1604742 # via -r requirements.in -pycparser==2.21 \ - --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ - --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 - # via - # -r requirements.in - # cffi pydantic==1.10.2 \ --hash=sha256:05e00dbebbe810b33c7a7362f231893183bcc4251f3f2ff991c31d5c08240c42 \ --hash=sha256:06094d18dd5e6f2bbf93efa54991c3240964bb663b87729ac340eb5014310624 \ @@ -208,46 +133,152 @@ pydantic==1.10.2 \ --hash=sha256:e0bedafe4bc165ad0a56ac0bd7695df25c50f76961da29c050712596cf092d6d \ --hash=sha256:e9069e1b01525a96e6ff49e25876d90d5a563bc31c658289a8772ae186552236 # via fastapi -pyjwt==1.7.1 \ - --hash=sha256:5c6eca3c2940464d106b99ba83b00c6add741c9becaec087fb7ccdefea71350e \ - --hash=sha256:8d59a976fb773f3e6a39c85636357c4f0e242707394cadadd9814f5cbaa20e96 - # via - # -r requirements.in - # fastapi-jwt-auth -pyparsing==3.0.9 \ - --hash=sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb \ - --hash=sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc - # via - # -r requirements.in - # packaging -pyrsistent==0.19.1 \ - --hash=sha256:06579d46d8ad69529b28f88711191a7fe7103c92d04a9f338dc754f71b92efa0 \ - --hash=sha256:1d0620474d509172e1c50b79d5626bfe1899f174bf650186a50c6ce31289ff52 \ - --hash=sha256:2032d971711643049b4f2c3ca5155a855d507d73bad26dac8d4349e5c5dd6758 \ - --hash=sha256:2c641111c3f110379bb9001dbb26b34eb8cafab3d0fa855dc161c391461a4aab \ - --hash=sha256:327f99800d04a9abcf580daecfd6dd4bfdb4a7e61c71bf2cd1189ef1ca44bade \ - --hash=sha256:39f15ad754384e744ac8b00805913bfa66c41131faaa3e4c45c4af0731f3e8f6 \ - --hash=sha256:4c58bd93c4d502f52938fccdbe6c9d70df3a585c6b39d900fab5f76b604282aa \ - --hash=sha256:62a41037387ae849a493cd945e22b34d167a843d57f75b07dbfad6d96cef485c \ - --hash=sha256:62b704f18526a8fc243152de8f3f40ae39c5172baff10f50c0c5d5331d6f2342 \ - --hash=sha256:6df99c3578dc4eb33f3eb26bc28277ab40a720b71649d940bff9c1f704377772 \ - --hash=sha256:6ef7430e45c5fa0bb6c361cada4a08ed9c184b5ed086815a85c3bc8c5054566b \ - --hash=sha256:73b2db09fe15b6e444c0bd566a125a385ca6493456224ce8b367d734f079f576 \ - --hash=sha256:73d4ec2997716af3c8f28f7e3d3a565d273a598982d2fe95639e07ce4db5da45 \ - --hash=sha256:73e3e2fd9da009d558050697cc22ad689f89a14a2ef2e67304628a913e59c947 \ - --hash=sha256:890f577aec554f142e01daf890221d10e4f93a9b1107998d631d3f075b55e8f8 \ - --hash=sha256:8a34a2a8b220247658f7ced871197c390b3a6371d796a5869ab1c62abe0be527 \ - --hash=sha256:8bc23e9ddcb523c3ffb4d712aa0bd5bc67b34ff4e2b23fb557012171bdb4013a \ - --hash=sha256:945297fc344fef4d540135180ce7babeb2291d124698cc6282f3eac624aa5e82 \ - --hash=sha256:aaa869d9199d7d4c70a57678aff21654cc179c0c32bcfde87f1d65d0ff47e520 \ - --hash=sha256:bc33fc20ddfd89b86b7710142963490d8c4ee8307ed6cc5e189a58fa72390eb9 \ - --hash=sha256:cfe6d8b293d123255fd3b475b5f4e851eb5cbaee2064c8933aa27344381744ae \ - --hash=sha256:d16ac5ab3d9db78fed40c884d67079524e4cf8276639211ad9e6fa73e727727e +pymongo==4.3.2 \ + --hash=sha256:006799ddba1f2e73ce27689f016791ab80e51876c52ae2265d8c76016baaa10e \ + --hash=sha256:02140c1a9f2107a16c074c9e558a556faafb0dc3c2e9332c6685c5506823ab9d \ + --hash=sha256:04597a5d877a984b5e3059e942b02d68f8af9bb4328592abca27e82015560112 \ + --hash=sha256:07e05784578bf7f8ecdfc6d0fd1e684e6259e9b5fdb5439a58c4f0df950fae29 \ + --hash=sha256:07f58d05d2289f93e16ddc93be6e0453fa67afd33c1b015f6bd3d9741c0963ff \ + --hash=sha256:0c8f061eabef3a6b3696f7f7be3eaed7928864ff84a2248429f9c7eb564343cc \ + --hash=sha256:0d7ad2112a705e992ca0cca98ccbb874276c495f8d9df627438c2ee94f810a3d \ + --hash=sha256:0f48c2562a1d1426b6db7567511dc62817df43357041e1fd4ea5c68278bfa11b \ + --hash=sha256:11630f5b3287375c85f5b7a788d3a7241671af24fda2b49a3396bc53cbf1c0c6 \ + --hash=sha256:178ffaa833d473b16fbd65c4a485af56484a50e2a201e8d0547f98cf5007f133 \ + --hash=sha256:1be15568e4b2be4c75bc54a542276c857628e09cbc283befcf4c45a0a22c1eec \ + --hash=sha256:21e1cfa3e73cd253afcad32e2a46a277f52553635ccc0dd4d643f5824af88428 \ + --hash=sha256:253faefea46482ffa87c77fdd01cd95d430cc84aae8d7a78ba920ea6cebcf3c7 \ + --hash=sha256:26dd79e60f883b6467b91c8af0be484147365b18cebf9248f8e72c035aecb693 \ + --hash=sha256:28ab644adc92c21a249570e2d677ebf4f2ef374630ddec98f19d2630dcb154c6 \ + --hash=sha256:2c821c897498e3e3c3254f7a90195f71473361f502201fd396281869d8108857 \ + --hash=sha256:315fe5f628e9aee67cc4c17b91ddf08c5c0917b764f433a5acf9aed33164a8f0 \ + --hash=sha256:352bc034e112c9f6a408e2796e74bae900d3167a804224b2c24ea75b5d57e9f9 \ + --hash=sha256:35e9eec45a212306143367b0702c2aff75c375290015af00fa8b653641c20b34 \ + --hash=sha256:36e7a74bdab9aa19f5ac94dfd74111d2164ccea752afbef0aa039d1266e7c404 \ + --hash=sha256:372307185d8e17ea31d2f3ff6943e213a6c379ccf547f18b05a58a1620d6f92a \ + --hash=sha256:39308580bcdbc368a2664c48761226c06b1d3368cc3ab3492d3cca88dc2e5e27 \ + --hash=sha256:3966dcba4b80dbc0eb4dd08d6f7127e3b1701cd829b6c13507a956c878b78546 \ + --hash=sha256:3f41781c8310fe1ae3ed0b809e2d7be6ebba9f0954c08e1d18ac443916b82b29 \ + --hash=sha256:404bc7f7190e8975f41f0c7498e303e9cb291f6384e1889ac4333448652a83d6 \ + --hash=sha256:4f7763c9e37e6d59406ce2defc25266980b24a86708ec6db753b02459db45715 \ + --hash=sha256:5231eb29e8174509250bc5fc609d6e8eceebfb209bf37bd6e014cbd7b6554344 \ + --hash=sha256:53dd2c034fb92c019e5e581cd361ed3fa9833abb56cc76725d56dcba169746fe \ + --hash=sha256:6049927b50c39e7dc51e75b5bb30c8501fbf1f08414b3447bcc9f9f967c116ed \ + --hash=sha256:6461d29a967e1980ba7798e4da8178dbe4245fe4a66ebb3aa07339c9da383c3a \ + --hash=sha256:6498ae9a76ad64617703373a43e3cd8454271bca0d7d395b393b4f31aa68f734 \ + --hash=sha256:64b010681019c0b312f342e3aae1f3091a7dc7ff4b7a3dec72fc0e7238be9477 \ + --hash=sha256:68213f4c1531b95dcfef40f79dd95e94484f69ec5949b7f42f82ad2bee135f7f \ + --hash=sha256:68320e5326e2b1e49dcd901e6dcbe3009b8a0fd0da0c618579a2be7cf5f2d7be \ + --hash=sha256:6db95d3e955aa5dbe42db691dd77cdddc0bc15f9883aa1def51f3ca40d49c1d6 \ + --hash=sha256:715ad027daff84e213ab74fa3ec98cad8dabb669653a71daa0dd6f80a1c32dd0 \ + --hash=sha256:7424b7c59b16e7889a720a5b2e2dda518753c6fec6c6582ab2fcedf97df3df75 \ + --hash=sha256:77436db17ab2baec2356cf38db32d13c7cd11267c8137864c67391f2dfdcc5e4 \ + --hash=sha256:7b2a8b2b7d9196d46e5181f88632eeca5bf79a69ca2e9911229c58f66aebfbeb \ + --hash=sha256:7c22ad464688a807bec103734cbdf712489c74d439cdd346e6f12095070bfbf5 \ + --hash=sha256:7ce5d43c011e03cd1a42a4dcc0d5c8772f18533cdfe672a63607942d62581df4 \ + --hash=sha256:7f907daec92208d748db4ea04568aa33e9254e0c27e4e40ac287e1b1ca8b12b5 \ + --hash=sha256:80bdfc7039674c670e1afbf95849ce2075731785527eeac7e3850e862dec239b \ + --hash=sha256:8817b17db2013354aa7f187d5825d65da0d7720b5ca697af37ff5efdf97e7f62 \ + --hash=sha256:8d81f6f5f6e66481aadd2fc087a937833312de23cd94b5ea1b225f35fafb0a00 \ + --hash=sha256:8f968621d019ed165f1cb5b037875ce3425ea7704407234895c7c52ad32190da \ + --hash=sha256:94639935caf13af551429bd13e4cb20e7c110a57d07f0c6a84a9bf3c2c9000ad \ + --hash=sha256:95913659d6c5fc714e662533d014836c988cc1561684f07b6a0a8343651afa66 \ + --hash=sha256:95f41c4e3b9e315655d6d1136d904ceda24fe5ea2d273ec6f9d66dbef06f3446 \ + --hash=sha256:98fd65c2aee7a55615dda1a1b0340ae8d756151983cb5040ea59a730083221e7 \ + --hash=sha256:a06c9ca15a2133478d1c775c4e7e5e782961b6254a3fc81ab5d0fb3cf9b8e358 \ + --hash=sha256:ab7f49c5ca3db7ae94743b0da1b21c5e7402a561a0614c1b0fba718aad591611 \ + --hash=sha256:aee8fafea8bb669deb0dd4878d947f79b2ef298e60f06e1fe799598929b68be2 \ + --hash=sha256:af46f635513c7339419374f46f4f662cee7140bfb86de4377885a2c1de2278d4 \ + --hash=sha256:b36da8aeb95cc1abea7b80e578fb6bcdbe395638d16b1b0068bc121e2111a00f \ + --hash=sha256:b4b683a40cf07b6d16704ead92a7aee24208d3af83d55d31248cdac003f8591c \ + --hash=sha256:b510843ea70e5bc9c096a93f683b28e8d43f1ad89da0126502d88b3d90f07ebe \ + --hash=sha256:c5db3bddbbc2657aa76088e76d24a616aefc98883c48dc27f3c3829ddb2ca10d \ + --hash=sha256:c8e82d6cc2f1cf5017485f55d67375bacf73d95c40903759e46024a987bab86f \ + --hash=sha256:cad31512e6956c95210fbd585d5b80df28425251260387164c6382894f0c6eca \ + --hash=sha256:cb47ba9c19da8fb4174f9d7bbbdb1796ad288c61dda35c96fb45d69e61d3a5cb \ + --hash=sha256:cc7b269af274ac0d5d9a5c8d035b03ccc34438baa01705bf8ec7cc6a31093ace \ + --hash=sha256:ce14598b8fa93e51aed0f400e446fddd6b26297ba5965fd0c0585614b60b9fc0 \ + --hash=sha256:cea32bd14d8c0725e22e5fcc607a81e3636650c689697c12423a34f9a125c7e2 \ + --hash=sha256:ceded83530f5507dadd873f8d004b56f996de44d9c3f56b7f26c22ca823f12ee \ + --hash=sha256:cefd851fdea191fc4db780157a28a11e0a80bccd34c454a73f252a287d28b2c7 \ + --hash=sha256:d7bdfac2f3c87d0971691f2a091427f55bb6b94b23d74213ed2de87d8facba85 \ + --hash=sha256:dab1d89f969046057be2b904a7bbf40df114f43aebfb3ccdceb054d9c40ec56d \ + --hash=sha256:db94b741dde2cc44ec038495d041c8f6dd4d510bb4e5d0be1b9f9aae4fbb28c6 \ + --hash=sha256:e4082d1b660e70d9df71da00050f7adb902b73a2287216e69ada124bd2f89636 \ + --hash=sha256:e96483316a799923f13bb61170f05feab22e8bd8630bf8cdcd440c78f307039a \ + --hash=sha256:f3861081540e1f06d1e5d131d1419b9fc507834b6865407e0f56735b4082566c \ + --hash=sha256:f423e066de040f4f93dcac0e6ceec37ffc25cc591a609ecc3ab20adfdbb787ae \ + --hash=sha256:f84b8428a41d7d7f2931762c27b09ffa8b3bc51e3b5dab40ab2b1d008091247e + # via motor +pyrsistent==0.19.2 \ + --hash=sha256:055ab45d5911d7cae397dc418808d8802fb95262751872c841c170b0dbf51eed \ + --hash=sha256:111156137b2e71f3a9936baf27cb322e8024dac3dc54ec7fb9f0bcf3249e68bb \ + --hash=sha256:187d5730b0507d9285a96fca9716310d572e5464cadd19f22b63a6976254d77a \ + --hash=sha256:21455e2b16000440e896ab99e8304617151981ed40c29e9507ef1c2e4314ee95 \ + --hash=sha256:2aede922a488861de0ad00c7630a6e2d57e8023e4be72d9d7147a9fcd2d30712 \ + --hash=sha256:3ba4134a3ff0fc7ad225b6b457d1309f4698108fb6b35532d015dca8f5abed73 \ + --hash=sha256:456cb30ca8bff00596519f2c53e42c245c09e1a4543945703acd4312949bfd41 \ + --hash=sha256:71d332b0320642b3261e9fee47ab9e65872c2bd90260e5d225dabeed93cbd42b \ + --hash=sha256:879b4c2f4d41585c42df4d7654ddffff1239dc4065bc88b745f0341828b83e78 \ + --hash=sha256:9cd3e9978d12b5d99cbdc727a3022da0430ad007dacf33d0bf554b96427f33ab \ + --hash=sha256:a178209e2df710e3f142cbd05313ba0c5ebed0a55d78d9945ac7a4e09d923308 \ + --hash=sha256:b39725209e06759217d1ac5fcdb510e98670af9e37223985f330b611f62e7425 \ + --hash=sha256:bfa0351be89c9fcbcb8c9879b826f4353be10f58f8a677efab0c017bf7137ec2 \ + --hash=sha256:bfd880614c6237243ff53a0539f1cb26987a6dc8ac6e66e0c5a40617296a045e \ + --hash=sha256:c43bec251bbd10e3cb58ced80609c5c1eb238da9ca78b964aea410fb820d00d6 \ + --hash=sha256:d690b18ac4b3e3cab73b0b7aa7dbe65978a172ff94970ff98d82f2031f8971c2 \ + --hash=sha256:d6982b5a0237e1b7d876b60265564648a69b14017f3b5f908c5be2de3f9abb7a \ + --hash=sha256:dec3eac7549869365fe263831f576c8457f6c833937c68542d08fde73457d291 \ + --hash=sha256:e371b844cec09d8dc424d940e54bba8f67a03ebea20ff7b7b0d56f526c71d584 \ + --hash=sha256:e5d8f84d81e3729c3b506657dddfe46e8ba9c330bf1858ee33108f8bb2adb38a \ + --hash=sha256:ea6b79a02a28550c98b6ca9c35b9f492beaa54d7c5c9e9949555893c8a9234d0 \ + --hash=sha256:f1258f4e6c42ad0b20f9cfcc3ada5bd6b83374516cd01c0960e3cb75fdca6770 # via jsonschema -pytest==7.2.0 \ - --hash=sha256:892f933d339f068883b6fd5a459f03d85bfcb355e4981e146d2c7616c21fef71 \ - --hash=sha256:c4014eb40e10f11f355ad4e3c2fb2c6c6d1919c73f3b5a433de4708202cade59 - # via -r requirements.in +python-dotenv==0.21.0 \ + --hash=sha256:1684eb44636dd462b66c3ee016599815514527ad99965de77f43e0944634a7e5 \ + --hash=sha256:b77d08274639e3d34145dfa6c7008e66df0f04b7be7a75fd0d5292c191d79045 + # via uvicorn +pyyaml==6.0 \ + --hash=sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf \ + --hash=sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293 \ + --hash=sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b \ + --hash=sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57 \ + --hash=sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b \ + --hash=sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4 \ + --hash=sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07 \ + --hash=sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba \ + --hash=sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9 \ + --hash=sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287 \ + --hash=sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513 \ + --hash=sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0 \ + --hash=sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782 \ + --hash=sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0 \ + --hash=sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92 \ + --hash=sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f \ + --hash=sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2 \ + --hash=sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc \ + --hash=sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1 \ + --hash=sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c \ + --hash=sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86 \ + --hash=sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4 \ + --hash=sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c \ + --hash=sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34 \ + --hash=sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b \ + --hash=sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d \ + --hash=sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c \ + --hash=sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb \ + --hash=sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7 \ + --hash=sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737 \ + --hash=sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3 \ + --hash=sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d \ + --hash=sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358 \ + --hash=sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53 \ + --hash=sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78 \ + --hash=sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803 \ + --hash=sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a \ + --hash=sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f \ + --hash=sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174 \ + --hash=sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5 + # via uvicorn requests==2.28.1 \ --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 \ --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349 @@ -268,28 +299,140 @@ starlette==0.20.4 \ --hash=sha256:42fcf3122f998fefce3e2c5ad7e5edbf0f02cf685d646a83a08d404726af5084 \ --hash=sha256:c0414d5a56297d37f3db96a84034d61ce29889b9eaccf65eb98a0b39441fcaa3 # via fastapi -toml==0.10.2 \ - --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \ - --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f - # via -r requirements.in -tomli==2.0.1 \ - --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \ - --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f - # via pytest typing-extensions==4.4.0 \ --hash=sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa \ --hash=sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e # via - # -r requirements.in # pydantic # starlette urllib3==1.26.12 \ --hash=sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e \ --hash=sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997 - # via - # -r requirements.in - # requests -uvicorn==0.19.0 \ + # via requests +uvicorn[standard]==0.19.0 \ --hash=sha256:cc277f7e73435748e69e075a721841f7c4a95dba06d12a72fe9874acced16f6f \ --hash=sha256:cf538f3018536edb1f4a826311137ab4944ed741d52aeb98846f52215de57f25 # via -r requirements.in +uvloop==0.17.0 \ + --hash=sha256:0949caf774b9fcefc7c5756bacbbbd3fc4c05a6b7eebc7c7ad6f825b23998d6d \ + --hash=sha256:0ddf6baf9cf11a1a22c71487f39f15b2cf78eb5bde7e5b45fbb99e8a9d91b9e1 \ + --hash=sha256:1436c8673c1563422213ac6907789ecb2b070f5939b9cbff9ef7113f2b531595 \ + --hash=sha256:23609ca361a7fc587031429fa25ad2ed7242941adec948f9d10c045bfecab06b \ + --hash=sha256:2a6149e1defac0faf505406259561bc14b034cdf1d4711a3ddcdfbaa8d825a05 \ + --hash=sha256:2deae0b0fb00a6af41fe60a675cec079615b01d68beb4cc7b722424406b126a8 \ + --hash=sha256:307958f9fc5c8bb01fad752d1345168c0abc5d62c1b72a4a8c6c06f042b45b20 \ + --hash=sha256:30babd84706115626ea78ea5dbc7dd8d0d01a2e9f9b306d24ca4ed5796c66ded \ + --hash=sha256:3378eb62c63bf336ae2070599e49089005771cc651c8769aaad72d1bd9385a7c \ + --hash=sha256:3d97672dc709fa4447ab83276f344a165075fd9f366a97b712bdd3fee05efae8 \ + --hash=sha256:3db8de10ed684995a7f34a001f15b374c230f7655ae840964d51496e2f8a8474 \ + --hash=sha256:3ebeeec6a6641d0adb2ea71dcfb76017602ee2bfd8213e3fcc18d8f699c5104f \ + --hash=sha256:45cea33b208971e87a31c17622e4b440cac231766ec11e5d22c76fab3bf9df62 \ + --hash=sha256:6708f30db9117f115eadc4f125c2a10c1a50d711461699a0cbfaa45b9a78e376 \ + --hash=sha256:68532f4349fd3900b839f588972b3392ee56042e440dd5873dfbbcd2cc67617c \ + --hash=sha256:6aafa5a78b9e62493539456f8b646f85abc7093dd997f4976bb105537cf2635e \ + --hash=sha256:7d37dccc7ae63e61f7b96ee2e19c40f153ba6ce730d8ba4d3b4e9738c1dccc1b \ + --hash=sha256:864e1197139d651a76c81757db5eb199db8866e13acb0dfe96e6fc5d1cf45fc4 \ + --hash=sha256:8887d675a64cfc59f4ecd34382e5b4f0ef4ae1da37ed665adba0c2badf0d6578 \ + --hash=sha256:8efcadc5a0003d3a6e887ccc1fb44dec25594f117a94e3127954c05cf144d811 \ + --hash=sha256:9b09e0f0ac29eee0451d71798878eae5a4e6a91aa275e114037b27f7db72702d \ + --hash=sha256:a4aee22ece20958888eedbad20e4dbb03c37533e010fb824161b4f05e641f738 \ + --hash=sha256:a5abddb3558d3f0a78949c750644a67be31e47936042d4f6c888dd6f3c95f4aa \ + --hash=sha256:c092a2c1e736086d59ac8e41f9c98f26bbf9b9222a76f21af9dfe949b99b2eb9 \ + --hash=sha256:c686a47d57ca910a2572fddfe9912819880b8765e2f01dc0dd12a9bf8573e539 \ + --hash=sha256:cbbe908fda687e39afd6ea2a2f14c2c3e43f2ca88e3a11964b297822358d0e6c \ + --hash=sha256:ce9f61938d7155f79d3cb2ffa663147d4a76d16e08f65e2c66b77bd41b356718 \ + --hash=sha256:dbbaf9da2ee98ee2531e0c780455f2841e4675ff580ecf93fe5c48fe733b5667 \ + --hash=sha256:f1e507c9ee39c61bfddd79714e4f85900656db1aec4d40c6de55648e85c2799c \ + --hash=sha256:ff3d00b70ce95adce264462c930fbaecb29718ba6563db354608f37e49e09024 + # via uvicorn +watchfiles==0.18.1 \ + --hash=sha256:00ea0081eca5e8e695cffbc3a726bb90da77f4e3f78ce29b86f0d95db4e70ef7 \ + --hash=sha256:0f9a22fff1745e2bb930b1e971c4c5b67ea3b38ae17a6adb9019371f80961219 \ + --hash=sha256:1b8e6db99e49cd7125d8a4c9d33c0735eea7b75a942c6ad68b75be3e91c242fb \ + --hash=sha256:4ec0134a5e31797eb3c6c624dbe9354f2a8ee9c720e0b46fc5b7bab472b7c6d4 \ + --hash=sha256:548d6b42303d40264118178053c78820533b683b20dfbb254a8706ca48467357 \ + --hash=sha256:6e0d8fdfebc50ac7569358f5c75f2b98bb473befccf9498cf23b3e39993bb45a \ + --hash=sha256:7102342d60207fa635e24c02a51c6628bf0472e5fef067f78a612386840407fc \ + --hash=sha256:888db233e06907c555eccd10da99b9cd5ed45deca47e41766954292dc9f7b198 \ + --hash=sha256:9891d3c94272108bcecf5597a592e61105279def1313521e637f2d5acbe08bc9 \ + --hash=sha256:9a26272ef3e930330fc0c2c148cc29706cc2c40d25760c7ccea8d768a8feef8b \ + --hash=sha256:9fb12a5e2b42e0b53769455ff93546e6bc9ab14007fbd436978d827a95ca5bd1 \ + --hash=sha256:a868ce2c7565137f852bd4c863a164dc81306cae7378dbdbe4e2aca51ddb8857 \ + --hash=sha256:b02e7fa03cd4059dd61ff0600080a5a9e7a893a85cb8e5178943533656eec65e \ + --hash=sha256:bc7c726855f04f22ac79131b51bf0c9f728cb2117419ed830a43828b2c4a5fcb \ + --hash=sha256:c541e0f2c3e95e83e4f84561c893284ba984e9d0025352057396d96dceb09f44 \ + --hash=sha256:cbaff354d12235002e62d9d3fa8bcf326a8490c1179aa5c17195a300a9e5952f \ + --hash=sha256:dde79930d1b28f15994ad6613aa2865fc7a403d2bb14585a8714a53233b15717 \ + --hash=sha256:e2b2bdd26bf8d6ed90763e6020b475f7634f919dbd1730ea1b6f8cb88e21de5d + # via uvicorn +websockets==10.4 \ + --hash=sha256:00213676a2e46b6ebf6045bc11d0f529d9120baa6f58d122b4021ad92adabd41 \ + --hash=sha256:00c870522cdb69cd625b93f002961ffb0c095394f06ba8c48f17eef7c1541f96 \ + --hash=sha256:0154f7691e4fe6c2b2bc275b5701e8b158dae92a1ab229e2b940efe11905dff4 \ + --hash=sha256:05a7233089f8bd355e8cbe127c2e8ca0b4ea55467861906b80d2ebc7db4d6b72 \ + --hash=sha256:09a1814bb15eff7069e51fed0826df0bc0702652b5cb8f87697d469d79c23576 \ + --hash=sha256:0cff816f51fb33c26d6e2b16b5c7d48eaa31dae5488ace6aae468b361f422b63 \ + --hash=sha256:185929b4808b36a79c65b7865783b87b6841e852ef5407a2fb0c03381092fa3b \ + --hash=sha256:2fc8709c00704194213d45e455adc106ff9e87658297f72d544220e32029cd3d \ + --hash=sha256:33d69ca7612f0ddff3316b0c7b33ca180d464ecac2d115805c044bf0a3b0d032 \ + --hash=sha256:389f8dbb5c489e305fb113ca1b6bdcdaa130923f77485db5b189de343a179393 \ + --hash=sha256:38ea7b82bfcae927eeffc55d2ffa31665dc7fec7b8dc654506b8e5a518eb4d50 \ + --hash=sha256:3d3cac3e32b2c8414f4f87c1b2ab686fa6284a980ba283617404377cd448f631 \ + --hash=sha256:40e826de3085721dabc7cf9bfd41682dadc02286d8cf149b3ad05bff89311e4f \ + --hash=sha256:4239b6027e3d66a89446908ff3027d2737afc1a375f8fd3eea630a4842ec9a0c \ + --hash=sha256:45ec8e75b7dbc9539cbfafa570742fe4f676eb8b0d3694b67dabe2f2ceed8aa6 \ + --hash=sha256:47a2964021f2110116cc1125b3e6d87ab5ad16dea161949e7244ec583b905bb4 \ + --hash=sha256:48c08473563323f9c9debac781ecf66f94ad5a3680a38fe84dee5388cf5acaf6 \ + --hash=sha256:4c6d2264f485f0b53adf22697ac11e261ce84805c232ed5dbe6b1bcb84b00ff0 \ + --hash=sha256:4f72e5cd0f18f262f5da20efa9e241699e0cf3a766317a17392550c9ad7b37d8 \ + --hash=sha256:56029457f219ade1f2fc12a6504ea61e14ee227a815531f9738e41203a429112 \ + --hash=sha256:5c1289596042fad2cdceb05e1ebf7aadf9995c928e0da2b7a4e99494953b1b94 \ + --hash=sha256:62e627f6b6d4aed919a2052efc408da7a545c606268d5ab5bfab4432734b82b4 \ + --hash=sha256:74de2b894b47f1d21cbd0b37a5e2b2392ad95d17ae983e64727e18eb281fe7cb \ + --hash=sha256:7c584f366f46ba667cfa66020344886cf47088e79c9b9d39c84ce9ea98aaa331 \ + --hash=sha256:7d27a7e34c313b3a7f91adcd05134315002aaf8540d7b4f90336beafaea6217c \ + --hash=sha256:7d3f0b61c45c3fa9a349cf484962c559a8a1d80dae6977276df8fd1fa5e3cb8c \ + --hash=sha256:82ff5e1cae4e855147fd57a2863376ed7454134c2bf49ec604dfe71e446e2193 \ + --hash=sha256:84bc2a7d075f32f6ed98652db3a680a17a4edb21ca7f80fe42e38753a58ee02b \ + --hash=sha256:884be66c76a444c59f801ac13f40c76f176f1bfa815ef5b8ed44321e74f1600b \ + --hash=sha256:8a5cc00546e0a701da4639aa0bbcb0ae2bb678c87f46da01ac2d789e1f2d2038 \ + --hash=sha256:8dc96f64ae43dde92530775e9cb169979f414dcf5cff670455d81a6823b42089 \ + --hash=sha256:8f38706e0b15d3c20ef6259fd4bc1700cd133b06c3c1bb108ffe3f8947be15fa \ + --hash=sha256:90fcf8929836d4a0e964d799a58823547df5a5e9afa83081761630553be731f9 \ + --hash=sha256:931c039af54fc195fe6ad536fde4b0de04da9d5916e78e55405436348cfb0e56 \ + --hash=sha256:932af322458da7e4e35df32f050389e13d3d96b09d274b22a7aa1808f292fee4 \ + --hash=sha256:942de28af58f352a6f588bc72490ae0f4ccd6dfc2bd3de5945b882a078e4e179 \ + --hash=sha256:9bc42e8402dc5e9905fb8b9649f57efcb2056693b7e88faa8fb029256ba9c68c \ + --hash=sha256:a7a240d7a74bf8d5cb3bfe6be7f21697a28ec4b1a437607bae08ac7acf5b4882 \ + --hash=sha256:a9f9a735deaf9a0cadc2d8c50d1a5bcdbae8b6e539c6e08237bc4082d7c13f28 \ + --hash=sha256:ae5e95cfb53ab1da62185e23b3130e11d64431179debac6dc3c6acf08760e9b1 \ + --hash=sha256:b029fb2032ae4724d8ae8d4f6b363f2cc39e4c7b12454df8df7f0f563ed3e61a \ + --hash=sha256:b0d15c968ea7a65211e084f523151dbf8ae44634de03c801b8bd070b74e85033 \ + --hash=sha256:b343f521b047493dc4022dd338fc6db9d9282658862756b4f6fd0e996c1380e1 \ + --hash=sha256:b627c266f295de9dea86bd1112ed3d5fafb69a348af30a2422e16590a8ecba13 \ + --hash=sha256:b9968694c5f467bf67ef97ae7ad4d56d14be2751000c1207d31bf3bb8860bae8 \ + --hash=sha256:ba089c499e1f4155d2a3c2a05d2878a3428cf321c848f2b5a45ce55f0d7d310c \ + --hash=sha256:bbccd847aa0c3a69b5f691a84d2341a4f8a629c6922558f2a70611305f902d74 \ + --hash=sha256:bc0b82d728fe21a0d03e65f81980abbbcb13b5387f733a1a870672c5be26edab \ + --hash=sha256:c57e4c1349fbe0e446c9fa7b19ed2f8a4417233b6984277cce392819123142d3 \ + --hash=sha256:c94ae4faf2d09f7c81847c63843f84fe47bf6253c9d60b20f25edfd30fb12588 \ + --hash=sha256:c9b27d6c1c6cd53dc93614967e9ce00ae7f864a2d9f99fe5ed86706e1ecbf485 \ + --hash=sha256:d210abe51b5da0ffdbf7b43eed0cfdff8a55a1ab17abbec4301c9ff077dd0342 \ + --hash=sha256:d58804e996d7d2307173d56c297cf7bc132c52df27a3efaac5e8d43e36c21c48 \ + --hash=sha256:d6a4162139374a49eb18ef5b2f4da1dd95c994588f5033d64e0bbfda4b6b6fcf \ + --hash=sha256:da39dd03d130162deb63da51f6e66ed73032ae62e74aaccc4236e30edccddbb0 \ + --hash=sha256:db3c336f9eda2532ec0fd8ea49fef7a8df8f6c804cdf4f39e5c5c0d4a4ad9a7a \ + --hash=sha256:dd500e0a5e11969cdd3320935ca2ff1e936f2358f9c2e61f100a1660933320ea \ + --hash=sha256:dd9becd5fe29773d140d68d607d66a38f60e31b86df75332703757ee645b6faf \ + --hash=sha256:e0cb5cc6ece6ffa75baccfd5c02cffe776f3f5c8bf486811f9d3ea3453676ce8 \ + --hash=sha256:e23173580d740bf8822fd0379e4bf30aa1d5a92a4f252d34e893070c081050df \ + --hash=sha256:e3a686ecb4aa0d64ae60c9c9f1a7d5d46cab9bfb5d91a2d303d00e2cd4c4c5cc \ + --hash=sha256:e789376b52c295c4946403bd0efecf27ab98f05319df4583d3c48e43c7342c2f \ + --hash=sha256:edc344de4dac1d89300a053ac973299e82d3db56330f3494905643bb68801269 \ + --hash=sha256:eef610b23933c54d5d921c92578ae5f89813438fded840c2e9809d378dc765d3 \ + --hash=sha256:f2c38d588887a609191d30e902df2a32711f708abfd85d318ca9b367258cfd0c \ + --hash=sha256:f55b5905705725af31ccef50e55391621532cd64fbf0bc6f4bac935f0fccec46 \ + --hash=sha256:f5fc088b7a32f244c519a048c170f14cf2251b849ef0e20cbbb0fdf0fdaf556f \ + --hash=sha256:fe10ddc59b304cb19a1bdf5bd0a7719cbbc9fbdd57ac80ed436b709fcf889106 \ + --hash=sha256:ff64a1d38d156d429404aaa84b27305e957fd10c30e5880d1765c9480bea490f + # via uvicorn diff --git a/src/collector/db.py b/src/collector/db.py index 0bfa014..3b16ef5 100755..100644 --- a/src/collector/db.py +++ b/src/collector/db.py @@ -1,148 +1,39 @@ -# A database storing dictionaries, keyed on a timestamp. value = A -# dict which will be stored as a JSON object encoded in UTF-8. Note -# that dict keys of type integer or float will become strings while -# values will keep their type. - -# Note that there's a (slim) chance that you'd stomp on the previous -# value if you're too quick with generating the timestamps, ie -# invoking time.time() several times quickly enough. - -from typing import Dict, List, Tuple, Union, Any -import os -import sys -import time - -from src import couch -from .schema import as_index_list, validate_collector_data - - -class DictDB: - def __init__(self) -> None: - """ - Check if the database exists, otherwise we will create it together - with the indexes specified in index.py. - """ - - print(os.environ) - - try: - self.database = os.environ["COUCHDB_NAME"] - self.hostname = os.environ["COUCHDB_HOSTNAME"] - self.username = os.environ["COUCHDB_USER"] - self.password = os.environ["COUCHDB_PASSWORD"] - except KeyError: - print( - "The environment variables COUCHDB_NAME, COUCHDB_HOSTNAME," - + " COUCHDB_USER and COUCHDB_PASSWORD must be set." - ) - sys.exit(-1) - - if "COUCHDB_PORT" in os.environ: - couchdb_port = os.environ["COUCHDB_PORT"] - else: - couchdb_port = "5984" - - self.server = couch.client.Server(f"http://{self.username}:{self.password}@{self.hostname}:{couchdb_port}/") - - try: - self.couchdb = self.server.database(self.database) - print("Database already exists") - except couch.exceptions.NotFound: - print("Creating database and indexes.") - self.couchdb = self.server.create(self.database) - - for i in as_index_list(): - self.couchdb.index(i) - - self._ts = time.time() - - def unique_key(self) -> int: - """ - Create a unique key based on the current time. We will use this as - the ID for any new documents we store in CouchDB. - """ - - ts = time.time() - while round(ts * 1000) == self._ts: - ts = time.time() - self._ts = round(ts * 1000) - - return self._ts - - # Why batch_write??? - def add(self, data: Union[List[Dict[str, Any]], Dict[str, Any]]) -> Union[str, Tuple[str, str]]: - """ - Store a document in CouchDB. - """ - - if isinstance(data, List): - for item in data: - error = validate_collector_data(item) - if error != "": - return error - item["_id"] = str(self.unique_key()) - ret: Tuple[str, str] = self.couchdb.save_bulk(data) +"""Our database module""" +from time import sleep +from sys import exit as app_exit +from dataclasses import dataclass + +from motor.motor_asyncio import ( + AsyncIOMotorClient, + AsyncIOMotorCollection, +) +from bson import ObjectId + + +@dataclass() +class DBClient: + """Class to hold database connections for us.""" + + client: AsyncIOMotorClient + collection: AsyncIOMotorCollection + + def __init__(self, username: str, password: str, collection: str) -> None: + self.client = AsyncIOMotorClient(f"mongodb://{username}:{password}@mongodb:27017/production", timeoutMS=2000) + self.collection = self.client["production"][collection] + + async def check_server(self) -> None: + """Try query the DB and exit the program if we fail after 5 times. + + :return: None + """ + for i in range(5): + try: + await self.collection.find_one({"_id": ObjectId("507f1f77bcf86cd799439011")}) + print("Connection to DB - OK") + break + except: # pylint: disable=bare-except + print(f"WARNING failed to connect to DB - {i} / 4", flush=True) + sleep(1) else: - error = validate_collector_data(data) - if error != "": - return error - data["_id"] = str(self.unique_key()) - ret = self.couchdb.save(data) - - return ret - - def get(self, key: int) -> Dict[str, Any]: - """ - Get a document based on its ID, return an empty dict if not found. - """ - - try: - doc: Dict[str, Any] = self.couchdb.get(key) - except couch.exceptions.NotFound: - doc = {} - - return doc - - # - # def slice(self, key_from=None, key_to=None): - # pass - - def search(self, limit: int = 25, skip: int = 0, **kwargs: Any) -> List[Dict[str, Any]]: - """ - Execute a Mango query, ideally we should have an index matching - the query otherwise things will be slow. - """ - - data: List[Dict[str, Any]] = [] - selector: Dict[str, Any] = {} - - try: - limit = int(limit) - skip = int(skip) - except ValueError: - limit = 25 - skip = 0 - - if kwargs: - selector = {"limit": limit, "skip": skip, "selector": {}} - - for key in kwargs: - if kwargs[key] and kwargs[key].isnumeric(): - kwargs[key] = int(kwargs[key]) - selector["selector"][key] = {"$eq": kwargs[key]} - - for doc in self.couchdb.find(selector, wrapper=None, limit=5): - data.append(doc) - - return data - - def delete(self, key: int) -> Union[int, None]: - """ - Delete a document based on its ID. - """ - try: - self.couchdb.delete(key) - except couch.exceptions.NotFound: - return None - - return key + print("Could not connect to DB - mongodb://REDACTED_USERNAME:REDACTED_PASSWORD@mongodb:27017/production") + app_exit(1) diff --git a/src/collector/main.py b/src/collector/main.py index c363885..096b788 100755 --- a/src/collector/main.py +++ b/src/collector/main.py @@ -1,267 +1,175 @@ -from typing import Dict, Union, List, Callable, Awaitable, Any -import json -import os +"""Our main module""" +from typing import Dict, Optional, List, Any +from os import environ +import asyncio import sys -import time +from json.decoder import JSONDecodeError -import uvicorn -from fastapi import Depends, FastAPI, Request, Response -from fastapi.middleware.cors import CORSMiddleware +from fastapi import FastAPI, Request from fastapi.responses import JSONResponse -from fastapi_jwt_auth import AuthJWT -from fastapi_jwt_auth.auth_config import AuthConfig -from fastapi_jwt_auth.exceptions import AuthJWTException from pydantic import BaseModel - -from .db import DictDB -from .schema import get_index_keys, validate_collector_data - -app = FastAPI() - -app.add_middleware( - CORSMiddleware, - allow_origins=["http://localhost:8001"], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], - expose_headers=["X-Total-Count"], +from pymongo.errors import OperationFailure +from bson import ( + ObjectId, + json_util, ) +from dotenv import load_dotenv -# TODO: X-Total-Count - - -@app.middleware("http") -async def mock_x_total_count_header(request: Request, call_next: Callable[[Request], Awaitable[Response]]) -> Response: - - print(type(call_next)) - - response: Response = await call_next(request) - response.headers["X-Total-Count"] = "100" - return response - - -for i in range(10): - try: - db = DictDB() - except Exception as e: - print(f"Database not responding, will try again soon. Attempt {i + 1} of 10.") - else: - break - time.sleep(1) -else: - print("Database did not respond after 10 attempts, quitting.") - sys.exit(-1) - - -def get_pubkey() -> str: - try: - if "JWT_PUBKEY_PATH" in os.environ: - keypath = os.environ["JWT_PUBKEY_PATH"] - else: - keypath = "/opt/certs/public.pem" - - with open(keypath, "r") as fd: - pubkey = fd.read() - except FileNotFoundError: - print(f"Could not find JWT certificate in {keypath}") - sys.exit(-1) - - return pubkey - - -def get_data( - key: Union[int, None] = None, - limit: int = 25, - skip: int = 0, - ip: Union[str, None] = None, - port: Union[int, None] = None, - asn: Union[str, None] = None, - domain: Union[str, None] = None, -) -> List[Dict[str, Any]]: - if key: - return [db.get(key)] +from .db import DBClient +from .schema import valid_schema - selectors: Dict[str, Any] = {} - indexes = get_index_keys() - selectors["domain"] = domain - if ip and "ip" in indexes: - selectors["ip"] = ip - if port and "port" in indexes: - selectors["port"] = port - if asn and "asn" in indexes: - selectors["asn"] = asn +load_dotenv() +# Get credentials +if "MONGODB_USERNAME" not in environ or "MONGODB_PASSWORD" not in environ or "MONGODB_COLLECTION" not in environ: + print("Missing MONGODB_USERNAME or MONGODB_PASSWORD or MONGODB_COLLECTION in env") + sys.exit(1) - data: List[Dict[str, Any]] = db.search(**selectors, limit=limit, skip=skip) +# Create DB object +db = DBClient(environ["MONGODB_USERNAME"], environ["MONGODB_PASSWORD"], environ["MONGODB_COLLECTION"]) - return data - - -class JWTConfig(BaseModel): - authjwt_algorithm: str = "ES256" - authjwt_public_key: str = get_pubkey() +# Check DB +loop = asyncio.get_running_loop() +startup_task = loop.create_task(db.check_server()) +app = FastAPI() -@AuthJWT.load_config # type: ignore -def jwt_config(): - return JWTConfig() +# @app.exception_handler(RuntimeError) +# def app_exception_handler(request: Request, exc: RuntimeError) -> JSONResponse: +# print(exc, flush=True) +# return JSONResponse(content={"status": "error", "message": str(exc.with_traceback(None))}, status_code=400) +# return JSONResponse(content={"status": "error", "message": "Error during processing"}, status_code=400) -@app.exception_handler(AuthJWTException) -def authjwt_exception_handler(request: Request, exc: AuthJWTException) -> JSONResponse: - return JSONResponse(content={"status": "error", "message": exc.message}, status_code=400) +class SearchInput(BaseModel): + """Handle search data for HTTP request""" -@app.exception_handler(RuntimeError) -def app_exception_handler(request: Request, exc: RuntimeError) -> JSONResponse: - return JSONResponse(content={"status": "error", "message": str(exc.with_traceback(None))}, status_code=400) + search: Optional[Dict[str, Any]] + limit: int = 25 + skip: int = 0 -@app.get("/sc/v0/get") -async def get( - key: Union[int, None] = None, - limit: int = 25, - skip: int = 0, - ip: Union[str, None] = None, - port: Union[int, None] = None, - asn: Union[str, None] = None, - Authorize: AuthJWT = Depends(), -) -> JSONResponse: +@app.post("/sc/v0/search") +async def search(search_data: SearchInput) -> JSONResponse: + """/sc/v0/search, POST method - Authorize.jwt_required() + :param search_data: The search data. + :return: JSONResponse + """ + data: List[Dict[str, Any]] = [] - data = [] - raw_jwt = Authorize.get_raw_jwt() + cursor = db.collection.find(search_data.search) + cursor.sort("timestamp", -1).limit(search_data.limit).skip(search_data.skip) - if "read" not in raw_jwt: + try: + async for document in cursor: + data.append(document) + except OperationFailure as exc: + print(f"DB failed to process: {exc.details}") return JSONResponse( content={ "status": "error", - "message": "Could not find read claim in JWT token", + "message": "Probably wrong syntax, note the dictionary for find: " + + "https://motor.readthedocs.io/en/stable/tutorial-asyncio.html#async-for", }, status_code=400, ) - else: - domains = raw_jwt["read"] - for domain in domains: - data.extend(get_data(key, limit, skip, ip, port, asn, domain)) + if not data: + return JSONResponse(content={"status": "error", "message": "Document not found"}, status_code=400) - return JSONResponse(content={"status": "success", "docs": data}) + return JSONResponse(content={"status": "success", "docs": json_util.dumps(data)}) -@app.get("/sc/v0/get/{key}") -async def get_key(key: Union[int, None] = None, Authorize: AuthJWT = Depends()) -> JSONResponse: +@app.post("/sc/v0") +async def create(request: Request) -> JSONResponse: + """/sc/v0, POST method - Authorize.jwt_required() + :param request: The request where we get the json body. + :return: JSONResponse + """ - raw_jwt = Authorize.get_raw_jwt() + try: + json_data = await request.json() + except JSONDecodeError: + return JSONResponse(content={"status": "error", "message": "Invalid JSON"}, status_code=400) - if "read" not in raw_jwt: - return JSONResponse( - content={ - "status": "error", - "message": "Could not find read claim in JWT token", - }, - status_code=400, - ) - else: - allowed_domains = raw_jwt["read"] + if not valid_schema(json_data): + return JSONResponse(content={"status": "error", "message": "Not our JSON schema"}, status_code=400) - data_list = get_data(key) + result = await db.collection.insert_one(json_data) + return JSONResponse(content={"status": "success", "key": str(result.inserted_id)}) - # Handle if missing - data = data_list[0] - if data and data["domain"] not in allowed_domains: - return JSONResponse( - content={ - "status": "error", - "message": "User not authorized to view this object", - }, - status_code=400, - ) - - return JSONResponse(content={"status": "success", "docs": data}) +@app.put("/sc/v0") +async def update(request: Request) -> JSONResponse: + """/sc/v0, PUT method - -# WHY IS AUTH OUTCOMMENTED??? -@app.post("/sc/v0/add") -async def add(data: Request, Authorize: AuthJWT = Depends()) -> JSONResponse: - # Authorize.jwt_required() + :param request: The request where we get the json body. + :return: JSONResponse + """ try: - json_data = await data.json() - except json.decoder.JSONDecodeError: - return JSONResponse( - content={ - "status": "error", - "message": "Invalid JSON.", - }, - status_code=400, - ) - - key = db.add(json_data) - - if isinstance(key, str): - return JSONResponse( - content={ - "status": "error", - "message": key, - }, - status_code=400, - ) - - return JSONResponse(content={"status": "success", "docs": key}) - + json_data = await request.json() + except JSONDecodeError: + return JSONResponse(content={"status": "error", "message": "Invalid JSON"}, status_code=400) + + if "_id" not in json_data: + return JSONResponse(content={"status": "error", "message": "Missing key '_id'"}, status_code=400) + + # Get the key + if isinstance(json_data["_id"], str): + object_id = ObjectId(json_data["_id"]) + elif ( + isinstance(json_data["_id"], dict) and "$oid" in json_data["_id"] and isinstance(json_data["_id"]["$oid"], str) + ): + object_id = ObjectId(json_data["_id"]["$oid"]) + else: + return JSONResponse(content={"status": "error", "message": "Missing key '_id' with valid id"}, status_code=400) -@app.delete("/sc/v0/delete/{key}") -async def delete(key: int, Authorize: AuthJWT = Depends()) -> JSONResponse: + # Ensure the updating key exist + document = await db.collection.find_one({"_id": object_id}) + if document is None: + return JSONResponse(content={"status": "error", "message": "Document not found"}, status_code=400) - Authorize.jwt_required() + # Ensure valid schema + del json_data["_id"] + if not valid_schema(json_data): + return JSONResponse(content={"status": "error", "message": "Not our JSON schema"}, status_code=400) - raw_jwt = Authorize.get_raw_jwt() + # Replace the data + json_data["_id"] = object_id + await db.collection.replace_one({"_id": object_id}, json_data) + return JSONResponse(content={"status": "success", "key": str(object_id)}) - if "write" not in raw_jwt: - return JSONResponse( - content={ - "status": "error", - "message": "Could not find write claim in JWT token", - }, - status_code=400, - ) - else: - allowed_domains = raw_jwt["write"] - data_list = get_data(key) +@app.get("/sc/v0/{key}") +async def get(key: str) -> JSONResponse: + """/sc/v0, POST method - # Handle if missing - data = data_list[0] + :param key: The document key in the database. + :return: JSONResponse + """ - if data and data["domain"] not in allowed_domains: - return JSONResponse( - content={ - "status": "error", - "message": "User not authorized to delete this object", - }, - status_code=400, - ) + document = await db.collection.find_one({"_id": ObjectId(key)}) - if db.delete(key) is None: + if document is None: return JSONResponse(content={"status": "error", "message": "Document not found"}, status_code=400) - return JSONResponse(content={"status": "success", "docs": data}) + return JSONResponse(content={"status": "success", "docs": json_util.dumps(document)}) -# def main(standalone: bool = False): -# print(type(app)) -# if not standalone: -# return app +@app.delete("/sc/v0/{key}") +async def delete(key: str) -> JSONResponse: + """/sc/v0, POST method -# uvicorn.run(app, host="0.0.0.0", port=8000, log_level="debug") + :param key: The document key in the database. + :return: JSONResponse + """ + result = await db.collection.delete_one({"_id": ObjectId(key)}) + if result.deleted_count == 0: + return JSONResponse(content={"status": "error", "message": "Document not found"}, status_code=400) -# if __name__ == "__main__": -# main(standalone=True) -# else: -# app = main() + return JSONResponse(content={"status": "success", "key": key}) diff --git a/src/collector/schema.py b/src/collector/schema.py index e291f10..221990a 100644 --- a/src/collector/schema.py +++ b/src/collector/schema.py @@ -1,8 +1,5 @@ -from typing import List, Any, Dict -import json -import sys -import traceback - +"""Our schema module""" +from typing import Any, Dict import jsonschema # fmt:off @@ -64,7 +61,8 @@ schema = { ] }, { - "required": [ + "required": + [ "display_name", "investigation_needed", # "reliability", # TODO: reliability is required if investigation_needed = true @@ -93,44 +91,17 @@ schema = { "result", ], } -# fmt:on - - -def get_index_keys() -> List[str]: - keys: List[str] = [] - for key in schema["properties"]: - keys.append(key) - return keys -def as_index_list() -> List[Dict[str, Any]]: - index_list: List[Dict[str, Any]] = [] - for key in schema["properties"]: - name = f"{key}-json-index" - index = { - "index": { - "fields": [ - key, - ] - }, - "name": name, - "type": "json", - } - index_list.append(index) - - return index_list +def valid_schema(json_data: Dict[str, Any]) -> bool: + """Check if json data follows the schema. - -def validate_collector_data(json_blob: Dict[str, Any]) -> str: + :param json_data: Json object + :return: bool + """ try: - jsonschema.validate(json_blob, schema, format_checker=jsonschema.FormatChecker()) - except jsonschema.exceptions.ValidationError as e: - return f"Validation failed with error: {e.message}" - return "" - - -if __name__ == "__main__": - with open(sys.argv[1]) as fd: - json_data = json.loads(fd.read()) - - print(validate_collector_data(json_data)) + jsonschema.validate(json_data, schema, format_checker=jsonschema.FormatChecker()) + except jsonschema.exceptions.ValidationError as exc: + print(f"Validation failed with error: {exc.message}") + return False + return True diff --git a/src/couch/__init__.py b/src/couch/__init__.py deleted file mode 100644 index 64e0252..0000000 --- a/src/couch/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -# -*- coding: utf-8 -*- - -__author__ = "Andrey Antukh" -__license__ = "BSD" -__version__ = "1.14.1" -__maintainer__ = "Rinat Sabitov" -__email__ = "rinat.sabitov@gmail.com" -__status__ = "Development" - - -from .client import Server # noqa: F401 diff --git a/src/couch/client.py b/src/couch/client.py deleted file mode 100644 index 96dc78a..0000000 --- a/src/couch/client.py +++ /dev/null @@ -1,801 +0,0 @@ -# -*- coding: utf-8 -*- -# Based on py-couchdb (https://github.com/histrio/py-couchdb) - -import os -import json -import uuid -import copy -import mimetypes -import warnings - -from .utils import ( - force_bytes, - force_text, - encode_view_options, - extract_credentials, -) -from .feedreader import ( - SimpleFeedReader, - BaseFeedReader, -) - -from .exceptions import ( - Conflict, - NotFound, - FeedReaderExited, - UnexpectedError, -) -from .resource import Resource - - -DEFAULT_BASE_URL = os.environ.get('COUCHDB_URL', 'http://localhost:5984/') - - -def _id_to_path(_id: str) -> str: - if _id[:1] == "_": - return _id.split("/", 1) - return [_id] - - -def _listen_feed(object, node, feed_reader, **kwargs): - if not callable(feed_reader): - raise UnexpectedError("feed_reader must be callable or class") - - if isinstance(feed_reader, BaseFeedReader): - reader = feed_reader(object) - else: - reader = SimpleFeedReader()(object, feed_reader) - - # Possible options: "continuous", "longpoll" - kwargs.setdefault("feed", "continuous") - data = force_bytes(json.dumps(kwargs.pop('data', {}))) - - (resp, result) = object.resource(node).post( - params=kwargs, data=data, stream=True) - try: - for line in resp.iter_lines(): - # ignore heartbeats - if not line: - reader.on_heartbeat() - else: - reader.on_message(json.loads(force_text(line))) - except FeedReaderExited: - reader.on_close() - - -class _StreamResponse(object): - """ - Proxy object for python-requests stream response. - - See more on: - http://docs.python-requests.org/en/latest/user/advanced/#streaming-requests - """ - - def __init__(self, response): - self._response = response - - def iter_content(self, chunk_size=1, decode_unicode=False): - return self._response.iter_content(chunk_size=chunk_size, - decode_unicode=decode_unicode) - - def iter_lines(self, chunk_size=512, decode_unicode=None): - return self._response.iter_lines(chunk_size=chunk_size, - decode_unicode=decode_unicode) - - @property - def raw(self): - return self._response.raw - - @property - def url(self): - return self._response.url - - -class Server(object): - """ - Class that represents a couchdb connection. - - :param verify: setup ssl verification. - :param base_url: a full url to couchdb (can contain auth data). - :param full_commit: If ``False``, couchdb not commits all data on a - request is finished. - :param authmethod: specify a authentication method. By default "basic" - method is used but also exists "session" (that requires - some server configuration changes). - - .. versionchanged: 1.4 - Set basic auth method as default instead of session method. - - .. versionchanged: 1.5 - Add verify parameter for setup ssl verificaton - - """ - - def __init__(self, base_url=DEFAULT_BASE_URL, full_commit=True, - authmethod="basic", verify=False): - - self.base_url, credentials = extract_credentials(base_url) - self.resource = Resource(self.base_url, full_commit, - credentials=credentials, - authmethod=authmethod, - verify=verify) - - def __repr__(self): - return '<CouchDB Server "{}">'.format(self.base_url) - - def __contains__(self, name): - try: - self.resource.head(name) - except NotFound: - return False - else: - return True - - def __iter__(self): - (r, result) = self.resource.get('_all_dbs') - return iter(result) - - def __len__(self): - (r, result) = self.resource.get('_all_dbs') - return len(result) - - def info(self): - """ - Get server info. - - :returns: dict with all data that couchdb returns. - :rtype: dict - """ - (r, result) = self.resource.get() - return result - - def delete(self, name): - """ - Delete some database. - - :param name: database name - :raises: :py:exc:`~pycouchdb.exceptions.NotFound` - if a database does not exists - """ - - self.resource.delete(name) - - def database(self, name): - """ - Get a database instance. - - :param name: database name - :raises: :py:exc:`~pycouchdb.exceptions.NotFound` - if a database does not exists - - :returns: a :py:class:`~pycouchdb.client.Database` instance - """ - (r, result) = self.resource.head(name) - if r.status_code == 404: - raise NotFound("Database '{0}' does not exists".format(name)) - - db = Database(self.resource(name), name) - return db - - # TODO: Config in 2.0 are applicable for nodes only - # TODO: Reimplement when nodes endpoint will be ready - # def config(self): - # pass - - def version(self): - """ - Get the current version of a couchdb server. - """ - (resp, result) = self.resource.get() - return result["version"] - - # TODO: Stats in 2.0 are applicable for nodes only - # TODO: Reimplement when nodes endpoint will be ready - # def stats(self, name=None): - # pass - - def create(self, name): - """ - Create a database. - - :param name: database name - :raises: :py:exc:`~pycouchdb.exceptions.Conflict` - if a database already exists - :returns: a :py:class:`~pycouchdb.client.Database` instance - """ - (resp, result) = self.resource.put(name) - if resp.status_code in (200, 201): - return self.database(name) - - def replicate(self, source, target, **kwargs): - """ - Replicate the source database to the target one. - - .. versionadded:: 1.3 - - :param source: full URL to the source database - :param target: full URL to the target database - """ - - data = {'source': source, 'target': target} - data.update(kwargs) - - data = force_bytes(json.dumps(data)) - - (resp, result) = self.resource.post('_replicate', data=data) - return result - - def changes_feed(self, feed_reader, **kwargs): - """ - Subscribe to changes feed of the whole CouchDB server. - - Note: this method is blocking. - - - :param feed_reader: callable or :py:class:`~BaseFeedReader` - instance - - .. [Ref] http://docs.couchdb.org/en/1.6.1/api/server/common.html#db-updates - .. versionadded: 1.10 - """ - object = self - _listen_feed(object, "_db_updates", feed_reader, **kwargs) - - -class Database(object): - """ - Class that represents a couchdb database. - """ - - def __init__(self, resource, name): - self.resource = resource - self.name = name - - def __repr__(self): - return '<CouchDB Database "{}">'.format(self.name) - - def __contains__(self, doc_id): - try: - (resp, result) = self.resource.head(_id_to_path(doc_id)) - return resp.status_code < 206 - except NotFound: - return False - - def config(self): - """ - Get database status data such as document count, update sequence etc. - :return: dict - """ - (resp, result) = self.resource.get() - return result - - def __nonzero__(self): - """Is the database available""" - resp, _ = self.resource.head() - return resp.status_code == 200 - - def __len__(self): - return self.config()['doc_count'] - - def delete(self, doc_or_id): - """ - Delete document by id. - - .. versionchanged:: 1.2 - Accept document or id. - - :param doc_or_id: document or id - :raises: :py:exc:`~pycouchdb.exceptions.NotFound` if a document - not exists - :raises: :py:exc:`~pycouchdb.exceptions.Conflict` if delete with - wrong revision. - """ - - _id = None - if isinstance(doc_or_id, dict): - if "_id" not in doc_or_id: - raise ValueError("Invalid document, missing _id attr") - _id = doc_or_id['_id'] - else: - _id = doc_or_id - - resource = self.resource(*_id_to_path(_id)) - - (r, result) = resource.head() - (r, result) = resource.delete( - params={"rev": r.headers["etag"].strip('"')}) - - def delete_bulk(self, docs, transaction=True): - """ - Delete a bulk of documents. - - .. versionadded:: 1.2 - - :param docs: list of docs - :raises: :py:exc:`~pycouchdb.exceptions.Conflict` if a delete - is not success - :returns: raw results from server - """ - - _docs = copy.copy(docs) - for doc in _docs: - if "_deleted" not in doc: - doc["_deleted"] = True - - data = force_bytes(json.dumps({"docs": _docs})) - params = {"all_or_nothing": "true" if transaction else "false"} - (resp, results) = self.resource.post( - "_bulk_docs", data=data, params=params) - - for result, doc in zip(results, _docs): - if "error" in result: - raise Conflict("one or more docs are not saved") - - return results - - def get(self, doc_id, params=None, **kwargs): - """ - Get a document by id. - - .. versionadded: 1.5 - Now the prefered method to pass params is via **kwargs - instead of params argument. **params** argument is now - deprecated and will be deleted in future versions. - - :param doc_id: document id - :raises: :py:exc:`~pycouchdb.exceptions.NotFound` if a document - not exists - - :returns: document (dict) - """ - - if params: - warnings.warn("params parameter is now deprecated in favor to" - "**kwargs usage.", DeprecationWarning) - - if params is None: - params = {} - - params.update(kwargs) - - (resp, result) = self.resource(*_id_to_path(doc_id)).get(params=params) - return result - - def save(self, doc, batch=False): - """ - Save or update a document. - - .. versionchanged:: 1.2 - Now returns a new document instead of modify the original. - - :param doc: document - :param batch: allow batch=ok inserts (default False) - :raises: :py:exc:`~pycouchdb.exceptions.Conflict` if save with wrong - revision. - :returns: doc - """ - - _doc = copy.copy(doc) - if "_id" not in _doc: - _doc['_id'] = uuid.uuid4().hex - - if batch: - params = {'batch': 'ok'} - else: - params = {} - - data = force_bytes(json.dumps(_doc)) - - print("gg1", flush=True) - print(data, flush=True) - print("vv1", flush=True) - - (resp, result) = self.resource(_doc['_id']).put( - data=data, params=params) - - print("gg3", flush=True) - print(resp.status_code) - print(resp.content) - #print(resp.contents) - - print("gg2", flush=True) - print(data, flush=True) - print(result, flush=True) - print("vv2", flush=True) - - if resp.status_code == 409: - raise Conflict(result['reason']) - - if "rev" in result and result["rev"] is not None: - _doc["_rev"] = result["rev"] - - return _doc - - def save_bulk(self, docs, try_setting_ids=True, transaction=True): - """ - Save a bulk of documents. - - .. versionchanged:: 1.2 - Now returns a new document list instead of modify the original. - - :param docs: list of docs - :param try_setting_ids: if ``True``, we loop through docs and generate/set - an id in each doc if none exists - :param transaction: if ``True``, couchdb do a insert in transaction - model. - :returns: docs - """ - - _docs = copy.deepcopy(docs) - - # Insert _id field if it not exists and try_setting_ids is true - if try_setting_ids: - for doc in _docs: - if "_id" not in doc: - doc["_id"] = uuid.uuid4().hex - - data = orce_bytes(json.dumps({"docs": _docs})) - params = {"all_or_nothing": "true" if transaction else "false"} - - (resp, results) = self.resource.post("_bulk_docs", data=data, - params=params) - - for result, doc in zip(results, _docs): - if "rev" in result: - doc['_rev'] = result['rev'] - - return _docs - - def all(self, wrapper=None, flat=None, as_list=False, **kwargs): - """ - Execute a builtin view for get all documents. - - :param wrapper: wrap result into a specific class. - :param as_list: return a list of results instead of a - default lazy generator. - :param flat: get a specific field from a object instead - of a complete object. - - .. versionadded: 1.4 - Add as_list parameter. - Add flat parameter. - - :returns: generator object - """ - - params = {"include_docs": "true"} - params.update(kwargs) - - data = None - - if "keys" in params: - data = {"keys": params.pop("keys")} - data = force_bytes(json.dumps(data)) - - params = encode_view_options(params) - if data: - (resp, result) = self.resource.post( - "_all_docs", params=params, data=data) - else: - (resp, result) = self.resource.get("_all_docs", params=params) - - if wrapper is None: - def wrapper(doc): return doc - - if flat is not None: - def wrapper(doc): return doc[flat] - - def _iterate(): - for row in result["rows"]: - yield wrapper(row) - - if as_list: - return list(_iterate()) - return _iterate() - - def cleanup(self): - """ - Execute a cleanup operation. - """ - (r, result) = self.resource('_view_cleanup').post() - return result - - def commit(self): - """ - Send commit message to server. - """ - (resp, result) = self.resource.post('_ensure_full_commit') - return result - - def compact(self): - """ - Send compact message to server. Compacting write-heavy databases - should be avoided, otherwise the process may not catch up with - the writes. Read load has no effect. - """ - (r, result) = self.resource("_compact").post() - return result - - def compact_view(self, ddoc): - """ - Execute compact over design view. - - :raises: :py:exc:`~pycouchdb.exceptions.NotFound` - if a view does not exists. - """ - (r, result) = self.resource("_compact", ddoc).post() - return result - - def revisions(self, doc_id, status='available', params=None, **kwargs): - """ - Get all revisions of one document. - - :param doc_id: document id - :param status: filter of revision status, set empty to list all - :raises: :py:exc:`~pycouchdb.exceptions.NotFound` - if a view does not exists. - - :returns: generator object - """ - if params: - warnings.warn("params parameter is now deprecated in favor to" - "**kwargs usage.", DeprecationWarning) - - if params is None: - params = {} - - params.update(kwargs) - - if not params.get('revs_info'): - params['revs_info'] = 'true' - - resource = self.resource(doc_id) - (resp, result) = resource.get(params=params) - if resp.status_code == 404: - raise NotFound("Document id `{0}` not found".format(doc_id)) - - for rev in result['_revs_info']: - if status and rev['status'] == status: - yield self.get(doc_id, rev=rev['rev']) - elif not status: - yield self.get(doc_id, rev=rev['rev']) - - def delete_attachment(self, doc, filename): - """ - Delete attachment by filename from document. - - .. versionchanged:: 1.2 - Now returns a new document instead of modify the original. - - :param doc: document dict - :param filename: name of attachment. - :raises: :py:exc:`~pycouchdb.exceptions.Conflict` - if save with wrong revision. - :returns: doc - """ - - _doc = copy.deepcopy(doc) - resource = self.resource(_doc['_id']) - - (resp, result) = resource.delete( - filename, params={'rev': _doc['_rev']}) - if resp.status_code == 404: - raise NotFound("filename {0} not found".format(filename)) - - if resp.status_code > 205: - raise Conflict(result['reason']) - - _doc['_rev'] = result['rev'] - try: - del _doc['_attachments'][filename] - - if not _doc['_attachments']: - del _doc['_attachments'] - except KeyError: - pass - - return _doc - - def get_attachment(self, doc, filename, stream=False, **kwargs): - """ - Get attachment by filename from document. - - :param doc: document dict - :param filename: attachment file name. - :param stream: setup streaming output (default: False) - - .. versionchanged: 1.5 - Add stream parameter for obtain very large attachments - without load all file to the memory. - - :returns: binary data or - """ - - params = {"rev": doc["_rev"]} - params.update(kwargs) - - r, result = self.resource(doc['_id']).get(filename, stream=stream, - params=params) - if stream: - return _StreamResponse(r) - - return r.content - - def put_attachment(self, doc, content, filename=None, content_type=None): - """ - Put a attachment to a document. - - .. versionchanged:: 1.2 - Now returns a new document instead of modify the original. - - :param doc: document dict. - :param content: the content to upload, either a file-like object or - bytes - :param filename: the name of the attachment file; if omitted, this - function tries to get the filename from the file-like - object passed as the `content` argument value - :raises: :py:exc:`~pycouchdb.exceptions.Conflict` - if save with wrong revision. - :raises: ValueError - :returns: doc - """ - - if filename is None: - if hasattr(content, 'name'): - filename = os.path.basename(content.name) - else: - raise ValueError('no filename specified for attachment') - - if content_type is None: - content_type = ';'.join( - filter(None, mimetypes.guess_type(filename))) - - headers = {"Content-Type": content_type} - resource = self.resource(doc['_id']) - - (resp, result) = resource.put( - filename, data=content, params={'rev': doc['_rev']}, headers=headers) - - if resp.status_code < 206: - return self.get(doc["_id"]) - - raise Conflict(result['reason']) - - def one(self, name, flat=None, wrapper=None, **kwargs): - """ - Execute a design document view query and returns a first - result. - - :param name: name of the view (eg: docidname/viewname). - :param wrapper: wrap result into a specific class. - :param flat: get a specific field from a object instead - of a complete object. - - .. versionadded: 1.4 - - :returns: object or None - """ - - params = {"limit": 1} - params.update(kwargs) - - path = _path_from_name(name, '_view') - data = None - - if "keys" in params: - data = {"keys": params.pop('keys')} - - if data: - data = force_bytes(json.dumps(data)) - - params = encode_view_options(params) - result = list(self._query(self.resource(*path), wrapper=wrapper, - flat=flat, params=params, data=data)) - - return result[0] if len(result) > 0 else None - - def _query(self, resource, data=None, params=None, headers=None, - flat=None, wrapper=None): - - if data is None: - (resp, result) = resource.get(params=params, headers=headers) - else: - (resp, result) = resource.post( - data=data, params=params, headers=headers) - - if wrapper is None: - def wrapper(row): return row - - if flat is not None: - def wrapper(row): return row[flat] - - for row in result["rows"]: - yield wrapper(row) - - def query(self, name, wrapper=None, flat=None, as_list=False, **kwargs): - """ - Execute a design document view query. - - :param name: name of the view (eg: docidname/viewname). - :param wrapper: wrap result into a specific class. - :param as_list: return a list of results instead of a - default lazy generator. - :param flat: get a specific field from a object instead - of a complete object. - - .. versionadded: 1.4 - Add as_list parameter. - Add flat parameter. - - :returns: generator object - """ - params = copy.copy(kwargs) - path = _path_from_name(name, '_view') - data = None - - if "keys" in params: - data = {"keys": params.pop('keys')} - - if data: - data = force_bytes(json.dumps(data)) - - params = encode_view_options(params) - result = self._query(self.resource(*path), wrapper=wrapper, - flat=flat, params=params, data=data) - - if as_list: - return list(result) - return result - - def changes_feed(self, feed_reader, **kwargs): - """ - Subscribe to changes feed of couchdb database. - - Note: this method is blocking. - - - :param feed_reader: callable or :py:class:`~BaseFeedReader` - instance - - .. versionadded: 1.5 - """ - - object = self - _listen_feed(object, "_changes", feed_reader, **kwargs) - - def changes_list(self, **kwargs): - """ - Obtain a list of changes from couchdb. - - .. versionadded: 1.5 - """ - - (resp, result) = self.resource("_changes").get(params=kwargs) - return result['last_seq'], result['results'] - - def find(self, selector, wrapper=None, **kwargs): - """ - Execute Mango querys using _find. - - :param selector: data to search - :param wrapper: wrap result into a specific class. - - """ - path = '_find' - data = force_bytes(json.dumps(selector)) - - (resp, result) = self.resource.post(path, data=data, params=kwargs) - - if wrapper is None: - def wrapper(doc): return doc - - for doc in result["docs"]: - yield wrapper(doc) - - def index(self, index_doc, **kwargs): - path = '_index' - data = force_bytes(json.dumps(index_doc)) - - (resp, result) = self.resource.post(path, data=data, params=kwargs) - - return result diff --git a/src/couch/exceptions.py b/src/couch/exceptions.py deleted file mode 100644 index d7e037b..0000000 --- a/src/couch/exceptions.py +++ /dev/null @@ -1,38 +0,0 @@ -# -*- coding: utf-8 -*- -# Based on py-couchdb (https://github.com/histrio/py-couchdb) - - -class Error(Exception): - pass - - -class UnexpectedError(Error): - pass - - -class FeedReaderExited(Error): - pass - - -class ApiError(Error): - pass - - -class GenericError(ApiError): - pass - - -class Conflict(ApiError): - pass - - -class NotFound(ApiError): - pass - - -class BadRequest(ApiError): - pass - - -class AuthenticationFailed(ApiError): - pass diff --git a/src/couch/feedreader.py b/src/couch/feedreader.py deleted file mode 100644 index aac51d3..0000000 --- a/src/couch/feedreader.py +++ /dev/null @@ -1,52 +0,0 @@ -# -*- coding: utf-8 -*- -# Based on py-couchdb (https://github.com/histrio/py-couchdb) -from __future__ import annotations - -class BaseFeedReader: - """ - Base interface class for changes feed reader. - """ - - def __call__(self, db) -> BaseFeedReader: - self.db = db - return self - - def on_message(self, message: str) -> None: - """ - Callback method that is called when change - message is received from couchdb. - - :param message: change object - :returns: None - """ - - raise NotImplementedError() - - def on_close(self) -> None: - """ - Callback method that is received when connection - is closed with a server. By default, does nothing. - """ - pass - - def on_heartbeat(self) -> None: - """ - Callback method invoked when a hearbeat (empty line) is received - from the _changes stream. Override this to purge the reader's internal - buffers (if any) if it waited too long without receiving anything. - """ - pass - - -class SimpleFeedReader(BaseFeedReader): - """ - Simple feed reader that encapsule any callable in - a valid feed reader interface. - """ - - def __call__(self, db, callback) -> BaseFeedReader: - self.callback = callback - return super(SimpleFeedReader, self).__call__(db) - - def on_message(self, message: str) -> None: - self.callback(message, db=self.db) diff --git a/src/couch/resource.py b/src/couch/resource.py deleted file mode 100644 index f110c8d..0000000 --- a/src/couch/resource.py +++ /dev/null @@ -1,139 +0,0 @@ -# -*- coding: utf-8 -*- -# Based on py-couchdb (https://github.com/histrio/py-couchdb) - -from __future__ import annotations -from __future__ import unicode_literals -from typing import Union, Tuple, Dict, Any - -import json -import requests - -from .utils import ( - urljoin, - as_json, - force_bytes, -) -from .exceptions import ( - GenericError, - NotFound, - BadRequest, - Conflict, - AuthenticationFailed, -) - - -class Resource: - def __init__(self, base_url: str, full_commit: bool = True, session: Union[requests.sessions.Session, None] = None, - credentials: Union[Tuple[str, str], None] = None, authmethod: str = "session", verify: bool = False) -> None: - - self.base_url = base_url -# self.verify = verify - - if not session: - self.session = requests.session() - - self.session.headers.update({"accept": "application/json", - "content-type": "application/json"}) - self._authenticate(credentials, authmethod) - - if not full_commit: - self.session.headers.update({'X-Couch-Full-Commit': 'false'}) - else: - self.session = session - self.session.verify = verify - - def _authenticate(self, credentials: Union[Tuple[str, str], None], method: str) -> None: - if not credentials: - return - - if method == "session": - data = {"name": credentials[0], "password": credentials[1]} - - post_url = urljoin(self.base_url, "_session") - r = self.session.post(post_url, data=force_bytes(json.dumps(data))) - if r and r.status_code != 200: - raise AuthenticationFailed() - - elif method == "basic": - self.session.auth = credentials - - else: - raise RuntimeError("Invalid authentication method") - - def __call__(self, *path: str) -> Resource: - base_url = urljoin(self.base_url, *path) - return self.__class__(base_url, session=self.session) - - def _check_result(self, response, result) -> None: - try: - error = result.get('error', None) - reason = result.get('reason', None) - except AttributeError: - error = None - reason = '' - - # This is here because couchdb can return http 201 - # but containing a list of conflict errors - if error == 'conflict' or error == "file_exists": - raise Conflict(reason or "Conflict") - - if response.status_code > 205: - if response.status_code == 404 or error == 'not_found': - raise NotFound(reason or 'Not found') - elif error == 'bad_request': - raise BadRequest(reason or "Bad request") - raise GenericError(result) - - - def request(self, method, path: Union[str, None], params=None, data=None, - headers=None, stream=False, **kwargs) -> Tuple[requests.models.Response, Union[Dict[str, Any], None]]: - - if headers is None: - headers = {} - - headers.setdefault('Accept', 'application/json') - - if path: - if not isinstance(path, (list, tuple)): - path = [path] - url = urljoin(self.base_url, *path) - else: - url = self.base_url - - response = self.session.request(method, url, stream=stream, - data=data, params=params, - headers=headers, **kwargs) - # Ignore result validation if - # request is with stream mode - - if stream and response.status_code < 400: - result = None - self._check_result(response, result) - else: - result = as_json(response) - - if result is None: - return response, result - - if isinstance(result, list): - for res in result: - self._check_result(response, res) - else: - self._check_result(response, result) - - return response, result - - def get(self, path: Union[str, None] = None, **kwargs: Any) -> Tuple[requests.models.Response, Union[Dict[str, Any], None]]: - return self.request("GET", path, **kwargs) - - def put(self, path: Union[str, None] = None, **kwargs: Any) -> Tuple[requests.models.Response, Union[Dict[str, Any], None]]: - return self.request("PUT", path, **kwargs) - - def post(self, path: Union[str, None] = None, **kwargs: Any) -> Tuple[requests.models.Response, Union[Dict[str, Any], None]]: - return self.request("POST", path, **kwargs) - - def delete(self, path: Union[str, None] = None, **kwargs: Any) -> Tuple[requests.models.Response, Union[Dict[str, Any], None]]: - return self.request("DELETE", path, **kwargs) - - def head(self, path: Union[str, None] = None, **kwargs: Any) -> Tuple[requests.models.Response, Union[Dict[str, Any], None]]: - return self.request("HEAD", path, **kwargs) diff --git a/src/couch/utils.py b/src/couch/utils.py deleted file mode 100644 index b3e5aa3..0000000 --- a/src/couch/utils.py +++ /dev/null @@ -1,144 +0,0 @@ -# -*- coding: utf-8 -*- -# Based on py-couchdb (https://github.com/histrio/py-couchdb) - -from typing import Tuple, Union, Dict, List, Any -import json -import sys - -import requests -from urllib.parse import unquote as _unquote -from urllib.parse import urlunsplit, urlsplit - -from functools import reduce - -URLSPLITTER = '/' - - -json_encoder = json.JSONEncoder() - - -def extract_credentials(url: str) -> Tuple[str, Union[Tuple[str, str], None]]: - """ - Extract authentication (user name and password) credentials from the - given URL. - - >>> extract_credentials('http://localhost:5984/_config/') - ('http://localhost:5984/_config/', None) - >>> extract_credentials('http://joe:secret@localhost:5984/_config/') - ('http://localhost:5984/_config/', ('joe', 'secret')) - >>> extract_credentials('http://joe%40example.com:secret@' - ... 'localhost:5984/_config/') - ('http://localhost:5984/_config/', ('joe@example.com', 'secret')) - """ - parts = urlsplit(url) - netloc = parts[1] - if '@' in netloc: - creds, netloc = netloc.split('@') - credentials = tuple(_unquote(i) for i in creds.split(':')) - parts_list = list(parts) - parts_list[1] = netloc - return urlunsplit(parts_list), (credentials[0], credentials[1]) - - parts_list = list(parts) - return urlunsplit(parts_list), None - - -def _join(head: str, tail: str) -> str: - parts = [head.rstrip(URLSPLITTER), tail.lstrip(URLSPLITTER)] - return URLSPLITTER.join(parts) - - -def urljoin(base: str, *path: str) -> str: - """ - Assemble a uri based on a base, any number of path segments, and query - string parameters. - - >>> urljoin('http://example.org', '_all_dbs') - 'http://example.org/_all_dbs' - - A trailing slash on the uri base is handled gracefully: - - >>> urljoin('http://example.org/', '_all_dbs') - 'http://example.org/_all_dbs' - - And multiple positional arguments become path parts: - - >>> urljoin('http://example.org/', 'foo', 'bar') - 'http://example.org/foo/bar' - - >>> urljoin('http://example.org/', 'foo/bar') - 'http://example.org/foo/bar' - - >>> urljoin('http://example.org/', 'foo', '/bar/') - 'http://example.org/foo/bar/' - - >>> urljoin('http://example.com', 'org.couchdb.user:username') - 'http://example.com/org.couchdb.user:username' - """ - return reduce(_join, path, base) - -# Probably bugs here -def as_json(response: requests.models.Response) -> Union[Dict[str, Any], None, str]: - if "application/json" in response.headers['content-type']: - response_src = response.content.decode('utf-8') - print(response.content) - if response.content != b'': - ret: Dict[str, Any] = json.loads(response_src) - return ret - else: - print("fff") - print("fff") - print(type(response_src)) - return response_src - return None - -def _path_from_name(name: str, type: str) -> List[str]: - """ - Expand a 'design/foo' style name to its full path as a list of - segments. - - >>> _path_from_name("_design/test", '_view') - ['_design', 'test'] - >>> _path_from_name("design/test", '_view') - ['_design', 'design', '_view', 'test'] - """ - if name.startswith('_'): - return name.split('/') - design, name = name.split('/', 1) - return ['_design', design, type, name] - - -def encode_view_options(options: Dict[str, Any]) -> Dict[str, Any]: - """ - Encode any items in the options dict that are sent as a JSON string to a - view/list function. - - >>> opts = {'key': 'foo', "notkey":"bar"} - >>> res = encode_view_options(opts) - >>> res["key"], res["notkey"] - ('"foo"', 'bar') - - >>> opts = {'startkey': 'foo', "endkey":"bar"} - >>> res = encode_view_options(opts) - >>> res['startkey'], res['endkey'] - ('"foo"', '"bar"') - """ - retval = {} - - for name, value in options.items(): - if name in ('key', 'startkey', 'endkey'): - value = json_encoder.encode(value) - retval[name] = value - return retval - - -def force_bytes(data: Union[str, bytes], encoding: str = "utf-8") -> bytes: - if isinstance(data, str): - data = data.encode(encoding) - return data - - -def force_text(data: Union[str, bytes], encoding: str = "utf-8") -> str: - if isinstance(data, bytes): - data = data.decode(encoding) - return data diff --git a/src/quickstart_test.sh b/src/quickstart_test.sh deleted file mode 100755 index 9254271..0000000 --- a/src/quickstart_test.sh +++ /dev/null @@ -1,67 +0,0 @@ -# Usage: ./quickstart_test.sh [-v] [-c] [-- <args to pytest>] - -export COUCHDB_NAME=unittest -export COUCHDB_HOSTNAME=localhost -export COUCHDB_USER=test -export COUCHDB_PASSWORD=test - -export DOCKER_JWT_PUBKEY_PATH="`pwd`/test/unittest_cert/" -export JWT_PUBKEY_PATH="`pwd`/test/unittest_cert/public.pem" - -virtualenv=no -couchdb=no - -while getopts ":vc" flag -do - case "$flag" in - v) virtualenv=yes;; - c) couchdb=yes;; - esac -done - -if [ -d test/unittest_cert ]; then - rm -r test/unittest_cert -fi - -if [ $virtualenv == "yes" ]; then - shift - if [ -d test/unittest_venv ]; then - rm -r test/unittest_venv - fi - - virtualenv test/unittest_venv - source test/unittest_venv/bin/activate - pip3 install -r ../requirements.txt -fi - -if [ $couchdb == "yes" ]; then - shift - docker run -it -p 6123:5984 --rm -d --name unittest_couchdb -e COUCHDB_USER=$COUCHDB_USER -e COUCHDB_PASSWORD=$COUCHDB_PASSWORD couchdb - - docker inspect unittest_couchdb > /dev/null - - if (( $? != 0 )); then - echo "Failed to start CouchDB container." - exit - fi - - export COUCHDB_PORT=6123 -fi - -mkdir test/unittest_cert - -cat <<EOF > test/unittest_cert/public.pem ------BEGIN PUBLIC KEY----- -MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEGHX8ipqVWtr49TXyX0f/L4GPhEpg -N0Erzy7hHkXVrkgKpnHSRLYWgbW4rscLoJAJeEv7Be5iH0TM8l09w8Q3wQ== ------END PUBLIC KEY----- -EOF - -shift -pytest --capture=tee-sys "$@" - -rm -r test/unittest_cert - -if [ $couchdb == "yes" ]; then - docker kill unittest_couchdb -fi diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/tests/__init__.py +++ /dev/null diff --git a/tests/test_api.py b/tests/test_api.py deleted file mode 100644 index 371fcf2..0000000 --- a/tests/test_api.py +++ /dev/null @@ -1,232 +0,0 @@ -import os -import time -import pytest -import random -import ipaddress - -from main import app -from fastapi import FastAPI -from fastapi import testclient - -client = testclient.TestClient(app) -JWT_TOKEN = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiJ9.eyJmcmVzaCI6ZmFsc2UsImlhdCI6MTY0MjE2ODkyMCwianRpIjoiNjM0NGFiNjEtMTIzZC00YWMyLTk3YjMtYmVlYTE2M2JiMWMwIiwidHlwZSI6ImFjY2VzcyIsInN1YiI6InVzZXIxIiwibmJmIjoxNjQyMTY4OTIwLCJyZWFkIjpbInN1bmV0LnNlIl0sIndyaXRlIjpbInN1bmV0LnNlIl19._bX9EHI9h0Vjw75UvYvypqaH3AmsgaATFSUSOT-cYLZHrfMlxios3emr7cyKw-OV_BN5h_XNyrMBV1gIoqAk3A' -JWT_HEADER = {'Authorization': f'Bearer {JWT_TOKEN}'} - - -def test_001(): - print("*** Adding document.") - - doc_port = random.randint(1, 65536) - doc_ip = str(ipaddress.IPv4Address(random.randint(1, 0xffffffff))) - doc_asn = str(doc_ip) + '_' + str(doc_port) - - json_data = { - 'ip': doc_ip, - 'port': doc_port, - 'whois_description': 'unittest', - 'asn': doc_asn, - 'asn_country_code': 'SE', - 'ptr': 'unittest.example.com', - 'abuse_mail': 'unittest@example.com', - 'domain': 'sunet.se', - 'timestamp_in_utc': '2021-06-21T14:06UTC', - 'producer_unique_keys': { - 'subject_cn': 'unittest', - 'subject_o': 'unittest', - 'full_name': 'unittest', - 'end_of_general_support': False, - 'cve_2021_21972': 'unittest', - 'cve_2021_21974': 'unittest', - 'cve_2021_21985': 'unittest' - } - } - - response = client.post("/sc/v0/add", headers=JWT_HEADER, json=json_data) - assert(response.status_code == 200) - assert(response.json()['status'] == 'success') - - response = client.get(f"/sc/v0/get?port={doc_port}", headers=JWT_HEADER) - assert(response.status_code == 200) - assert(response.json()['status'] == 'success') - assert(len(response.json()['docs']) == 1) - assert(response.json()['docs'][0]['port'] == doc_port) - - response = client.get(f"/sc/v0/get?asn={doc_asn}", headers=JWT_HEADER) - assert(response.status_code == 200) - assert(response.json()['status'] == 'success') - assert(len(response.json()['docs']) == 1) - assert(response.json()['docs'][0]['asn'] == doc_asn) - - response = client.get(f"/sc/v0/get?ip={doc_ip}", headers=JWT_HEADER) - assert(response.status_code == 200) - assert(response.json()['status'] == 'success') - assert(len(response.json()['docs']) == 1) - assert(response.json()['docs'][0]['ip'] == doc_ip) - - -def test_002(): - nr_documents = 100 - starttime = time.time() - - for i in range(nr_documents): - doc_port = random.randint(1, 65536) - doc_ip = str(ipaddress.IPv4Address(random.randint(1, 0xffffffff))) - doc_asn = str(doc_ip) + '_' + str(doc_port) - - json_data = { - 'ip': doc_ip, - 'port': doc_port, - 'whois_description': 'unittest', - 'asn': doc_asn, - 'asn_country_code': 'SE', - 'ptr': 'unittest.example.com', - 'abuse_mail': 'unittest@example.com', - 'domain': 'sunet.se', - 'timestamp_in_utc': '2021-06-21T14:06UTC', - 'producer_unique_keys': { - 'subject_cn': 'unittest', - 'subject_o': 'unittest', - 'full_name': 'unittest', - 'end_of_general_support': False, - 'cve_2021_21972': 'unittest', - 'cve_2021_21974': 'unittest', - 'cve_2021_21985': 'unittest' - } - } - - response = client.post( - "/sc/v0/add", headers=JWT_HEADER, json=json_data) - assert(response.status_code == 200) - assert(response.json()['status'] == 'success') - - response = client.get( - f"/sc/v0/get?port={doc_port}", headers=JWT_HEADER) - assert(response.status_code == 200) - assert(response.json()['status'] == 'success') - assert(len(response.json()['docs']) == 1) - assert(response.json()['docs'][0]['port'] == doc_port) - - response = client.get(f"/sc/v0/get?asn={doc_asn}", headers=JWT_HEADER) - assert(response.status_code == 200) - assert(response.json()['status'] == 'success') - assert(len(response.json()['docs']) == 1) - assert(response.json()['docs'][0]['asn'] == doc_asn) - - response = client.get(f"/sc/v0/get?ip={doc_ip}", headers=JWT_HEADER) - assert(response.status_code == 200) - assert(response.json()['status'] == 'success') - assert(len(response.json()['docs']) == 1) - assert(response.json()['docs'][0]['ip'] == doc_ip) - - stop_time = str(time.time() - starttime) - print(f"*** Adding {nr_documents} documents took {stop_time} seconds.") - - -def test_003(): - response = client.get("/sc/v0/get", headers=JWT_HEADER) - assert(response.status_code == 200) - - for doc in response.json()['docs']: - doc_id = doc['_id'] - - response_doc = client.get(f"/sc/v0/get/{doc_id}", headers=JWT_HEADER) - assert(response_doc.status_code == 200) - assert(response_doc.json()['status'] == 'success') - assert(type(response_doc.json()['docs']) == type(dict())) - assert(response_doc.json()['docs']['domain'] == 'sunet.se') - - -def test_004(): - response = client.get("/sc/v0/get?limit=1000", headers=JWT_HEADER) - assert(response.status_code == 200) - - nr_documents = len(response.json()['docs']) - starttime = time.time() - - for doc in response.json()['docs']: - doc_id = doc['_id'] - response_doc = client.delete( - f"/sc/v0/delete/{doc_id}", headers=JWT_HEADER) - assert(response_doc.status_code == 200) - assert(response_doc.json()['status'] == 'success') - response_doc = client.get( - f"/sc/v0/get/{doc_id}", headers=JWT_HEADER) - assert(response_doc.status_code == 200) - assert(response_doc.json()['status'] == 'success') - assert(response_doc.json()['docs'] == {}) - - stop_time = str(time.time() - starttime) - print(f"*** Removing {nr_documents} documents took {stop_time} seconds.") - - print("*** Removing document with invalid ID.") - response = client.delete( - "/sc/v0/delete/nonexistent", headers=JWT_HEADER) - assert(response.status_code == 400) - assert(response.json()['status'] == 'error') - - -def test_005(): - print("*** Accessing endpoints without JWT token...") - - response = client.get("/sc/v0/get?limit=1000") - assert(response.status_code == 400) - assert(response.json()['status'] == 'error') - - response = client.get("/sc/v0/get/unittest") - assert(response.status_code == 400) - assert(response.json()['status'] == 'error') - - response = client.post("/sc/v0/add", json={"data": "nothing"}) - assert(response.status_code == 200) - assert(response.json()['status'] == 'success') - - response = client.delete("/sc/v0/delete/unittest") - assert(response.status_code == 400) - assert(response.json()['status'] == 'error') - -def test_006(): - print("*** Add doc for unauthorized domain (this is allowed, currently)") - - doc_port = random.randint(1, 65536) - doc_ip = str(ipaddress.IPv4Address(random.randint(1, 0xffffffff))) - doc_asn = str(doc_ip) + '_' + str(doc_port) - - json_data = { - 'ip': doc_ip, - 'port': doc_port, - 'whois_description': 'unittest', - 'asn': doc_asn, - 'asn_country_code': 'SE', - 'ptr': 'unittest.example.com', - 'abuse_mail': 'unittest@example.com', - 'domain': 'sunet.se', - 'timestamp_in_utc': '2021-06-21T14:06UTC', - 'producer_unique_keys': { - 'subject_cn': 'unittest', - 'subject_o': 'unittest', - 'full_name': 'unittest', - 'end_of_general_support': False, - 'cve_2021_21972': 'unittest', - 'cve_2021_21974': 'unittest', - 'cve_2021_21985': 'unittest' - } - } - - response = client.post( - "/sc/v0/add", headers=JWT_HEADER, json=dict(json_data, domain="example.com") - ) - assert(response.status_code == 200) - assert(response.json()['status'] == 'success') - - print("*** Get doc for unauthorized domain (not allowed)") - doc_id = response.json()['docs']['_id'] - response = client.get(f"/sc/v0/get/{doc_id}", headers=JWT_HEADER) - assert(response.status_code == 400) - assert(response.json()['status'] == 'error') - assert(response.json()['message'] == 'User not authorized to view this object') - - print("*** Delete doc for unauthorized domain (not allowed)") - response = client.delete(f"/sc/v0/delete/{doc_id}", headers=JWT_HEADER) - assert(response.status_code == 400) - assert(response.json()['status'] == 'error') - assert(response.json()['message'] == 'User not authorized to delete this object') diff --git a/tools/feeder.py b/tools/feeder.py deleted file mode 100644 index cf52c36..0000000 --- a/tools/feeder.py +++ /dev/null @@ -1,169 +0,0 @@ -import requests -import random -import socket -import ipaddress -import datetime -import itertools -import json -import threading -import time - - -def random_string(iters=1): - full_string = '' - random_string = '' - - for i in range(iters): - for _ in range(10): - random_integer = random.randint(97, 97 + 26 - 1) - flip_bit = random.randint(0, 1) - random_integer = random_integer - 32 if flip_bit == 1 else random_integer - random_string += (chr(random_integer)) - full_string += random_string - - return full_string.lower() - - -def random_ipv4(): - randint = random.randint(1, 0xffffffff) - - return str(ipaddress.IPv4Address(randint)) - - -def random_ipv6(): - randint = random.randint(0, 2**128-1) - - return str(ipaddress.IPv6Address(randint)) - - -def random_port(): - return random.randint(1, 65536) - - -def random_asn(): - randint = random.randint(1, 65536) - - return f'AS{randint}' - - -def random_country(): - countries = [ - 'AF', 'AL', 'DZ', 'AD', 'AO', 'AI', 'AQ', 'AR', 'AM', 'AW', - 'AU', 'AT', 'AZ', 'BS', 'BH', 'BD', 'BB', 'BY', 'BE', 'BZ', 'BJ', 'BM', - 'BT', 'BO', 'BQ', 'BW', 'BR', 'BG', 'BI', 'KH', 'CM', 'CA', 'TD', 'CL', - 'CN', 'CO', 'HR', 'CU', 'CW', 'CY', 'CZ', 'DK', 'DJ', 'DM', 'EC', 'EG', - 'GU', 'ER', 'EE', 'SZ', 'ET', 'FJ', 'FI', 'FR', 'GA', 'GH', 'GE', 'DE', - 'GH', 'GI', 'GR', 'GL', 'GD', 'GP', 'GU', 'GT', 'GG', 'GN', 'GW', 'GY', - 'HT', 'HN', 'HU', 'IS', 'IN', 'ID', 'IQ', 'IE', 'IL', 'IT', 'JM', 'JP', - 'JE', 'JO', 'KZ', 'KE', 'KI', 'KW', 'KG', 'LV', 'LB', 'LS', 'LR', 'LY', - 'LI', 'LT', 'LU', 'MO', 'MG', 'MW', 'MY', 'MV', 'ML', 'MT', 'MQ', 'MR', - 'MU', 'YT', 'MX', 'MC', 'MN', 'ME', 'MS', 'MA', 'MZ', 'MM', 'NA', 'NR', - 'NP', 'NI', 'NG', 'NU', 'NO', 'OM', 'PK', 'PW', 'PA', 'PY', 'PE', 'PN', - 'PL', 'PT', 'QA', 'RO', 'RW', 'RE', 'WS', 'SN', 'RS', 'SC', 'SG', 'SK', - 'SI', 'SO', 'ES', 'SR', 'SE', 'CH', 'TJ', 'TH', 'TL', 'TG', 'TK', 'TO', - 'TN', 'TR', 'TM', 'TV', 'UG', 'UA', 'UY', 'UZ', 'VU', 'YE', 'ZM', 'ZW', - 'AX' - ] - - return random.choice(countries) - - -def random_tld(): - tlds = [ - '.ac', '.ad', '.ae', '.af', '.ag', '.ai', '.al', '.am', - '.ao', '.aq', '.ar', '.as', '.at', '.au', '.aw', '.ax', - '.az', '.ba', '.bb', '.bd', '.be', '.bf', '.bg', '.bh', - '.bi', '.bj', '.bm', '.bn', '.bo', '.bq', '.br', '.bs', - '.bt', '.bw', '.by', '.bz', '.ca', '.cc', '.cd', '.cf', - '.cg', '.ch', '.ci', '.ck', '.cl', '.cm', '.cn', 'Als', - '.co', '.cr', '.cu', '.cv', '.cw', '.cx', '.cy', '.cz', - '.de', '.dj', '.dk', '.dm', '.do', '.dz', '.ec', '.ee', - '.eg', '.eh', '.er', '.es', '.et', '.eu', '.fi', '.fj', - '.fk', '.fm', '.fo', '.fr', '.ga', '.gd', '.ge', '.gf', - '.gg', '.gh', '.gi', '.gl', '.gm', '.gn', '.gp', '.gq', - '.gr', '.gs', '.gt', '.gu', '.gw', '.gy', '.hk', '.hm', - '.hn', '.hr', '.ht', '.hu', '.id', '.ie', '.il', '.im', - '.in', '.io', '.iq', '.ir', '.is', '.it', '.je', '.jm', - '.jo', '.jp', '.ke', '.kg', '.kh', '.ki', '.km', '.kn', - '.kp', '.kr', '.kw', '.ky', '.kz', '.la', '.lb', '.lc', - '.li', '.lk', '.lr', '.ls', '.lt', '.lu', '.lv', '.ly', - '.ma', '.mc', '.md', '.me', '.mg', '.mh', '.mk', '.ml', - '.mm', '.mn', '.mo', '.mp', '.mq', '.mr', '.ms', '.mt', - '.mu', '.mv', '.mw', '.mx', '.my', '.mz', '.na', '.nc', - '.ne', '.nf', '.ng', '.ni', '.nl', '.no', 'Als', '.np', - '.nr', '.nu', '.nz', '.om', '.pa', '.pe', '.pf', '.pg', - '.ph', '.pk', '.pl', '.pm', '.pn', '.pr', '.ps', '.pt', - '.pw', '.py', '.qa', '.re', '.ro', '.rs', '.ru', '.rw', - '.sb', '.sc', '.sd', '.se', '.sg', '.sh', '.si', '.sk', - '.sl', '.sm', '.sn', '.so', '.sr', '.ss', '.st', '.su', - '.sv', '.sx', '.sy', '.sz', '.tc', '.td', '.tf', '.tg', - '.th', '.tj', '.tk', '.tl', '.tm', '.tn', '.to', '.tr', - '.tt', '.tv', '.tw', '.tz', '.ua', '.ug', '.uk', '.us', - '.uy', '.uz', '.va', '.vc', '.ve', '.vg', '.vi', '.vn', - '.vu', '.wf', '.ws', '.ye', '.yt', '.za', '.zm', '.zw' - ] - - return random.choice(tlds) - - -def get_timestamp(): - utc_datetime = datetime.datetime.utcnow() - - return utc_datetime.strftime("%Y-%m-%dT%H:%M:%SUTC") - - -def random_email(): - return random_string() + '@' + random_string(2) + random_tld() - - -def generate_blobs(nr_blobs=1): - blobs = [] - - for _ in range(nr_blobs): - blobs.append({ - 'ip': random_ipv4(), - 'port': random_port(), - 'whois_description': random_string(2), - 'asn': random_asn(), - 'asn_country_code': random_country(), - 'ptr': random_string(2), - 'abuse_mail': random_email(), - 'domain': "sunet.se", - 'timestamp_in_utc': get_timestamp(), - 'user_presentation': { - 'description': 'A presentation of the observation as a whole (optional)', - 'data': { - random_string(): { - 'display_name': random_string(), - 'data': random_string(), - 'description': random_string(), - }, - random_string(): { - 'display_name': random_string(), - 'data': random_string(), - }, - random_string(): { - 'display_name': random_string(), - 'data': random_string(), - }, - } - }, - }) - - return blobs - - -def send_blob(): - url = 'http://localhost:8000/sc/v0/add' - - blobs = generate_blobs(10000) - - start = time.process_time() - requests.post(url, json=blobs) - print(time.process_time() - start) - - -if __name__ == '__main__': - for _ in range(100): - send_blob() - print('Saved 10000 docs') diff --git a/tools/jwt_producer.py b/tools/jwt_producer.py deleted file mode 100644 index a024c13..0000000 --- a/tools/jwt_producer.py +++ /dev/null @@ -1,79 +0,0 @@ -import getopt -import sys - -import jwt - - -def usage(): - progname = sys.argv[0] - - print(f'Usage: {progname} [-p <path>] [-w <value>] [-r <value>]\n' + - ' -p <path to private key>\n' + - ' -w <write, comma separated list of domains>\n' + - ' -r <read, comma separated list of domains>\n' + - ' -e <print export statement') - sys.exit(0) - - -def create_token(private_key, write_domains, read_domains): - write_claim = list() - read_claim = list() - - if write_domains: - write_claim = write_domains.split(',') - - if read_domains: - read_claim = read_domains.split(',') - - payload = { - 'sub': 'test', - 'fresh': False, - 'type': 'access', - 'write': write_claim, - 'read': read_claim - } - - with open(private_key, "r") as fd: - key = fd.read() - - return jwt.encode(payload=payload, algorithm='ES256', key=key) - - -if __name__ == '__main__': - read_domains = None - write_domains = None - private_key = None - export = False - - try: - opts, args = getopt.getopt(sys.argv[1:], 'p:w:r:eh') - except getopt.GetoptError: - usage() - - for opt, arg in opts: - if opt == '-p': - private_key = arg - elif opt == '-w': - write_domains = arg - elif opt == '-r': - read_domains = arg - elif opt == '-e': - export = True - elif opt == '-h': - usage() - else: - usage() - - if not private_key: - usage() - - if not write_domains and not read_domains: - usage() - - token = create_token(private_key, write_domains, - read_domains) - - if export: - print(f'export JWT_TOKEN={token}') - else: - print(f'{token}') |