From ea0442b4797fdc34414d1b0c3153298e214766f6 Mon Sep 17 00:00:00 2001 From: Leif Johansson Date: Wed, 31 Oct 2012 00:11:52 +0100 Subject: stats in DB --- meetingtools/apps/room/views.py | 6 +- meetingtools/apps/sco/models.py | 3 +- meetingtools/apps/stats/admin.py | 10 +++ meetingtools/apps/stats/forms.py | 8 +- meetingtools/apps/stats/management/__init__.py | 1 + .../apps/stats/management/commands/__init__.py | 1 + .../stats/management/commands/import_sessions.py | 20 +++++ meetingtools/apps/stats/models.py | 66 +++++++++++++++++ meetingtools/apps/stats/tasks.py | 33 +++++++++ meetingtools/apps/stats/views.py | 85 ++++++++++++++++++++++ meetingtools/context_processors.py | 5 +- meetingtools/multiresponse.py | 2 +- meetingtools/settings.py | 8 +- meetingtools/urls.py | 1 + 14 files changed, 237 insertions(+), 12 deletions(-) create mode 100644 meetingtools/apps/stats/admin.py create mode 100644 meetingtools/apps/stats/management/__init__.py create mode 100644 meetingtools/apps/stats/management/commands/__init__.py create mode 100644 meetingtools/apps/stats/management/commands/import_sessions.py create mode 100644 meetingtools/apps/stats/models.py create mode 100644 meetingtools/apps/stats/tasks.py (limited to 'meetingtools') diff --git a/meetingtools/apps/room/views.py b/meetingtools/apps/room/views.py index 92c9f8d..c19a242 100644 --- a/meetingtools/apps/room/views.py +++ b/meetingtools/apps/room/views.py @@ -319,7 +319,7 @@ def list_rooms(request,username=None): rooms = [] if user: - rooms = Room.objects.filter(creator=user).order_by('name').all() + rooms = Room.objects.filter(creator=user).order_by('name').all().prefetch_related("creator","sco","folder_sco","source_sco","deleted_sco") return respond_to(request, {'text/html':'apps/room/list.html'}, @@ -461,7 +461,7 @@ def _room2dict(request,room): # should not require login def list_by_tag(request,tn): tags = tn.split('+') - rooms = TaggedItem.objects.get_by_model(Room, tags).order_by('name').all() + rooms = TaggedItem.objects.get_by_model(Room, tags).order_by('name').all().prefetch_related("creator","sco","folder_sco","source_sco","deleted_sco") title = 'Rooms tagged with %s' % " and ".join(tags) return respond_to(request, {'text/html':'apps/room/list.html', @@ -547,7 +547,7 @@ def room_recordings(request,room): 'url': room.sco.acc.make_url(ar.urlpath), 'dl': room.sco.acc.make_url(ar.urlpath), 'date_created': ar.timecreated, - 'date_modified': ar.lastupdated} for ar in room.archives.all() + 'date_modified': ar.lastupdated} for ar in room.archives.all().prefetch_related("creator","sco","folder_sco","source_sco","deleted_sco") ] @login_required def recordings(request,rid): diff --git a/meetingtools/apps/sco/models.py b/meetingtools/apps/sco/models.py index 325baae..3607579 100644 --- a/meetingtools/apps/sco/models.py +++ b/meetingtools/apps/sco/models.py @@ -1,6 +1,7 @@ """ Abstract sco objects and utility methods """ +import logging __author__ = 'leifj' @@ -53,7 +54,7 @@ def get_sco(acc,sco_id): if sco is None: sco,created = ACObject.objects.get_or_create(acc=acc,sco_id=sco_id) assert sco is not None - cache.set(key,sco) + cache.set(key,sco,30) return sco def get_sco_shortcuts(acc,shortcut_id): diff --git a/meetingtools/apps/stats/admin.py b/meetingtools/apps/stats/admin.py new file mode 100644 index 0000000..f9ae86e --- /dev/null +++ b/meetingtools/apps/stats/admin.py @@ -0,0 +1,10 @@ +''' +Created on Jan 31, 2011 + +@author: leifj +''' + +from django.contrib import admin +from meetingtools.apps.stats.models import UserMeetingTransaction + +admin.site.register(UserMeetingTransaction) \ No newline at end of file diff --git a/meetingtools/apps/stats/forms.py b/meetingtools/apps/stats/forms.py index 9fa5225..fbe9d5b 100644 --- a/meetingtools/apps/stats/forms.py +++ b/meetingtools/apps/stats/forms.py @@ -3,9 +3,15 @@ Created on Jan 16, 2012 @author: leifj """ +from django.contrib.auth.models import User +from django.forms import ModelChoiceField from django.forms.forms import Form -from django.forms.fields import DateTimeField +from django.forms.fields import DateTimeField, CharField +from meetingtools.apps.sco.models import ACObject class StatCaledarForm(Form): + tags = CharField(required=False) + user = ModelChoiceField(User.objects,required=False) + sco = ModelChoiceField(ACObject.objects,required=False) begin = DateTimeField(required=False) end = DateTimeField(required=False) \ No newline at end of file diff --git a/meetingtools/apps/stats/management/__init__.py b/meetingtools/apps/stats/management/__init__.py new file mode 100644 index 0000000..3929ed7 --- /dev/null +++ b/meetingtools/apps/stats/management/__init__.py @@ -0,0 +1 @@ +__author__ = 'leifj' diff --git a/meetingtools/apps/stats/management/commands/__init__.py b/meetingtools/apps/stats/management/commands/__init__.py new file mode 100644 index 0000000..3929ed7 --- /dev/null +++ b/meetingtools/apps/stats/management/commands/__init__.py @@ -0,0 +1 @@ +__author__ = 'leifj' diff --git a/meetingtools/apps/stats/management/commands/import_sessions.py b/meetingtools/apps/stats/management/commands/import_sessions.py new file mode 100644 index 0000000..d98b081 --- /dev/null +++ b/meetingtools/apps/stats/management/commands/import_sessions.py @@ -0,0 +1,20 @@ +from optparse import make_option +from django.core.management import BaseCommand +from meetingtools.apps.stats.tasks import import_acc_sessions +from meetingtools.apps.cluster.models import ACCluster + +__author__ = 'leifj' + +class Command(BaseCommand): + + option_list = BaseCommand.option_list + ( + make_option('--since', + type='int', + dest='since', + default=0, + help='Import all sessions seconds ago'), + ) + + def handle(self, *args, **options): + for acc in ACCluster.objects.all(): + import_acc_sessions(acc,since=options['since']) \ No newline at end of file diff --git a/meetingtools/apps/stats/models.py b/meetingtools/apps/stats/models.py new file mode 100644 index 0000000..495bd65 --- /dev/null +++ b/meetingtools/apps/stats/models.py @@ -0,0 +1,66 @@ +import logging +from django.contrib.auth.models import User +from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned +from iso8601 import iso8601 +import tagging +from tagging.models import Tag +from meetingtools.apps.sco.models import ACObject, get_sco + +__author__ = 'leifj' + +from django.db import models +from django.db.models import fields, ForeignKey, DateTimeField, IntegerField +import lxml.etree as etree + +class UserMeetingTransaction(models.Model): + sco = ForeignKey(ACObject) + user = ForeignKey(User) + txid = IntegerField() + date_created = DateTimeField() + date_closed = DateTimeField() + + def seconds(self): + delta = self.date_closed - self.date_created + return delta.total_seconds() + + def __unicode__(self): + return "(%d) %d seconds in %s" % (self.txid,self.seconds(),self.sco) + + @staticmethod + def create(acc,row): + txid = int(row.get('transaction-id')) + sco_id = int(row.get('sco-id')) + + status = row.findtext("status") + if not status or status != "completed": + logging.debug("Ignoring transaction %s" % etree.tostring(row)) + return + + txo = None + try: + txo = UserMeetingTransaction.objects.get(sco__acc=acc,txid=txid) + except MultipleObjectsReturned,ex: + logging.error(ex) + except ObjectDoesNotExist: + login = row.findtext("login") + if not login: + raise ValueError("No user for transaction %d" % txid) + user,created = User.objects.get_or_create(username=login) + date_created=iso8601.parse_date(row.findtext("date-created")) + #date_created = date_created.replace(tzinfo=None) + date_closed=iso8601.parse_date(row.findtext("date-closed")) + #date_close = date_closed.replace(tzinfo=None) + txo = UserMeetingTransaction.objects.create(sco=get_sco(acc,sco_id), + txid=txid, + user=user, + date_created=date_created, + date_closed=date_closed) + tags = [] + for group in txo.user.groups.all(): + tags.append("group:%s" % group.name) + + (local,domain) = txo.user.username.split("@") + tags.append("domain:%s" % domain) + Tag.objects.update_tags(txo, ' '.join(tags)) + +tagging.register(UserMeetingTransaction) \ No newline at end of file diff --git a/meetingtools/apps/stats/tasks.py b/meetingtools/apps/stats/tasks.py new file mode 100644 index 0000000..fb4b5c7 --- /dev/null +++ b/meetingtools/apps/stats/tasks.py @@ -0,0 +1,33 @@ +import logging +from celery.schedules import crontab +from celery.task import periodic_task +from meetingtools.ac import ac_api_client +from meetingtools.apps.cluster.models import ACCluster +from datetime import datetime,timedelta +from meetingtools.apps.stats.models import UserMeetingTransaction + +__author__ = 'leifj' + +def import_acc_sessions(acc,since=0): + with ac_api_client(acc) as api: + p = {'sort': 'asc','sort1': 'date-created','filter-type': 'meeting'} + + begin = None + if since > 0: + begin = datetime.now()-timedelta(seconds=since) + begin = begin.replace(microsecond=0) + + if begin is not None: + p['filter-gte-date-created'] = begin.isoformat() + + r = api.request('report-bulk-consolidated-transactions',p,True) + for tx in r.et.findall(".//row"): + try: + UserMeetingTransaction.create(acc,tx) + except Exception,ex: + logging.error(ex) + +@periodic_task(run_every=crontab(hour="*", minute="*", day_of_week="*")) +def import_sessions(since=3700): + for acc in ACCluster.objects.all(): + import_acc_sessions(acc,since) \ No newline at end of file diff --git a/meetingtools/apps/stats/views.py b/meetingtools/apps/stats/views.py index 252b827..60628a2 100644 --- a/meetingtools/apps/stats/views.py +++ b/meetingtools/apps/stats/views.py @@ -3,10 +3,12 @@ Created on Jan 16, 2012 @author: leifj """ +import logging from django.contrib.auth.decorators import login_required from django.contrib.humanize.templatetags.humanize import naturalday from django.http import HttpResponseForbidden, HttpResponseBadRequest +from meetingtools.apps.stats.models import UserMeetingTransaction from meetingtools.ac import ac_api_client from iso8601 import iso8601 from time import mktime @@ -19,6 +21,9 @@ def _iso2datesimple(iso): (date,rest) = iso.split("T") return date +def _dt2ts(dt): + return mktime(dt.timetuple())*1000 + def _iso2ts(iso): return mktime(iso8601.parse_date(iso).timetuple())*1000 @@ -126,6 +131,7 @@ def user_minutes_api(request,username=None): #logging.debug("midnight: %d (%d)" % (ts_date_ts,ts_closed)) ms = (ts_closed - ts_date_ts) #logging.debug("nms: %d" % ms) + if curdate is not None and ms > 0: series.append([_date_ts(curdate),int(ms/60000)]) @@ -133,6 +139,85 @@ def user_minutes_api(request,username=None): return json_response({'data': sorted(series,key=lambda x: x[0]), 'rooms': len(rc.keys()), 'minutes': int(t_ms/60000)},request) @login_required +def tagged_minutes_api(request): + form = StatCaledarForm(request.GET) # convenient way to parse dates + if not form.is_valid(): + return HttpResponseBadRequest() + + tags = filter(lambda x: bool(x), form.cleaned_data['tags'].strip().split("+")) + sco = form.cleaned_data['sco'] + begin = form.cleaned_data['begin'] + end = form.cleaned_data['end'] + user = form.cleaned_data['user'] + + qs = UserMeetingTransaction.objects + + if user: + qs = qs.filter(user=user) + if sco: + qs = qs.filter(sco=sco) + if begin: + qs = qs.filter(date_created__gt=begin) + if end: + qs = qs.filter(date_closed__lt=end) + + if len(tags) > 0: + qs = UserMeetingTransaction.tagged.with_all(tags,qs) + + series = [] + d_created = None + d_closed = None + ms = 0 + curdate = None + t_ms = 0 + rc = {} + uc = {} + + for tx in qs.all().prefetch_related("sco").prefetch_related("user"): + rc[tx.sco.id] = True + uc[tx.user.username] = True + ts_created = _dt2ts(tx.date_created) + ts_closed = _dt2ts(tx.date_closed) + + d1 = tx.date_created + if d_created is None: + d_created = d1 + + d2 = tx.date_closed + if d_closed is None: + d_closed = d2 + + if curdate is None: + curdate = d1 + + if curdate != d1: + series.append([_dt2ts(curdate),int(ms/60)]) + ms = 0 + curdate = d1 + + if d1 == d2: #same date + diff = (ts_closed - ts_created) + ms += diff + t_ms += diff + else: # meeting spanned midnight + ts_date_ts = _dt2ts(d2) + ms += ts_date_ts - ts_created + series.append([_dt2ts(d1),int(ms/60)]) + t_ms += ms + curdate = d2 + ms = (ts_closed - ts_date_ts) + + if curdate is not None and ms > 0: + series.append([_date_ts(curdate),int(ms/60)]) + + return json_response({'data': sorted(series,key=lambda x: x[0]), + 'rooms': len(rc.keys()), + 'begin': naturalday(begin), + 'end': naturalday(end), + 'users': len(uc.keys()), + 'minutes': int(t_ms/60)},request) + +@login_required def domain_minutes_api(request,domain): with ac_api_client(request) as api: p = {'sort': 'asc','sort1': 'date-created','filter-type': 'meeting'} diff --git a/meetingtools/context_processors.py b/meetingtools/context_processors.py index f284867..f20b3fb 100644 --- a/meetingtools/context_processors.py +++ b/meetingtools/context_processors.py @@ -22,4 +22,7 @@ def theme(request): return _w(ctx) def misc_urls(request): - return {'LOGIN_URL': settings.LOGIN_URL,'BASE_URL':base_url(request)} \ No newline at end of file + return {'LOGIN_URL': settings.LOGIN_URL,'BASE_URL':base_url(request)} + +def request(request): + return {'request': request} \ No newline at end of file diff --git a/meetingtools/multiresponse.py b/meetingtools/multiresponse.py index 42139b5..1510760 100644 --- a/meetingtools/multiresponse.py +++ b/meetingtools/multiresponse.py @@ -30,7 +30,7 @@ def make_response_dict(request,d=dict()): if request.user.is_authenticated(): d['user'] = request.user - ctx = RequestContext(request,d,[context_processors.theme,context_processors.misc_urls]) + ctx = RequestContext(request,d,[context_processors.theme,context_processors.misc_urls,context_processors.request]) print repr(ctx['theme']) return ctx diff --git a/meetingtools/settings.py b/meetingtools/settings.py index dac6890..0bb0ca1 100644 --- a/meetingtools/settings.py +++ b/meetingtools/settings.py @@ -56,6 +56,7 @@ USE_I18N = True # If you set this to False, Django will not format dates, numbers and # calendars according to the current locale USE_L10N = True +USE_TZ = True STATIC_ROOT = "%s/static" % BASE_DIR STATIC_URL = "/static/" @@ -149,18 +150,15 @@ djcelery.setup_loader() NOREPLY = "no-reply@sunet.se" AUTH_PROFILE_MODULE = "userprofile.UserProfile" -from django.conf import settings -from saml2 import BINDING_HTTP_POST, BINDING_HTTP_REDIRECT - LOGIN_URL = '/saml2/sp/login/' LOGIN_REDIRECT_URL = "/rooms" -AUTHENTICATION_BACKENDS += ('asgard.saml.Saml2Backend',) - AUTO_REMOTE_SUPERUSERS = ['leifj@nordu.net'] try: from asgard.loader import * DEBUG=True + + AUTHENTICATION_BACKENDS += ('asgard.saml.Saml2Backend',) except ImportError,ex: print ex diff --git a/meetingtools/urls.py b/meetingtools/urls.py index 7e39f5c..cc0c3fb 100644 --- a/meetingtools/urls.py +++ b/meetingtools/urls.py @@ -55,6 +55,7 @@ urlpatterns = patterns('', (r'^widget/?\+?(.*)$','meetingtools.apps.room.views.widget'), # Uncomment the admin/doc line below to enable admin documentation: # (r'^admin/doc/', include('django.contrib.admindocs.urls')), + (r'^api/stats/?','meetingtools.apps.stats.views.tagged_minutes_api'), (r'^api/stats/user/(.*)$','meetingtools.apps.stats.views.user_minutes_api'), (r'^api/stats/domain/(.+)$','meetingtools.apps.stats.views.domain_minutes_api'), (r'^api/stats/room/(\d+)$','meetingtools.apps.stats.views.room_minutes_api'), -- cgit v1.1