diff options
| -rw-r--r-- | meetingtools/apps/content/admin.py | 40 | ||||
| -rw-r--r-- | meetingtools/apps/content/migrations/0001_initial.py | 109 | ||||
| -rw-r--r-- | meetingtools/apps/content/migrations/__init__.py | 0 | ||||
| -rw-r--r-- | meetingtools/apps/content/models.py | 136 | ||||
| -rw-r--r-- | meetingtools/apps/content/tasks.py | 75 |
5 files changed, 303 insertions, 57 deletions
diff --git a/meetingtools/apps/content/admin.py b/meetingtools/apps/content/admin.py new file mode 100644 index 0000000..af80348 --- /dev/null +++ b/meetingtools/apps/content/admin.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +__author__ = 'lundberg' + +from datetime import date +from django.contrib import admin +from meetingtools.apps.content.models import Content + + +class YearLastViewedFilter(admin.SimpleListFilter): + title = 'year last visited' + parameter_name = 'year' + + def lookups(self, request, model_admin): + return ( + ('None', 'None'), + ('2009', '2009'), + ('2010', '2010'), + ('2011', '2011'), + ('2012', '2012'), + ('2013', '2013'), + ('2014', '2014'), + ) + + def queryset(self, request, queryset): + if self.value() == 'None': + return queryset.filter(lastviewed=None) + try: + year = int(self.value()) + return queryset.filter(lastviewed__gte=date(year, 1, 1), lastviewed__lte=date(year, 12, 31)) + except TypeError: + return None + + +class ContentAdmin(admin.ModelAdmin): + date_hierarchy = 'created' + search_fields = ['creator__username', 'name', 'sco__sco_id'] + list_display = ('created', 'creator', 'name', 'type', 'urlpath', 'sco', 'bytecount', 'modified', 'views', 'lastviewed') + list_filter = (YearLastViewedFilter, 'sco__acc',) + +admin.site.register(Content, ContentAdmin)
\ No newline at end of file diff --git a/meetingtools/apps/content/migrations/0001_initial.py b/meetingtools/apps/content/migrations/0001_initial.py new file mode 100644 index 0000000..2e80d6e --- /dev/null +++ b/meetingtools/apps/content/migrations/0001_initial.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'Content' + db.create_table('content_content', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('creator', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'], null=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=128)), + ('sco', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['sco.ACObject'], unique=True, null=True)), + ('folder_sco', self.gf('django.db.models.fields.related.ForeignKey')(related_name='content_folders', null=True, to=orm['sco.ACObject'])), + ('type', self.gf('django.db.models.fields.CharField')(max_length=128)), + ('urlpath', self.gf('django.db.models.fields.CharField')(max_length=128)), + ('bytecount', self.gf('django.db.models.fields.BigIntegerField')()), + ('created', self.gf('django.db.models.fields.DateTimeField')()), + ('modified', self.gf('django.db.models.fields.DateTimeField')()), + ('views', self.gf('django.db.models.fields.IntegerField')()), + ('lastviewed', self.gf('django.db.models.fields.DateTimeField')(null=True, blank=True)), + )) + db.send_create_signal('content', ['Content']) + + + def backwards(self, orm): + # Deleting model 'Content' + db.delete_table('content_content') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'cluster.accluster': { + 'Meta': {'object_name': 'ACCluster'}, + 'api_url': ('django.db.models.fields.URLField', [], {'max_length': '200'}), + 'cross_domain_sso': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'default_template_sco_id': ('django.db.models.fields.IntegerField', [], {'unique': 'True', 'blank': 'True'}), + 'domain_match': ('django.db.models.fields.TextField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '200'}), + 'user': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'content.content': { + 'Meta': {'object_name': 'Content'}, + 'bytecount': ('django.db.models.fields.BigIntegerField', [], {}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'creator': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True'}), + 'folder_sco': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'content_folders'", 'null': 'True', 'to': "orm['sco.ACObject']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'lastviewed': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'modified': ('django.db.models.fields.DateTimeField', [], {}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'sco': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['sco.ACObject']", 'unique': 'True', 'null': 'True'}), + 'type': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'urlpath': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'views': ('django.db.models.fields.IntegerField', [], {}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'sco.acobject': { + 'Meta': {'unique_together': "(('acc', 'sco_id'),)", 'object_name': 'ACObject'}, + 'acc': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['cluster.ACCluster']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_deleted': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'lastupdated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'sco_id': ('django.db.models.fields.IntegerField', [], {}), + 'timecreated': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}) + } + } + + complete_apps = ['content']
\ No newline at end of file diff --git a/meetingtools/apps/content/migrations/__init__.py b/meetingtools/apps/content/migrations/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/meetingtools/apps/content/migrations/__init__.py diff --git a/meetingtools/apps/content/models.py b/meetingtools/apps/content/models.py new file mode 100644 index 0000000..58233ac --- /dev/null +++ b/meetingtools/apps/content/models.py @@ -0,0 +1,136 @@ +# -*- coding: utf-8 -*- +__author__ = 'lundberg' + +from django.db import models +from django.db.models.fields import CharField, BigIntegerField, IntegerField +from django.db.models.fields.related import ForeignKey +from django.contrib.auth.models import User +from meetingtools.apps.sco.models import ACObject, get_sco +import tagging +from tagging.models import Tag +from django.core.cache import cache + + +class Content(models.Model): + creator = ForeignKey(User, editable=False, null=True) + name = CharField(max_length=128) + sco = ForeignKey(ACObject, editable=False, null=True, unique=True) + folder_sco = ForeignKey(ACObject, null=True, related_name="content_folders") + type = CharField(max_length=128) + urlpath = CharField(max_length=128) + bytecount = BigIntegerField() + created = models.DateTimeField() + modified = models.DateTimeField() + views = IntegerField() + lastviewed = models.DateTimeField(blank=True, null=True) + + class Meta: + verbose_name_plural = 'Content' + + def __unicode__(self): + return "%s (sco_id=%s,folder_sco_id=%s,urlpath=%s)" % (self.name, self.sco.sco_id, self.folder_sco.sco_id, + self.urlpath) + + @staticmethod + def by_sco(sco): + return Content.objects.get(sco=sco) + + @staticmethod + def by_id(acc, sco_id): + return Content.by_sco(get_sco(acc, sco_id)) + + @staticmethod + def by_name(acc, name): + return Content.objects.get(sco__acc=acc, name=name) + + def download_url(self): + url = self.sco.acc.url + return "%s%s" % (url, self.urlpath) + + @staticmethod + def create(acc, api, row): + sco_id = row.get('sco-id') + byte_count = api.get_byte_count(sco_id) + if byte_count or byte_count == 0: + sco_element = api.get_sco_info(sco_id) + if not sco_element.get('source-sco-id'): # Object is not a reference + views = api.get_sco_views(sco_id) + + owner = api.get_owner(sco_element.findtext('url-path')) + if not owner: + owner = get_owner_by_folder(api, acc, sco_element) + try: + domain = owner['login'].split('@')[1] + user, created = User.objects.get_or_create(username=owner['login']) + if created: + user.set_unusable_password() + except IndexError: + user = None + domain = None + + datecreated = row.findtext('date-created') + if not datecreated: + datecreated = row.findtext('date-modified') + + content, created = Content.objects.get_or_create( + sco=get_sco(acc, sco_id), + creator=user, + name=row.findtext('name'), + folder_sco=get_sco(acc, sco_element.get('folder-id')), + type=row.get('icon'), + urlpath=row.findtext('url'), + bytecount=byte_count, + created=datecreated, + modified=row.findtext('date-modified'), + views=views['views'], + lastviewed=views['last-viewed-date'] + ) + if not created: + Content.objects.filter(sco=content.sco).update( + creator=user, + name=row.findtext('name'), + folder_sco=get_sco(acc, sco_element.get('folder-id')), + type=row.get('icon'), + urlpath=row.findtext('url'), + bytecount=byte_count, + modified=row.findtext('date-modified'), + views=views['views'], + lastviewed=views['last-viewed-date'] + ) + + if user and domain: + tags = [] + for group in user.groups.all(): + tags.append("group:%s" % group.name) + tags.append("domain:%s" % domain) + Tag.objects.update_tags(content, ' '.join(tags)) + +tagging.register(Content) + + +def get_owner_by_folder(api, acc, sco): + default_folders = ['Shared Templates', 'Shared Content', 'User Content', 'Shared Meetings', 'User Meetings', + '{tree-type-account-custom}', 'Forced Recordings', 'Chat Transcripts'] + key = 'ac:owner:%s/%s' % (acc, sco.get('sco-id')) + owner = cache.get(key) + if owner is None: + fid = sco.get('folder-id') + if not fid: + return None + folder_id = int(fid) + r = api.request('sco-info', {'sco-id': folder_id}, False) + if r.status_code() == 'no-data': + return None + parent = r.et.xpath("//sco")[0] + if parent is not None: + if parent.findtext('name') in default_folders: + owner = { + 'login': sco.findtext('name'), # To match api.get_owner + 'sco_id': sco.get('sco-id'), + } + else: + owner = get_owner_by_folder(api, acc, parent) + + if owner is not None: + cache.set(key, owner, 30) + return owner
\ No newline at end of file diff --git a/meetingtools/apps/content/tasks.py b/meetingtools/apps/content/tasks.py index ddee09e..a5a230c 100644 --- a/meetingtools/apps/content/tasks.py +++ b/meetingtools/apps/content/tasks.py @@ -2,44 +2,15 @@ __author__ = 'lundberg' from meetingtools.ac import ac_api_client -from django.core.cache import cache +from meetingtools.apps.content.models import Content import logging from datetime import datetime, timedelta - - -def get_owner(api, acc, sco): - default_folders = ['Shared Templates', 'Shared Content', 'User Content', 'Shared Meetings', 'User Meetings', - '{tree-type-account-custom}', 'Forced Recordings', 'Chat Transcripts'] - key = 'ac:owner:%s/%s' % (acc, sco.get('sco-id')) - owner = cache.get(key) - if owner is None: - fid = sco.get('folder-id') - if not fid: - return None - - folder_id = int(fid) - r = api.request('sco-info', {'sco-id': folder_id}, False) - if r.status_code() == 'no-data': - return None - - parent = r.et.xpath("//sco")[0] - if parent is not None: - if parent.findtext('name') in default_folders: - owner = { - 'login': sco.findtext('name'), # To match api.get_owner - 'sco_id': sco.get('sco-id'), - } - else: - owner = get_owner(api, acc, parent) - - if owner is not None: - cache.set(key, owner, 30) - - return owner +from celery.task import periodic_task +from celery.schedules import crontab +from meetingtools.apps.cluster.models import ACCluster def import_acc(acc, since=0): - result = [] with ac_api_client(acc) as api: if since > 0: then = datetime.now()-timedelta(seconds=since) @@ -51,28 +22,18 @@ def import_acc(acc, since=0): if r: nr = 0 for row in r.et.xpath("//row"): - sco_id = row.get('sco-id') - byte_count = api.get_byte_count(sco_id) - if byte_count or byte_count == 0: - sco_element = api.get_sco_info(sco_id) - if not sco_element.get('source-sco-id'): # Object is not a reference - owner = api.get_owner(sco_element.findtext('url-path')) - if not owner: - owner = get_owner(api, acc, sco_element) - views = api.get_sco_views(sco_id) - item = { - 'sco-id': sco_id, - 'folder-id': sco_element.get('folder-id'), - 'type': row.get('icon'), - 'name': row.findtext('name'), - 'byte_count': byte_count, - 'url-path': row.findtext('url'), - 'created': row.findtext('date-created'), - 'modified': row.findtext('date-modified'), - 'owner': owner, - 'views': views, - } - result.append(item) - nr += 1 + Content.create(acc, api, row) + nr += 1 logging.info("%s: Imported %d objects." % (acc, nr)) - return result + + +@periodic_task(run_every=crontab(hour="*", minute="*/15", day_of_week="*")) +def import_all_content(): + for acc in ACCluster.objects.all(): + import_acc(acc, since=960) + + +#@periodic_task(run_every=crontab(hour="1", minute="0", day_of_week="*")) +def timed_full_import(): + for acc in ACCluster.objects.all(): + import_acc(acc)
\ No newline at end of file |