summaryrefslogtreecommitdiff
path: root/meetingtools
diff options
context:
space:
mode:
authorJohan Lundberg <lundberg@nordu.net>2014-04-14 13:35:21 +0200
committerJohan Lundberg <lundberg@nordu.net>2014-04-14 13:35:21 +0200
commit4ae116f4091623f89df8a382694c301ffb712432 (patch)
tree33ee658c6f3b5261b29c41757e24e401a18cd66b /meetingtools
parente25b7d32251e5c255b1d7c133158b287c505e35a (diff)
Spread the full import to fewer months at a time.
Diffstat (limited to 'meetingtools')
-rw-r--r--meetingtools/apps/content/tasks.py2
-rw-r--r--meetingtools/apps/room/tasks.py35
-rw-r--r--meetingtools/apps/stats/tasks.py48
3 files changed, 44 insertions, 41 deletions
diff --git a/meetingtools/apps/content/tasks.py b/meetingtools/apps/content/tasks.py
index eef0e58..ad426ef 100644
--- a/meetingtools/apps/content/tasks.py
+++ b/meetingtools/apps/content/tasks.py
@@ -93,7 +93,7 @@ def get_domain_content(domain_tag):
#@periodic_task(run_every=crontab(hour="1", minute="0", day_of_week="*"))
def timed_full_import():
years = [2009, 2010, 2011, 2012, 2013, 2014]
- months = [(1, 7), (8, 12)] # Ugly hack as June does not have 31 days
+ months = [(1, 3), (4, 7), (8, 10), (9, 12)]
for acc in ACCluster.objects.all():
nr = 0
for year in years:
diff --git a/meetingtools/apps/room/tasks.py b/meetingtools/apps/room/tasks.py
index 4fc0c74..518669b 100644
--- a/meetingtools/apps/room/tasks.py
+++ b/meetingtools/apps/room/tasks.py
@@ -314,25 +314,26 @@ def clean_old_rooms():
send_message.apply_async([room.creator,"You have an unused meetingroom at %s" % acc.name ,"Do you still need %s (%s)?" % (room.name,room.permalink())])
-
#@periodic_task(run_every=crontab(hour="3", minute="0", day_of_week="*"))
def timed_full_import():
years = [2009, 2010, 2011, 2012, 2013, 2014]
+ months = [(1, 3), (4, 7), (8, 10), (9, 12)]
for acc in ACCluster.objects.all():
for year in years:
- begin = datetime(year=year, month=1, day=1)
- end = datetime(year=year, month=12, day=31)
- with ac_api_client(acc) as api:
- r = api.request('report-bulk-objects', {'filter-type': 'meeting',
- 'filter-gte-date-modified': begin.isoformat(),
- 'filter-lte-date-modified': end.isoformat()})
- nr = 0
- ne = 0
- for row in r.et.xpath("//row"):
- try:
- _import_one_room(acc, api, row)
- nr += 1
- except Exception, ex:
- logging.error(ex)
- ne += 1
- logging.info("%s: Imported %d rooms and got %d errors" % (acc, nr, ne)) \ No newline at end of file
+ for month in months:
+ begin = datetime(year=year, month=month[0], day=1)
+ end = datetime(year=year, month=month[1], day=31)
+ with ac_api_client(acc) as api:
+ r = api.request('report-bulk-objects', {'filter-type': 'meeting',
+ 'filter-gte-date-modified': begin.isoformat(),
+ 'filter-lte-date-modified': end.isoformat()})
+ nr = 0
+ ne = 0
+ for row in r.et.xpath("//row"):
+ try:
+ _import_one_room(acc, api, row)
+ nr += 1
+ except Exception, ex:
+ logging.error(ex)
+ ne += 1
+ logging.info("%s: Imported %d rooms and got %d errors" % (acc, nr, ne)) \ No newline at end of file
diff --git a/meetingtools/apps/stats/tasks.py b/meetingtools/apps/stats/tasks.py
index 77e7513..acbf76e 100644
--- a/meetingtools/apps/stats/tasks.py
+++ b/meetingtools/apps/stats/tasks.py
@@ -55,28 +55,30 @@ def _hourly_import():
#@periodic_task(run_every=crontab(hour="5", minute="0", day_of_week="*"))
def timed_full_import():
years = [2009, 2010, 2011, 2012, 2013, 2014]
+ months = [(1, 3), (4, 7), (8, 10), (9, 12)]
for acc in ACCluster.objects.all():
for year in years:
- begin = datetime(year=year, month=1, day=1)
- end = datetime(year=year, month=12, day=31)
- with ac_api_client(acc) as api:
- p = {'sort': 'asc', 'sort1': 'date-created', 'filter-type': 'meeting',
- 'filter-gte-date-created': begin.isoformat(), 'filter-lte-date-created': end.isoformat()}
- r = api.request('report-bulk-consolidated-transactions', p, False)
- nr = 0
- ne = 0
- for tx in r.et.findall(".//row"):
- try:
- tx = UserMeetingTransaction.create(acc, tx)
- if tx:
- rooms = Room.objects.filter(sco=tx.sco)
- if len(rooms) == 1:
- room = rooms[0]
- if room.lastvisited is None or room.lastvisited < tx.date_closed:
- room.lastvisited = tx.date_created
- room.save()
- nr += 1
- except Exception, ex:
- logging.error(ex)
- ne += 1
- logging.info("%s: Imported %d transactions with %d errors" % (acc, nr, ne)) \ No newline at end of file
+ for month in months:
+ begin = datetime(year=year, month=month[0], day=1)
+ end = datetime(year=year, month=month[1], day=31)
+ with ac_api_client(acc) as api:
+ p = {'sort': 'asc', 'sort1': 'date-created', 'filter-type': 'meeting',
+ 'filter-gte-date-created': begin.isoformat(), 'filter-lte-date-created': end.isoformat()}
+ r = api.request('report-bulk-consolidated-transactions', p, False)
+ nr = 0
+ ne = 0
+ for tx in r.et.findall(".//row"):
+ try:
+ tx = UserMeetingTransaction.create(acc, tx)
+ if tx:
+ rooms = Room.objects.filter(sco=tx.sco)
+ if len(rooms) == 1:
+ room = rooms[0]
+ if room.lastvisited is None or room.lastvisited < tx.date_closed:
+ room.lastvisited = tx.date_created
+ room.save()
+ nr += 1
+ except Exception, ex:
+ logging.error(ex)
+ ne += 1
+ logging.info("%s: Imported %d transactions with %d errors" % (acc, nr, ne)) \ No newline at end of file