Split up full import to do it year by year to keep the results batches down.

3 files changed, 64 insertions, 9 deletions

diff --git a/meetingtools/apps/content/tasks.py b/meetingtools/apps/content/tasks.py
index a5a230c..16feb34 100644
--- a/meetingtools/apps/content/tasks.py
+++ b/meetingtools/apps/content/tasks.py
@@ -35,5 +35,18 @@ def import_all_content():
 
 #@periodic_task(run_every=crontab(hour="1", minute="0", day_of_week="*"))
 def timed_full_import():
+    years = [2009, 2010, 2011, 2012, 2013, 2014]
     for acc in ACCluster.objects.all():
-        import_acc(acc)
\ No newline at end of file
+        for year in years:
+            begin = datetime(year=year, month=1, day=1)
+            end = datetime(year=year, month=12, day=31)
+            with ac_api_client(acc) as api:
+                r = api.request('report-bulk-objects', {'filter-out-type': 'meeting',
+                                                        'filter-gte-date-modified': begin.isoformat(),
+                                                        'filter-lte-date-modified': end.isoformat()})
+                if r:
+                    nr = 0
+                    for row in r.et.xpath("//row"):
+                        Content.create(acc, api, row)
+                        nr += 1
+                    logging.info("%s: Imported %d objects." % (acc, nr))
\ No newline at end of file
diff --git a/meetingtools/apps/room/tasks.py b/meetingtools/apps/room/tasks.py
index 0b90b60..4fc0c74 100644
--- a/meetingtools/apps/room/tasks.py
+++ b/meetingtools/apps/room/tasks.py
@@ -220,12 +220,6 @@ def import_all_rooms():
         import_acc(acc,since=3700)
 
 
-#@periodic_task(run_every=crontab(hour="3", minute="0", day_of_week="*"))
-def timed_full_import():
-    for acc in ACCluster.objects.all():
-        import_acc(acc)
-
-
 def start_user_counts_poll(room,niter):
     poll_user_counts.apply_async(args=[room],kwargs={'niter': niter})
 
@@ -317,4 +311,28 @@ def clean_old_rooms():
         with ac_api_client(acc) as api:
             for room in Room.objects.filter(lastvisited__lt=then):
                 logging.debug("room %s was last used %s" % (room.name,humanize.naturalday(room.lastvisited)))
-                send_message.apply_async([room.creator,"You have an unused meetingroom at %s" % acc.name ,"Do you still need %s (%s)?" % (room.name,room.permalink())])
\ No newline at end of file
+                send_message.apply_async([room.creator,"You have an unused meetingroom at %s" % acc.name ,"Do you still need %s (%s)?" % (room.name,room.permalink())])
+
+
+
+#@periodic_task(run_every=crontab(hour="3", minute="0", day_of_week="*"))
+def timed_full_import():
+    years = [2009, 2010, 2011, 2012, 2013, 2014]
+    for acc in ACCluster.objects.all():
+        for year in years:
+            begin = datetime(year=year, month=1, day=1)
+            end = datetime(year=year, month=12, day=31)
+            with ac_api_client(acc) as api:
+                r = api.request('report-bulk-objects', {'filter-type': 'meeting',
+                                                        'filter-gte-date-modified': begin.isoformat(),
+                                                        'filter-lte-date-modified': end.isoformat()})
+                nr = 0
+                ne = 0
+                for row in r.et.xpath("//row"):
+                    try:
+                        _import_one_room(acc, api, row)
+                        nr += 1
+                    except Exception, ex:
+                        logging.error(ex)
+                        ne += 1
+                logging.info("%s: Imported %d rooms and got %d errors" % (acc, nr, ne))
\ No newline at end of file
diff --git a/meetingtools/apps/stats/tasks.py b/meetingtools/apps/stats/tasks.py
index 49e9abd..0a26d66 100644
--- a/meetingtools/apps/stats/tasks.py
+++ b/meetingtools/apps/stats/tasks.py
@@ -54,5 +54,29 @@ def _hourly_import():
 
 #@periodic_task(run_every=crontab(hour="5", minute="0", day_of_week="*"))
 def timed_full_import():
+    years = [2009, 2010, 2011, 2012, 2013, 2014]
     for acc in ACCluster.objects.all():
-        import_acc_sessions(acc, room_last_visited=True)
\ No newline at end of file
+        for year in years:
+            begin = datetime(year=year, month=1, day=1)
+            end = datetime(year=year, month=12, day=31)
+            with ac_api_client(acc) as api:
+                p = {'sort': 'asc', 'sort1': 'date-created', 'filter-type': 'meeting',
+                     'filter-gte-date-created': begin.isoformat(), 'filter-lte-date-created': end.isoformat()}
+                r = api.request('report-bulk-consolidated-transactions', p, True)
+                nr = 0
+                ne = 0
+                for tx in r.et.findall(".//row"):
+                    try:
+                        tx = UserMeetingTransaction.create(acc, tx)
+                        if tx:
+                            rooms = Room.objects.filter(sco=tx.sco)
+                            if len(rooms) == 1:
+                                room = rooms[0]
+                                if room.lastvisited is None or room.lastvisited < tx.date_closed:
+                                    room.lastvisited = tx.date_created
+                                    room.save()
+                            nr += 1
+                    except Exception, ex:
+                        logging.error(ex)
+                        ne += 1
+                logging.info("%s: Imported %d transactions with %d errors" % (acc, nr, ne))
\ No newline at end of file