Commit: 5f1d5e17934804045a8ab0b9f8e8b76ae44b783e - allura (git)

allura

Commit MetaInfo

• Revision: 5f1d5e17934804045a8ab0b9f8e8b76ae44b783e (tree)
• Time: 2011-04-01 02:54:06
• Author: Rick Copeland <rcopeland@geek...>
• Commiter: Rick Copeland

Log Message

[#1849] Make sure our reindex only tries to load 1024 project records at a time

Signed-off-by: Rick Copeland <[e-mail]>

Change Summary

• modified: Allura/allura/command/show_models.py
• modified: Allura/allura/lib/utils.py

Incremental Difference

--- a/Allura/allura/command/show_models.py
+++ b/Allura/allura/command/show_models.py
@@ -7,6 +7,7 @@ from ming.orm import MappedClass, mapper, ThreadLocalORMSession, session, state
 
 import allura.tasks.index_tasks
 from allura.lib.exceptions import CompoundError
+from allura.lib import utils
 from . import base
 
 class ShowModelsCommand(base.Command):
@@ -39,37 +40,38 @@ class ReindexCommand(base.Command):
         self.basic_setup()
         graph = build_model_inheritance_graph()
         if self.options.project:
-            projects = [ M.Project.query.get(shortname=self.options.project) ]
+            q_project = M.Project.query.find(dict(shortname=self.options.project))
         elif self.options.neighborhood:
             neighborhood_id = M.Neighborhood.query.get(
                 url_prefix='/%s/' % self.options.neighborhood)._id
-            projects = M.Project.query.find(dict(neighborhood_id=neighborhood_id))
+            q_project = M.Project.query.find(dict(neighborhood_id=neighborhood_id))
         else:
-            projects = M.Project.query.find()
+            q_project = M.Project.query.find()
         seen_dbs = set()
-        for p in projects:
-            if p.database_uri in seen_dbs: continue
-            seen_dbs.add(p.database_uri)
-            base.log.info('Reindex project %s', p.shortname)
-            c.project = p
-            for _, a_cls in dfs(M.Artifact, graph):
-                base.log.info('  %s', a_cls)
-                ref_ids = []
-                for a in a_cls.query.find():
+        for projects in utils.chunked_iterator(q_project):
+            for p in projects:
+                if p.database_uri in seen_dbs: continue
+                seen_dbs.add(p.database_uri)
+                base.log.info('Reindex project %s', p.shortname)
+                c.project = p
+                for _, a_cls in dfs(M.Artifact, graph):
+                    base.log.info('  %s', a_cls)
+                    ref_ids = []
+                    for a in a_cls.query.find():
+                        try:
+                            M.ArtifactReference.from_artifact(a)
+                        except:
+                            base.log.exception('Making ArtifactReference from %s', a)
+                            continue
+                        ref_ids.append(a.index_id())
+                    M.artifact_orm_session.clear()
                     try:
-                        M.ArtifactReference.from_artifact(a)
-                    except:
-                        base.log.exception('Making ArtifactReference from %s', a)
-                        continue
-                    ref_ids.append(a.index_id())
-                M.artifact_orm_session.clear()
-                try:
-                    allura.tasks.index_tasks.add_artifacts(ref_ids)
-                except CompoundError, err:
-                    base.log.exception('Error indexing artifacts:\n%r', err)
-                    base.log.error('%s', err.format_error())
-                M.main_orm_session.flush()
-                M.main_orm_session.clear()
+                        allura.tasks.index_tasks.add_artifacts(ref_ids)
+                    except CompoundError, err:
+                        base.log.exception('Error indexing artifacts:\n%r', err)
+                        base.log.error('%s', err.format_error())
+                    M.main_orm_session.flush()
+                    M.main_orm_session.clear()
 
 class EnsureIndexCommand(base.Command):
     min_args=0

--- a/Allura/allura/lib/utils.py
+++ b/Allura/allura/lib/utils.py
@@ -87,3 +87,15 @@ class StatsHandler(WatchedFileHandler):
             if v is not None)
         record.exc_info = None # Never put tracebacks in the rtstats log
         WatchedFileHandler.emit(self, record)
+
+def chunked_iterator(query, pagesize=1024):
+    page = 0
+    while True:
+        results = (
+            query
+            .skip(pagesize*page)
+            .limit(pagesize)
+            .all())
+        if not results: break
+        yield results
+        page += 1