improve overall build speed by caching the result of the pages() method

cactus/site.py

@@ -34,6 +34,7 @@
 from cactus.browser import browserReload, browserReloadCSS
 from cactus.utils import ipc
 
+from datetime import datetime
 
 logger = logging.getLogger(__name__)
 
@@ -44,6 +45,7 @@ class Site(SiteCompatibilityLayer):
     _path = None
     _parallel = PARALLEL_CONSERVATIVE  #TODO: Test me
     _static = None
+    _page_cache = []
 
     def __init__(self, path, config_paths=None, ui=None,
         PluginManagerClass=None, ExternalManagerClass=None, DeploymentEngineClass=None):
@@ -230,12 +232,16 @@ def build(self):
         """
 
         logger.debug("*** BUILD %s", self.path)
+        start = datetime.now()
 
         self.verify_url()
 
         # Reset the static content
         self._static = None
 
+        # Reset page cache
+        self._page_cache = []
+
         #TODO: Facility to reset the site, and reload config.
         #TODO: Currently, we can't build a site instance multiple times
         self.plugin_manager.reload()  # Reload in case we're running on the server # We're still loading twice!
@@ -280,6 +286,9 @@ def build(self):
             if os.path.isdir(static.pre_dir):
                 shutil.rmtree(static.pre_dir)
 
+        duration = datetime.now() - start
+        logger.info('Site built in {:.2f} seconds.'.format(duration.total_seconds()))
+
     def static(self):
         """
         Retrieve a list of static files for the site
@@ -339,23 +348,16 @@ def pages(self):
         List of pages.
         """
 
-        if not hasattr(self, "_page_cache"):
-            self._page_cache = {}
-
-        pages = []
+        if self._page_cache:
+            return self._page_cache
 
         for path in fileList(self.page_path, relative=True):
-
             if path.endswith("~"):
                 continue
+            logger.debug("Found page: %s", path)
+            self._page_cache.append(Page(self, path))
 
-            if path not in self._page_cache:
-                logger.debug("Found page: %s", path)
-                self._page_cache[path] = Page(self, path)
-
-            pages.append(self._page_cache[path])
-
-        return pages
+        return self._page_cache
 
     def _rebuild_should_ignore(self, file_path):
 

cactus/utils/filesystem.py

@@ -19,21 +19,18 @@ def fileList(paths, relative=False, folders=False):
 
     files = []
 
-    for path in paths:
-        for fileName in os.listdir(path):
-
-            if fileName.startswith('.'):
-                continue
-
-            filePath = os.path.join(path, fileName)
-
-            if os.path.isdir(filePath):
-                if folders:
-                    files.append(filePath)
-                files += fileList(filePath)
-            else:
-                files.append(filePath)
+    def append(directory, name):
+        if not name.startswith('.'):
+            path = os.path.join(directory, name)
+            files.append(path)
 
+    for path in paths:
+        for directory, dirnames, filenames in os.walk(path, followlinks=True):
+            if folders:
+                for dirname in dirnames:
+                    append(directory, dirname)
+            for filename in filenames:
+                append(directory, filename)
         if relative:
             files = map_apply(lambda x: x[len(path) + 1:], files)