improved collecting of media files by reducing the number of stat()

calls

diff --git a/motioneye.py b/motioneye.py
index dab704868bff8f291cfcdea20cfebb682806d3a7..0115016f943fe55eed571459d0b31edc4a71206c 100755 (executable)
--- a/motioneye.py
+++ b/motioneye.py
@@ -227,7 +227,7 @@ def _do_thumbnails():
         
         target_dir = camera_config['target_dir']
         
-        for full_path in mediafiles._list_media_files(target_dir, mediafiles._MOVIE_EXTS):
+        for (full_path, st) in mediafiles._list_media_files(target_dir, mediafiles._MOVIE_EXTS):  # @UnusedVariable
             mediafiles.make_movie_preview(camera_config, full_path)
     
     logging.info('done.')

diff --git a/src/handlers.py b/src/handlers.py
index dbb025e2c7d0af06915ab5f25e443e7e4d234e81..f604d634b400f001bbf053d0536bdd8af20f8c79 100644 (file)
--- a/src/handlers.py
+++ b/src/handlers.py
@@ -611,13 +611,11 @@ class PictureHandler(BaseHandler):
                     camera_config.get('@password'),
                     camera_config.get('@remote_camera_id'), on_response,
                     media_type='picture',
-                    prefix=self.get_argument('prefix', None),
-                    stat=self.get_argument('stat', None))
+                    prefix=self.get_argument('prefix', None))
         
         else:
             pictures = mediafiles.list_media(camera_config, media_type='picture',
-                    prefix=self.get_argument('prefix', None),
-                    stat=self.get_argument('stat', None))
+                    prefix=self.get_argument('prefix', None))
             
             self.finish_json({
                 'mediaList': pictures,
@@ -632,9 +630,6 @@ class PictureHandler(BaseHandler):
         if camera_id not in config.get_camera_ids():
             raise HTTPError(404, 'no such camera')
         
-        import time
-        time.sleep(100)
-        
         camera_config = config.get_camera(camera_id)
         if camera_config['@proto'] != 'v4l2':
             def on_response(response):
@@ -771,13 +766,11 @@ class MovieHandler(BaseHandler):
                     camera_config.get('@password'),
                     camera_config.get('@remote_camera_id'), on_response,
                     media_type='movie',
-                    prefix=self.get_argument('prefix', None),
-                    stat=self.get_argument('stat', None))
+                    prefix=self.get_argument('prefix', None))
         
         else:
             movies = mediafiles.list_media(camera_config, media_type='movie',
-                    prefix=self.get_argument('prefix', None),
-                    stat=self.get_argument('stat', None))
+                    prefix=self.get_argument('prefix', None))
             
             self.finish_json({
                 'mediaList': movies,

diff --git a/src/mediafiles.py b/src/mediafiles.py
index e7d45ed26fcbcf6c3bc4821753c3bdb4b1b0736b..579acdc12fb8ca26a76860a3e7bb218862429f5f 100644 (file)
--- a/src/mediafiles.py
+++ b/src/mediafiles.py
@@ -18,6 +18,7 @@
 import datetime
 import logging
 import os.path
+import stat
 import StringIO
 import subprocess
 
@@ -36,9 +37,11 @@ _MOVIE_EXTS = ['.avi', '.mp4']
 # tuples of (sequence, width, content)
 _current_pictures_cache = {}
 
+_previewless_movie_files = []
+
 
 def _list_media_files(dir, exts, prefix=None):
-    full_paths = []
+    media_files = []
     
     if prefix is not None:
         if prefix == 'ungrouped':
@@ -50,15 +53,16 @@ def _list_media_files(dir, exts, prefix=None):
                 continue
                 
             full_path = os.path.join(root, name)
-            if not os.path.isfile(full_path):
+            st = os.stat(full_path)
+            if not stat.S_ISREG(st.st_mode): # not a regular file
                 continue
-             
+
             full_path_lower = full_path.lower()
             if not [e for e in exts if full_path_lower.endswith(e)]:
                 continue
             
-            full_paths.append(full_path)
-    
+            media_files.append((full_path, st))
+
     else:    
         for root, dirs, files in os.walk(dir):  # @UnusedVariable
             for name in files:
@@ -66,23 +70,25 @@ def _list_media_files(dir, exts, prefix=None):
                     continue
                 
                 full_path = os.path.join(root, name)
-                if not os.path.isfile(full_path):
+                st = os.stat(full_path)
+                
+                if not stat.S_ISREG(st.st_mode): # not a regular file
                     continue
                  
                 full_path_lower = full_path.lower()
                 if not [e for e in exts if full_path_lower.endswith(e)]:
                     continue
                 
-                full_paths.append(full_path)
+                media_files.append((full_path, st))
     
-    return full_paths
+    return media_files
 
 
 def _remove_older_files(dir, moment, exts):
-    for full_path in _list_media_files(dir, exts):
+    for (full_path, st) in _list_media_files(dir, exts):
         # TODO files listed here may not belong to the given camera
         
-        file_moment = datetime.datetime.fromtimestamp(os.path.getmtime(full_path))
+        file_moment = datetime.datetime.fromtimestamp(st.st_mtime)
         if file_moment < moment:
             logging.debug('removing file %(path)s...' % {
                     'path': full_path})
@@ -159,33 +165,45 @@ def make_movie_preview(camera_config, full_path):
 
 
 def make_next_movie_preview():
+    global _previewless_movie_files
+    
     logging.debug('making preview for the next movie...')
     
-    for camera_id in config.get_camera_ids():
-        camera_config = config.get_camera(camera_id)
-        if camera_config.get('@proto') != 'v4l2':
-            continue
-        
-        target_dir = camera_config['target_dir']
+    if _previewless_movie_files:
+        (camera_config, path) = _previewless_movie_files.pop(0)
         
-        for full_path in _list_media_files(target_dir, _MOVIE_EXTS):
-            # TODO files listed here may not belong to the given camera
+        make_movie_preview(camera_config, path)
+    
+    else:
+        logging.debug('gathering movies without preview...')
         
-            if os.path.exists(full_path + '.thumb'):
+        count = 0
+        for camera_id in config.get_camera_ids():
+            camera_config = config.get_camera(camera_id)
+            if camera_config.get('@proto') != 'v4l2':
                 continue
             
-            logging.debug('found a movie without preview: %(path)s' % {
-                    'path': full_path})
-                
-            make_movie_preview(camera_config, full_path)
+            target_dir = camera_config['target_dir']
             
-            break
-        
-        else:
-            logging.debug('all movies have preview')
+            for (full_path, st) in _list_media_files(target_dir, _MOVIE_EXTS):  # @UnusedVariable
+                # TODO files listed here may not belong to the given camera
             
+                if os.path.exists(full_path + '.thumb'):
+                    continue
+                
+                logging.debug('found a movie without preview: %(path)s' % {
+                        'path': full_path})
+                
+                _previewless_movie_files.append((camera_config, full_path))
+                count += 1
+        
+        logging.debug('found %(count)d movies without preview' % {'count': count})    
+        
+        if count:
+            make_next_movie_preview()
+
 
-def list_media(camera_config, media_type, prefix=None, stat=False):
+def list_media(camera_config, media_type, prefix=None):
     target_dir = camera_config.get('target_dir')
 
     if media_type == 'picture':
@@ -194,34 +212,20 @@ def list_media(camera_config, media_type, prefix=None, stat=False):
     elif media_type == 'movie':
         exts = _MOVIE_EXTS
         
-    full_paths = _list_media_files(target_dir, exts=exts, prefix=prefix)
     media_files = []
     
-    for p in full_paths:
+    for (p, st) in _list_media_files(target_dir, exts=exts, prefix=prefix):
         path = p[len(target_dir):]
         if not path.startswith('/'):
             path = '/' + path
 
-        timestamp = None
-        size = None
-        
-        if stat:
-            try:
-                stat = os.stat(p)
-             
-            except Exception as e:
-                logging.error('stat call failed for file %(path)s: %(msg)s' % {
-                        'path': path, 'msg': unicode(e)})
-                 
-                continue
- 
-            timestamp = stat.st_mtime
-            size = stat.st_size
+        timestamp = st.st_mtime
+        size = st.st_size
         
         media_files.append({
             'path': path,
-            'momentStr': timestamp and utils.pretty_date_time(datetime.datetime.fromtimestamp(timestamp)),
-            'sizeStr': size and utils.pretty_size(size),
+            'momentStr': utils.pretty_date_time(datetime.datetime.fromtimestamp(timestamp)),
+            'sizeStr': utils.pretty_size(size),
             'timestamp': timestamp
         })
     

diff --git a/src/remote.py b/src/remote.py
index e284f39f5ef7f44e33949bc134b27397fe3e1b4b..856326e602f81fe018a40f065b05c706eb584450 100644 (file)
--- a/src/remote.py
+++ b/src/remote.py
@@ -203,7 +203,7 @@ def get_current_picture(host, port, username, password, camera_id, callback, wid
     http_client.fetch(request, on_response)
 
 
-def list_media(host, port, username, password, camera_id, callback, media_type, prefix=None, stat=False):
+def list_media(host, port, username, password, camera_id, callback, media_type, prefix=None):
     logging.debug('getting media list for remote camera %(id)s on %(host)s:%(port)s' % {
             'id': camera_id,
             'host': host,
@@ -213,9 +213,6 @@ def list_media(host, port, username, password, camera_id, callback, media_type,
     if prefix is not None:
         query['prefix'] = prefix
     
-    if stat:
-        query['stat'] = 'true'
-    
     request = _make_request(host, port, username, password, '/%(media_type)s/%(id)s/list/' % {
             'id': camera_id, 'media_type': media_type}, query=query)
     

diff --git a/static/js/main.js b/static/js/main.js
index c48a7aeec74fce01512f97cead68d3a96fbc0baa..c6c95fd722396c1c43ca56ccd6dc0fa74c757e3d 100644 (file)
--- a/static/js/main.js
+++ b/static/js/main.js
@@ -1474,7 +1474,7 @@ function runMediaDialog(cameraId, mediaType) {
             var previewImg = $('<img class="media-list-progress" src="' + staticUrl + 'img/modal-progress.gif"/>');
             mediaListDiv.append(previewImg);
             
-            var url = '/' + mediaType + '/' + cameraId + '/list/?prefix=' + (key || 'ungrouped')+ '&stat=true';
+            var url = '/' + mediaType + '/' + cameraId + '/list/?prefix=' + (key || 'ungrouped');
             ajax('GET', url, null, function (data) {
                 previewImg.remove();