From 2bc20262925d826c189fa0681bc591697c1a8a5e Mon Sep 17 00:00:00 2001 From: Calin Crisan Date: Sun, 1 Dec 2013 16:36:51 +0200 Subject: [PATCH] improved collecting of media files by reducing the number of stat() calls --- motioneye.py | 2 +- src/handlers.py | 15 ++------ src/mediafiles.py | 98 ++++++++++++++++++++++++----------------------- src/remote.py | 5 +-- static/js/main.js | 2 +- 5 files changed, 58 insertions(+), 64 deletions(-) diff --git a/motioneye.py b/motioneye.py index dab7048..0115016 100755 --- a/motioneye.py +++ b/motioneye.py @@ -227,7 +227,7 @@ def _do_thumbnails(): target_dir = camera_config['target_dir'] - for full_path in mediafiles._list_media_files(target_dir, mediafiles._MOVIE_EXTS): + for (full_path, st) in mediafiles._list_media_files(target_dir, mediafiles._MOVIE_EXTS): # @UnusedVariable mediafiles.make_movie_preview(camera_config, full_path) logging.info('done.') diff --git a/src/handlers.py b/src/handlers.py index dbb025e..f604d63 100644 --- a/src/handlers.py +++ b/src/handlers.py @@ -611,13 +611,11 @@ class PictureHandler(BaseHandler): camera_config.get('@password'), camera_config.get('@remote_camera_id'), on_response, media_type='picture', - prefix=self.get_argument('prefix', None), - stat=self.get_argument('stat', None)) + prefix=self.get_argument('prefix', None)) else: pictures = mediafiles.list_media(camera_config, media_type='picture', - prefix=self.get_argument('prefix', None), - stat=self.get_argument('stat', None)) + prefix=self.get_argument('prefix', None)) self.finish_json({ 'mediaList': pictures, @@ -632,9 +630,6 @@ class PictureHandler(BaseHandler): if camera_id not in config.get_camera_ids(): raise HTTPError(404, 'no such camera') - import time - time.sleep(100) - camera_config = config.get_camera(camera_id) if camera_config['@proto'] != 'v4l2': def on_response(response): @@ -771,13 +766,11 @@ class MovieHandler(BaseHandler): camera_config.get('@password'), camera_config.get('@remote_camera_id'), on_response, media_type='movie', - prefix=self.get_argument('prefix', None), - stat=self.get_argument('stat', None)) + prefix=self.get_argument('prefix', None)) else: movies = mediafiles.list_media(camera_config, media_type='movie', - prefix=self.get_argument('prefix', None), - stat=self.get_argument('stat', None)) + prefix=self.get_argument('prefix', None)) self.finish_json({ 'mediaList': movies, diff --git a/src/mediafiles.py b/src/mediafiles.py index e7d45ed..579acdc 100644 --- a/src/mediafiles.py +++ b/src/mediafiles.py @@ -18,6 +18,7 @@ import datetime import logging import os.path +import stat import StringIO import subprocess @@ -36,9 +37,11 @@ _MOVIE_EXTS = ['.avi', '.mp4'] # tuples of (sequence, width, content) _current_pictures_cache = {} +_previewless_movie_files = [] + def _list_media_files(dir, exts, prefix=None): - full_paths = [] + media_files = [] if prefix is not None: if prefix == 'ungrouped': @@ -50,15 +53,16 @@ def _list_media_files(dir, exts, prefix=None): continue full_path = os.path.join(root, name) - if not os.path.isfile(full_path): + st = os.stat(full_path) + if not stat.S_ISREG(st.st_mode): # not a regular file continue - + full_path_lower = full_path.lower() if not [e for e in exts if full_path_lower.endswith(e)]: continue - full_paths.append(full_path) - + media_files.append((full_path, st)) + else: for root, dirs, files in os.walk(dir): # @UnusedVariable for name in files: @@ -66,23 +70,25 @@ def _list_media_files(dir, exts, prefix=None): continue full_path = os.path.join(root, name) - if not os.path.isfile(full_path): + st = os.stat(full_path) + + if not stat.S_ISREG(st.st_mode): # not a regular file continue full_path_lower = full_path.lower() if not [e for e in exts if full_path_lower.endswith(e)]: continue - full_paths.append(full_path) + media_files.append((full_path, st)) - return full_paths + return media_files def _remove_older_files(dir, moment, exts): - for full_path in _list_media_files(dir, exts): + for (full_path, st) in _list_media_files(dir, exts): # TODO files listed here may not belong to the given camera - file_moment = datetime.datetime.fromtimestamp(os.path.getmtime(full_path)) + file_moment = datetime.datetime.fromtimestamp(st.st_mtime) if file_moment < moment: logging.debug('removing file %(path)s...' % { 'path': full_path}) @@ -159,33 +165,45 @@ def make_movie_preview(camera_config, full_path): def make_next_movie_preview(): + global _previewless_movie_files + logging.debug('making preview for the next movie...') - for camera_id in config.get_camera_ids(): - camera_config = config.get_camera(camera_id) - if camera_config.get('@proto') != 'v4l2': - continue - - target_dir = camera_config['target_dir'] + if _previewless_movie_files: + (camera_config, path) = _previewless_movie_files.pop(0) - for full_path in _list_media_files(target_dir, _MOVIE_EXTS): - # TODO files listed here may not belong to the given camera + make_movie_preview(camera_config, path) + + else: + logging.debug('gathering movies without preview...') - if os.path.exists(full_path + '.thumb'): + count = 0 + for camera_id in config.get_camera_ids(): + camera_config = config.get_camera(camera_id) + if camera_config.get('@proto') != 'v4l2': continue - logging.debug('found a movie without preview: %(path)s' % { - 'path': full_path}) - - make_movie_preview(camera_config, full_path) + target_dir = camera_config['target_dir'] - break - - else: - logging.debug('all movies have preview') + for (full_path, st) in _list_media_files(target_dir, _MOVIE_EXTS): # @UnusedVariable + # TODO files listed here may not belong to the given camera + if os.path.exists(full_path + '.thumb'): + continue + + logging.debug('found a movie without preview: %(path)s' % { + 'path': full_path}) + + _previewless_movie_files.append((camera_config, full_path)) + count += 1 + + logging.debug('found %(count)d movies without preview' % {'count': count}) + + if count: + make_next_movie_preview() + -def list_media(camera_config, media_type, prefix=None, stat=False): +def list_media(camera_config, media_type, prefix=None): target_dir = camera_config.get('target_dir') if media_type == 'picture': @@ -194,34 +212,20 @@ def list_media(camera_config, media_type, prefix=None, stat=False): elif media_type == 'movie': exts = _MOVIE_EXTS - full_paths = _list_media_files(target_dir, exts=exts, prefix=prefix) media_files = [] - for p in full_paths: + for (p, st) in _list_media_files(target_dir, exts=exts, prefix=prefix): path = p[len(target_dir):] if not path.startswith('/'): path = '/' + path - timestamp = None - size = None - - if stat: - try: - stat = os.stat(p) - - except Exception as e: - logging.error('stat call failed for file %(path)s: %(msg)s' % { - 'path': path, 'msg': unicode(e)}) - - continue - - timestamp = stat.st_mtime - size = stat.st_size + timestamp = st.st_mtime + size = st.st_size media_files.append({ 'path': path, - 'momentStr': timestamp and utils.pretty_date_time(datetime.datetime.fromtimestamp(timestamp)), - 'sizeStr': size and utils.pretty_size(size), + 'momentStr': utils.pretty_date_time(datetime.datetime.fromtimestamp(timestamp)), + 'sizeStr': utils.pretty_size(size), 'timestamp': timestamp }) diff --git a/src/remote.py b/src/remote.py index e284f39..856326e 100644 --- a/src/remote.py +++ b/src/remote.py @@ -203,7 +203,7 @@ def get_current_picture(host, port, username, password, camera_id, callback, wid http_client.fetch(request, on_response) -def list_media(host, port, username, password, camera_id, callback, media_type, prefix=None, stat=False): +def list_media(host, port, username, password, camera_id, callback, media_type, prefix=None): logging.debug('getting media list for remote camera %(id)s on %(host)s:%(port)s' % { 'id': camera_id, 'host': host, @@ -213,9 +213,6 @@ def list_media(host, port, username, password, camera_id, callback, media_type, if prefix is not None: query['prefix'] = prefix - if stat: - query['stat'] = 'true' - request = _make_request(host, port, username, password, '/%(media_type)s/%(id)s/list/' % { 'id': camera_id, 'media_type': media_type}, query=query) diff --git a/static/js/main.js b/static/js/main.js index c48a7ae..c6c95fd 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -1474,7 +1474,7 @@ function runMediaDialog(cameraId, mediaType) { var previewImg = $(''); mediaListDiv.append(previewImg); - var url = '/' + mediaType + '/' + cameraId + '/list/?prefix=' + (key || 'ungrouped')+ '&stat=true'; + var url = '/' + mediaType + '/' + cameraId + '/list/?prefix=' + (key || 'ungrouped'); ajax('GET', url, null, function (data) { previewImg.remove(); -- 2.39.5