From 05375adc97b0e460ff2f9398fb13cdc2c5524baa Mon Sep 17 00:00:00 2001 From: Calin Crisan Date: Sat, 11 Jan 2014 16:13:46 +0200 Subject: [PATCH] cleanup, thumbnailer and media listing are now executed in a separate subprocess to avoid server blocking --- motioneye.py | 109 +++++++++++++------------------------------- settings_default.py | 2 +- src/cleanup.py | 84 ++++++++++++++++++++++++++++++++++ src/handlers.py | 36 +++++++++------ src/mediafiles.py | 75 ++++++++++++++++++++---------- src/motionctl.py | 11 ++--- src/smbctl.py | 16 ------- src/thumbnailer.py | 83 +++++++++++++++++++++++++++++++++ src/update.py | 4 +- src/v4l2ctl.py | 20 ++++++++ 10 files changed, 300 insertions(+), 140 deletions(-) create mode 100644 src/cleanup.py delete mode 100644 src/smbctl.py create mode 100644 src/thumbnailer.py diff --git a/motioneye.py b/motioneye.py index 9f1e1bb..3474511 100755 --- a/motioneye.py +++ b/motioneye.py @@ -19,6 +19,7 @@ import datetime import inspect import logging +import multiprocessing import os.path import re import signal @@ -93,8 +94,10 @@ def _test_requirements(): def _configure_signals(): def bye_handler(signal, frame): - import tornado.ioloop + import cleanup import motionctl + import thumbnailer + import tornado.ioloop logging.info('interrupt signal received, shutting down...') @@ -102,16 +105,27 @@ def _configure_signals(): ioloop = tornado.ioloop.IOLoop.instance() if ioloop.running(): ioloop.stop() + logging.info('server stopped') - logging.info('server stopped') - + if thumbnailer.running(): + thumbnailer.stop() + logging.info('thumbnailer stopped') + + if cleanup.running(): + cleanup.stop() + logging.info('cleanup stopped') + if motionctl.running(): motionctl.stop() logging.info('motion stopped') + + def child_handler(signal, frame): + # this is required for the multiprocessing mechanism to work + multiprocessing.active_children() signal.signal(signal.SIGINT, bye_handler) signal.signal(signal.SIGTERM, bye_handler) - signal.signal(signal.SIGCHLD, signal.SIG_IGN) + signal.signal(signal.SIGCHLD, child_handler) def _configure_logging(): @@ -190,7 +204,6 @@ def _print_help(): print('available options: ') options = list(inspect.getmembers(settings)) - options.append(('THUMBNAILS', None)) for (name, value) in sorted(options): if name.upper() != name: @@ -211,28 +224,6 @@ def _print_help(): print('') -def _do_thumbnails(): - import config - import mediafiles - - logging.info('recreating thumbnails for all video files...') - - for camera_id in config.get_camera_ids(): - camera_config = config.get_camera(camera_id) - if camera_config.get('@proto') != 'v4l2': - continue - - logging.info('listing movie files for camera %(name)s' % { - 'name': camera_config['@name']}) - - target_dir = camera_config['target_dir'] - - for (full_path, st) in mediafiles._list_media_files(target_dir, mediafiles._MOVIE_EXTS): # @UnusedVariable - mediafiles.make_movie_preview(camera_config, full_path) - - logging.info('done.') - - def _start_server(): import tornado.ioloop import server @@ -261,7 +252,7 @@ def _start_motion(): except Exception as e: logging.error('failed to start motion: %(msg)s' % { - 'msg': unicode(e)}) + 'msg': unicode(e)}, exc_info=True) ioloop.add_timeout(datetime.timedelta(seconds=settings.MOTION_CHECK_INTERVAL), checker) @@ -269,69 +260,31 @@ def _start_motion(): def _start_cleanup(): - import tornado.ioloop - import mediafiles + import cleanup - ioloop = tornado.ioloop.IOLoop.instance() - - def do_cleanup(): - if ioloop._stopped: - return - - try: - mediafiles.cleanup_media('picture') - mediafiles.cleanup_media('movie') - - except Exception as e: - logging.error('failed to cleanup media files: %(msg)s' % { - 'msg': unicode(e)}) - - ioloop.add_timeout(datetime.timedelta(seconds=settings.CLEANUP_INTERVAL), do_cleanup) - - ioloop.add_timeout(datetime.timedelta(seconds=settings.CLEANUP_INTERVAL), do_cleanup) + cleanup.start() + logging.info('cleanup started') -def _start_movie_thumbnailer(): - import tornado.ioloop - import mediafiles - - ioloop = tornado.ioloop.IOLoop.instance() - - def do_next_movie_thumbail(): - if ioloop._stopped: - return - - try: - mediafiles.make_next_movie_preview() - - except Exception as e: - logging.error('failed to make movie thumbnail: %(msg)s' % { - 'msg': unicode(e)}) +def _start_thumbnailer(): + import thumbnailer - ioloop.add_timeout(datetime.timedelta(seconds=settings.THUMBNAILER_INTERVAL), do_next_movie_thumbail) - - ioloop.add_timeout(datetime.timedelta(seconds=settings.THUMBNAILER_INTERVAL), do_next_movie_thumbail) + thumbnailer.start() + logging.info('thumbnailer started') if __name__ == '__main__': if not _test_requirements(): - sys.exit(01) + sys.exit(-1) cmd = _configure_settings() _configure_signals() _configure_logging() - if cmd: - if cmd == 'thumbnails': - _do_thumbnails() - - else: - print('unknown command line option: ' + cmd) - sys.exit(-1) - - sys.exit(0) - _start_motion() _start_cleanup() - _start_movie_thumbnailer() + + if settings.THUMBNAILER_INTERVAL: + _start_thumbnailer() + _start_server() diff --git a/settings_default.py b/settings_default.py index 1109f51..c88f9f4 100644 --- a/settings_default.py +++ b/settings_default.py @@ -35,7 +35,7 @@ MOTION_CHECK_INTERVAL = 10 # interval in seconds at which the janitor is called to remove old pictures and movies CLEANUP_INTERVAL = 43200 -# interval in seconds at which the thumbnail mechanism runs +# interval in seconds at which the thumbnail mechanism runs (set to 0 to disable) THUMBNAILER_INTERVAL = 60 # timeout in seconds to wait for responses when contacting a remote server diff --git a/src/cleanup.py b/src/cleanup.py new file mode 100644 index 0000000..7ac6a39 --- /dev/null +++ b/src/cleanup.py @@ -0,0 +1,84 @@ + +# Copyright (c) 2013 Calin Crisan +# This file is part of motionEye. +# +# motionEye is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import datetime +import logging +import multiprocessing +import os +import signal +import tornado + +import mediafiles +import settings + + +_process = None + + +def start(): + if running(): + raise Exception('cleanup is already running') + + _run_process() + + +def stop(): + global _process + + if not running(): + raise Exception('cleanup is not running') + + _process.join(timeout=10) + if _process.is_alive(): + logging.error('cleanup process did not finish in time, killing it...') + os.kill(_process.pid, signal.SIGKILL) + + _process = None + + +def running(): + return _process is not None + + +def _run_process(): + global _process + + if not _process or not _process.is_alive(): # check that the previous process has finished + logging.debug('running cleanup process...') + + _process = multiprocessing.Process(target=_do_cleanup) + _process.start() + + # schedule the next call + ioloop = tornado.ioloop.IOLoop.instance() + ioloop.add_timeout(datetime.timedelta(seconds=settings.CLEANUP_INTERVAL), _run_process) + + +def _do_cleanup(): + # this will be executed in a separate subprocess + + # ignore the terminate and interrupt signals in this subprocess + signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGTERM, signal.SIG_IGN) + + try: + mediafiles.cleanup_media('picture') + mediafiles.cleanup_media('movie') + + except Exception as e: + logging.error('failed to cleanup media files: %(msg)s' % { + 'msg': unicode(e)}, exc_info=True) diff --git a/src/handlers.py b/src/handlers.py index 5229a6f..fe48233 100644 --- a/src/handlers.py +++ b/src/handlers.py @@ -615,14 +615,18 @@ class PictureHandler(BaseHandler): prefix=self.get_argument('prefix', None)) else: - pictures = mediafiles.list_media(camera_config, media_type='picture', - prefix=self.get_argument('prefix', None)) + def on_media_list(media_list): + if media_list is None: + return self.finish_json({'error': 'Failed to get pictures list.'}) + + self.finish_json({ + 'mediaList': media_list, + 'cameraName': camera_config['@name'] + }) - self.finish_json({ - 'mediaList': pictures, - 'cameraName': camera_config['@name'] - }) - + mediafiles.list_media(camera_config, media_type='picture', + callback=on_media_list, prefix=self.get_argument('prefix', None)) + @BaseHandler.auth() def download(self, camera_id, filename): logging.debug('downloading picture %(filename)s of camera %(id)s' % { @@ -771,14 +775,18 @@ class MovieHandler(BaseHandler): prefix=self.get_argument('prefix', None)) else: - movies = mediafiles.list_media(camera_config, media_type='movie', - prefix=self.get_argument('prefix', None)) + def on_media_list(media_list): + if media_list is None: + return self.finish_json({'error': 'Failed to get movies list.'}) + + self.finish_json({ + 'mediaList': media_list, + 'cameraName': camera_config['@name'] + }) + + mediafiles.list_media(camera_config, media_type='movie', + callback=on_media_list, prefix=self.get_argument('prefix', None)) - self.finish_json({ - 'mediaList': movies, - 'cameraName': camera_config['@name'] - }) - @BaseHandler.auth() def download(self, camera_id, filename): logging.debug('downloading movie %(filename)s of camera %(id)s' % { diff --git a/src/mediafiles.py b/src/mediafiles.py index 9e8d235..60697f8 100644 --- a/src/mediafiles.py +++ b/src/mediafiles.py @@ -17,10 +17,12 @@ import datetime import logging +import multiprocessing import os.path import stat import StringIO import subprocess +import tornado from PIL import Image @@ -37,6 +39,7 @@ _MOVIE_EXTS = ['.avi', '.mp4'] # tuples of (sequence, width, content) _current_pictures_cache = {} +# a cache list of paths to movies without preview _previewless_movie_files = [] @@ -86,8 +89,6 @@ def _list_media_files(dir, exts, prefix=None): def _remove_older_files(dir, moment, exts): for (full_path, st) in _list_media_files(dir, exts): - # TODO files listed here may not belong to the given camera - file_moment = datetime.datetime.fromtimestamp(st.st_mtime) if file_moment < moment: logging.debug('removing file %(path)s...' % { @@ -195,8 +196,6 @@ def make_next_movie_preview(): target_dir = camera_config['target_dir'] for (full_path, st) in _list_media_files(target_dir, _MOVIE_EXTS): # @UnusedVariable - # TODO files listed here may not belong to the given camera - if os.path.exists(full_path + '.thumb'): continue @@ -212,7 +211,7 @@ def make_next_movie_preview(): make_next_movie_preview() -def list_media(camera_config, media_type, prefix=None): +def list_media(camera_config, media_type, callback, prefix=None): target_dir = camera_config.get('target_dir') if media_type == 'picture': @@ -220,28 +219,58 @@ def list_media(camera_config, media_type, prefix=None): elif media_type == 'movie': exts = _MOVIE_EXTS - - media_files = [] - - for (p, st) in _list_media_files(target_dir, exts=exts, prefix=prefix): - path = p[len(target_dir):] - if not path.startswith('/'): - path = '/' + path - timestamp = st.st_mtime - size = st.st_size + # create a subprocess to retrieve media files + def do_list_media(pipe): + for (p, st) in _list_media_files(target_dir, exts=exts, prefix=prefix): + path = p[len(target_dir):] + if not path.startswith('/'): + path = '/' + path + + timestamp = st.st_mtime + size = st.st_size + + pipe.send({ + 'path': path, + 'momentStr': utils.pretty_date_time(datetime.datetime.fromtimestamp(timestamp)), + 'sizeStr': utils.pretty_size(size), + 'timestamp': timestamp + }) - media_files.append({ - 'path': path, - 'momentStr': utils.pretty_date_time(datetime.datetime.fromtimestamp(timestamp)), - 'sizeStr': utils.pretty_size(size), - 'timestamp': timestamp - }) + pipe.close() + + logging.debug('starting media listing process...') + + (parent_pipe, child_pipe) = multiprocessing.Pipe(duplex=False) + process = multiprocessing.Process(target=do_list_media, args=(child_pipe, )) + process.start() + + # poll the subprocess to see when it has finished + started = datetime.datetime.now() + def poll_process(): + ioloop = tornado.ioloop.IOLoop.instance() + if process.is_alive(): # not finished yet + now = datetime.datetime.now() + delta = now - started + if delta.seconds < 120: + ioloop.add_timeout(datetime.timedelta(seconds=0.1), poll_process) + + else: # process did not finish within 2 minutes + logging.error('timeout waiting for the media listing process to finish') + + callback(None) + + else: # finished + media_list = [] + while parent_pipe.poll(): + media_list.append(parent_pipe.recv()) + + logging.debug('media listing process has returned %(count)s files' % {'count': len(media_list)}) + + callback(media_list) - # TODO files listed here may not belong to the given camera + poll_process() - return media_files - def get_media_content(camera_config, path, media_type): target_dir = camera_config.get('target_dir') diff --git a/src/motionctl.py b/src/motionctl.py index b7aeb13..c73497f 100644 --- a/src/motionctl.py +++ b/src/motionctl.py @@ -51,8 +51,7 @@ def start(): log_file = open(motion_log_path, 'w') - process = subprocess.Popen(args, stdout=log_file, stderr=log_file, close_fds=True, - cwd=settings.CONF_PATH) + process = subprocess.Popen(args, stdout=log_file, stderr=log_file, close_fds=True, cwd=settings.CONF_PATH) # wait 2 seconds to see that the process has successfully started for i in xrange(20): # @UnusedVariable @@ -84,22 +83,22 @@ def stop(): # wait 5 seconds for the process to exit for i in xrange(50): # @UnusedVariable + os.waitpid(pid, os.WNOHANG) time.sleep(0.1) - os.kill(pid, 0) - + # send the KILL signal once os.kill(pid, signal.SIGKILL) # wait 2 seconds for the process to exit for i in xrange(20): # @UnusedVariable time.sleep(0.1) - os.kill(pid, 0) + os.waitpid(pid, os.WNOHANG) # the process still did not exit raise Exception('could not terminate the motion process') except OSError as e: - if e.errno != errno.ESRCH: + if e.errno != errno.ECHILD: raise diff --git a/src/smbctl.py b/src/smbctl.py deleted file mode 100644 index 28fb229..0000000 --- a/src/smbctl.py +++ /dev/null @@ -1,16 +0,0 @@ - -# Copyright (c) 2013 Calin Crisan -# This file is part of motionEye. -# -# motionEye is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . diff --git a/src/thumbnailer.py b/src/thumbnailer.py new file mode 100644 index 0000000..23fefd0 --- /dev/null +++ b/src/thumbnailer.py @@ -0,0 +1,83 @@ + +# Copyright (c) 2013 Calin Crisan +# This file is part of motionEye. +# +# motionEye is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import datetime +import logging +import multiprocessing +import os +import signal +import tornado + +import mediafiles +import settings + + +_process = None + + +def start(): + if running(): + raise Exception('thumbnailer is already running') + + _run_process() + + +def stop(): + global _process + + if not running(): + raise Exception('thumbnailer is not running') + + _process.join(timeout=10) + if _process.is_alive(): + logging.error('thumbnailer process did not finish in time, killing it...') + os.kill(_process.pid, signal.SIGKILL) + + _process = None + + +def running(): + return _process is not None + + +def _run_process(): + global _process + + if not _process or not _process.is_alive(): # check that the previous process has finished + logging.debug('running thumbnailer process...') + + _process = multiprocessing.Process(target=_do_next_movie_thumbail) + _process.start() + + # schedule the next call + ioloop = tornado.ioloop.IOLoop.instance() + ioloop.add_timeout(datetime.timedelta(seconds=settings.THUMBNAILER_INTERVAL), _run_process) + + +def _do_next_movie_thumbail(): + # this will be executed in a separate subprocess + + # ignore the terminate and interrupt signals in this subprocess + signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGTERM, signal.SIG_IGN) + + try: + mediafiles.make_next_movie_preview() + + except Exception as e: + logging.error('failed to make movie thumbnail: %(msg)s' % { + 'msg': unicode(e)}, exc_info=True) diff --git a/src/update.py b/src/update.py index 18d2564..21d8c75 100644 --- a/src/update.py +++ b/src/update.py @@ -62,7 +62,7 @@ def get_all_versions(): return sorted(versions, cmp=compare_versions) except Exception as e: - logging.error('could not get versions: %(msg)s' % {'msg': unicode(e)}) + logging.error('could not get versions: %(msg)s' % {'msg': unicode(e)}, exc_info=True) return [] @@ -191,6 +191,6 @@ def perform_update(version): return True except Exception as e: - logging.error('could not perform update: %(msg)s' % {'msg': unicode(e)}) + logging.error('could not perform update: %(msg)s' % {'msg': unicode(e)}, exc_info=True) return False \ No newline at end of file diff --git a/src/v4l2ctl.py b/src/v4l2ctl.py index b17a20d..505579e 100644 --- a/src/v4l2ctl.py +++ b/src/v4l2ctl.py @@ -88,6 +88,26 @@ def list_resolutions(device): logging.debug('found resolution %(width)sx%(height)s for device %(device)s' % { 'device': device, 'width': width, 'height': height}) + if not resolutions: + logging.debug('no resolutions found for device %(device)s, adding the defaults' % {'device': device}) + + # no resolution returned by v4l2-ctl call, add common default resolutions + resolutions.add((160, 120)) + resolutions.add((320, 240)) + resolutions.add((640, 480)) + resolutions.add((800, 480)) + resolutions.add((800, 600)) + resolutions.add((1024, 576)) + resolutions.add((1024, 768)) + resolutions.add((1280, 720)) + resolutions.add((1280, 800)) + resolutions.add((1280, 960)) + resolutions.add((1366, 768)) + resolutions.add((1440, 900)) + resolutions.add((1680, 1050)) + resolutions.add((1920, 1080)) + resolutions.add((1920, 1440)) + resolutions = list(sorted(resolutions, key=lambda r: (r[0], r[1]))) _resolutions_cache[device] = resolutions -- 2.39.5