]> www.vanbest.org Git - motioneye-debian/commitdiff
cleanup, thumbnailer and media listing are now executed in a separate
authorCalin Crisan <ccrisan@gmail.com>
Sat, 11 Jan 2014 14:13:46 +0000 (16:13 +0200)
committerCalin Crisan <ccrisan@gmail.com>
Sat, 11 Jan 2014 14:45:43 +0000 (16:45 +0200)
subprocess to avoid server blocking

motioneye.py
settings_default.py
src/cleanup.py [new file with mode: 0644]
src/handlers.py
src/mediafiles.py
src/motionctl.py
src/smbctl.py [deleted file]
src/thumbnailer.py [new file with mode: 0644]
src/update.py
src/v4l2ctl.py

index 9f1e1bbf76dc7838b8ed6e2c229f3c716432beb0..34745115d6dbaa589215b673b30c373d75aabead 100755 (executable)
@@ -19,6 +19,7 @@
 import datetime
 import inspect
 import logging
+import multiprocessing
 import os.path
 import re
 import signal
@@ -93,8 +94,10 @@ def _test_requirements():
         
 def _configure_signals():
     def bye_handler(signal, frame):
-        import tornado.ioloop
+        import cleanup
         import motionctl
+        import thumbnailer
+        import tornado.ioloop
         
         logging.info('interrupt signal received, shutting down...')
 
@@ -102,16 +105,27 @@ def _configure_signals():
         ioloop = tornado.ioloop.IOLoop.instance()
         if ioloop.running():
             ioloop.stop()
+            logging.info('server stopped')
         
-        logging.info('server stopped')
-        
+        if thumbnailer.running():
+            thumbnailer.stop()
+            logging.info('thumbnailer stopped')
+    
+        if cleanup.running():
+            cleanup.stop()
+            logging.info('cleanup stopped')
+
         if motionctl.running():
             motionctl.stop()
             logging.info('motion stopped')
+        
+    def child_handler(signal, frame):
+        # this is required for the multiprocessing mechanism to work
+        multiprocessing.active_children()
 
     signal.signal(signal.SIGINT, bye_handler)
     signal.signal(signal.SIGTERM, bye_handler)
-    signal.signal(signal.SIGCHLD, signal.SIG_IGN)
+    signal.signal(signal.SIGCHLD, child_handler)
 
 
 def _configure_logging():
@@ -190,7 +204,6 @@ def _print_help():
     print('available options: ')
     
     options = list(inspect.getmembers(settings))
-    options.append(('THUMBNAILS', None))
     
     for (name, value) in sorted(options):
         if name.upper() != name:
@@ -211,28 +224,6 @@ def _print_help():
     print('')
 
 
-def _do_thumbnails():
-    import config
-    import mediafiles
-    
-    logging.info('recreating thumbnails for all video files...')
-    
-    for camera_id in config.get_camera_ids():
-        camera_config = config.get_camera(camera_id)
-        if camera_config.get('@proto') != 'v4l2':
-            continue
-        
-        logging.info('listing movie files for camera %(name)s' % {
-                'name': camera_config['@name']})
-        
-        target_dir = camera_config['target_dir']
-        
-        for (full_path, st) in mediafiles._list_media_files(target_dir, mediafiles._MOVIE_EXTS):  # @UnusedVariable
-            mediafiles.make_movie_preview(camera_config, full_path)
-    
-    logging.info('done.')
-
-
 def _start_server():
     import tornado.ioloop
     import server
@@ -261,7 +252,7 @@ def _start_motion():
             
             except Exception as e:
                 logging.error('failed to start motion: %(msg)s' % {
-                        'msg': unicode(e)})
+                        'msg': unicode(e)}, exc_info=True)
 
         ioloop.add_timeout(datetime.timedelta(seconds=settings.MOTION_CHECK_INTERVAL), checker)
     
@@ -269,69 +260,31 @@ def _start_motion():
 
 
 def _start_cleanup():
-    import tornado.ioloop
-    import mediafiles
+    import cleanup
 
-    ioloop = tornado.ioloop.IOLoop.instance()
-    
-    def do_cleanup():
-        if ioloop._stopped:
-            return
-        
-        try:
-            mediafiles.cleanup_media('picture')
-            mediafiles.cleanup_media('movie')
-            
-        except Exception as e:
-            logging.error('failed to cleanup media files: %(msg)s' % {
-                    'msg': unicode(e)})
-
-        ioloop.add_timeout(datetime.timedelta(seconds=settings.CLEANUP_INTERVAL), do_cleanup)
-
-    ioloop.add_timeout(datetime.timedelta(seconds=settings.CLEANUP_INTERVAL), do_cleanup)
+    cleanup.start()
+    logging.info('cleanup started')
 
 
-def _start_movie_thumbnailer():
-    import tornado.ioloop
-    import mediafiles
-
-    ioloop = tornado.ioloop.IOLoop.instance()
-    
-    def do_next_movie_thumbail():
-        if ioloop._stopped:
-            return
-        
-        try:
-            mediafiles.make_next_movie_preview()
-            
-        except Exception as e:
-            logging.error('failed to make movie thumbnail: %(msg)s' % {
-                    'msg': unicode(e)})
+def _start_thumbnailer():
+    import thumbnailer
 
-        ioloop.add_timeout(datetime.timedelta(seconds=settings.THUMBNAILER_INTERVAL), do_next_movie_thumbail)
-    
-    ioloop.add_timeout(datetime.timedelta(seconds=settings.THUMBNAILER_INTERVAL), do_next_movie_thumbail)
+    thumbnailer.start()
+    logging.info('thumbnailer started')
 
 
 if __name__ == '__main__':
     if not _test_requirements():
-        sys.exit(01)
+        sys.exit(-1)
     
     cmd = _configure_settings()
     _configure_signals()
     _configure_logging()
     
-    if cmd:
-        if cmd == 'thumbnails':
-            _do_thumbnails()
-        
-        else:
-            print('unknown command line option: ' + cmd)
-            sys.exit(-1)
-        
-        sys.exit(0)
-    
     _start_motion()
     _start_cleanup()
-    _start_movie_thumbnailer()
+    
+    if settings.THUMBNAILER_INTERVAL:
+        _start_thumbnailer()
+    
     _start_server()
index 1109f51b91c9b5a7a16ab2f281784b8a59bc7619..c88f9f4f344a05ab48cadb9ca3c4647945353cc3 100644 (file)
@@ -35,7 +35,7 @@ MOTION_CHECK_INTERVAL = 10
 # interval in seconds at which the janitor is called to remove old pictures and movies
 CLEANUP_INTERVAL = 43200
 
-# interval in seconds at which the thumbnail mechanism runs 
+# interval in seconds at which the thumbnail mechanism runs (set to 0 to disable) 
 THUMBNAILER_INTERVAL = 60
 
 # timeout in seconds to wait for responses when contacting a remote server
diff --git a/src/cleanup.py b/src/cleanup.py
new file mode 100644 (file)
index 0000000..7ac6a39
--- /dev/null
@@ -0,0 +1,84 @@
+
+# Copyright (c) 2013 Calin Crisan
+# This file is part of motionEye.
+#
+# motionEye is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>. 
+
+import datetime
+import logging
+import multiprocessing
+import os
+import signal
+import tornado
+
+import mediafiles
+import settings
+
+
+_process = None
+
+
+def start():
+    if running():
+        raise Exception('cleanup is already running')
+
+    _run_process()
+
+
+def stop():
+    global _process
+    
+    if not running():
+        raise Exception('cleanup is not running')
+    
+    _process.join(timeout=10)
+    if _process.is_alive():
+        logging.error('cleanup process did not finish in time, killing it...')
+        os.kill(_process.pid, signal.SIGKILL)
+    
+    _process = None
+
+
+def running():
+    return _process is not None
+
+
+def _run_process():
+    global _process
+    
+    if not _process or not _process.is_alive(): # check that the previous process has finished
+        logging.debug('running cleanup process...')
+
+        _process = multiprocessing.Process(target=_do_cleanup)
+        _process.start()
+
+    # schedule the next call
+    ioloop = tornado.ioloop.IOLoop.instance()
+    ioloop.add_timeout(datetime.timedelta(seconds=settings.CLEANUP_INTERVAL), _run_process)
+
+
+def _do_cleanup():
+    # this will be executed in a separate subprocess
+    
+    # ignore the terminate and interrupt signals in this subprocess
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
+    signal.signal(signal.SIGTERM, signal.SIG_IGN)
+    
+    try:
+        mediafiles.cleanup_media('picture')
+        mediafiles.cleanup_media('movie')
+         
+    except Exception as e:
+        logging.error('failed to cleanup media files: %(msg)s' % {
+                'msg': unicode(e)}, exc_info=True)
index 5229a6f4d708b714b27a583123a0a8f4d343774c..fe48233532da0a9390df9964a50bb9d28cc60d10 100644 (file)
@@ -615,14 +615,18 @@ class PictureHandler(BaseHandler):
                     prefix=self.get_argument('prefix', None))
         
         else:
-            pictures = mediafiles.list_media(camera_config, media_type='picture',
-                    prefix=self.get_argument('prefix', None))
+            def on_media_list(media_list):
+                if media_list is None:
+                    return self.finish_json({'error': 'Failed to get pictures list.'})
+
+                self.finish_json({
+                    'mediaList': media_list,
+                    'cameraName': camera_config['@name']
+                })
             
-            self.finish_json({
-                'mediaList': pictures,
-                'cameraName': camera_config['@name']
-            })
-        
+            mediafiles.list_media(camera_config, media_type='picture',
+                    callback=on_media_list, prefix=self.get_argument('prefix', None))
+
     @BaseHandler.auth()
     def download(self, camera_id, filename):
         logging.debug('downloading picture %(filename)s of camera %(id)s' % {
@@ -771,14 +775,18 @@ class MovieHandler(BaseHandler):
                     prefix=self.get_argument('prefix', None))
         
         else:
-            movies = mediafiles.list_media(camera_config, media_type='movie',
-                    prefix=self.get_argument('prefix', None))
+            def on_media_list(media_list):
+                if media_list is None:
+                    return self.finish_json({'error': 'Failed to get movies list.'})
+
+                self.finish_json({
+                    'mediaList': media_list,
+                    'cameraName': camera_config['@name']
+                })
+            
+            mediafiles.list_media(camera_config, media_type='movie',
+                    callback=on_media_list, prefix=self.get_argument('prefix', None))
             
-            self.finish_json({
-                'mediaList': movies,
-                'cameraName': camera_config['@name']
-            })
-        
     @BaseHandler.auth()
     def download(self, camera_id, filename):
         logging.debug('downloading movie %(filename)s of camera %(id)s' % {
index 9e8d2357c3d1a711697f0516ad3e712c13de368e..60697f887365a3961d9cfaf332c142c664c8276f 100644 (file)
 
 import datetime
 import logging
+import multiprocessing
 import os.path
 import stat
 import StringIO
 import subprocess
+import tornado
 
 from PIL import Image
 
@@ -37,6 +39,7 @@ _MOVIE_EXTS = ['.avi', '.mp4']
 # tuples of (sequence, width, content)
 _current_pictures_cache = {}
 
+# a cache list of paths to movies without preview
 _previewless_movie_files = []
 
 
@@ -86,8 +89,6 @@ def _list_media_files(dir, exts, prefix=None):
 
 def _remove_older_files(dir, moment, exts):
     for (full_path, st) in _list_media_files(dir, exts):
-        # TODO files listed here may not belong to the given camera
-        
         file_moment = datetime.datetime.fromtimestamp(st.st_mtime)
         if file_moment < moment:
             logging.debug('removing file %(path)s...' % {
@@ -195,8 +196,6 @@ def make_next_movie_preview():
             target_dir = camera_config['target_dir']
             
             for (full_path, st) in _list_media_files(target_dir, _MOVIE_EXTS):  # @UnusedVariable
-                # TODO files listed here may not belong to the given camera
-            
                 if os.path.exists(full_path + '.thumb'):
                     continue
                 
@@ -212,7 +211,7 @@ def make_next_movie_preview():
             make_next_movie_preview()
 
 
-def list_media(camera_config, media_type, prefix=None):
+def list_media(camera_config, media_type, callback, prefix=None):
     target_dir = camera_config.get('target_dir')
 
     if media_type == 'picture':
@@ -220,28 +219,58 @@ def list_media(camera_config, media_type, prefix=None):
         
     elif media_type == 'movie':
         exts = _MOVIE_EXTS
-        
-    media_files = []
-    
-    for (p, st) in _list_media_files(target_dir, exts=exts, prefix=prefix):
-        path = p[len(target_dir):]
-        if not path.startswith('/'):
-            path = '/' + path
 
-        timestamp = st.st_mtime
-        size = st.st_size
+    # create a subprocess to retrieve media files
+    def do_list_media(pipe):
+        for (p, st) in _list_media_files(target_dir, exts=exts, prefix=prefix):
+            path = p[len(target_dir):]
+            if not path.startswith('/'):
+                path = '/' + path
+    
+            timestamp = st.st_mtime
+            size = st.st_size
+            
+            pipe.send({
+                'path': path,
+                'momentStr': utils.pretty_date_time(datetime.datetime.fromtimestamp(timestamp)),
+                'sizeStr': utils.pretty_size(size),
+                'timestamp': timestamp
+            })
         
-        media_files.append({
-            'path': path,
-            'momentStr': utils.pretty_date_time(datetime.datetime.fromtimestamp(timestamp)),
-            'sizeStr': utils.pretty_size(size),
-            'timestamp': timestamp
-        })
+        pipe.close()
+    
+    logging.debug('starting media listing process...')
+    
+    (parent_pipe, child_pipe) = multiprocessing.Pipe(duplex=False)
+    process = multiprocessing.Process(target=do_list_media, args=(child_pipe, ))
+    process.start()
+    
+    # poll the subprocess to see when it has finished
+    started = datetime.datetime.now()
+    def poll_process():
+        ioloop = tornado.ioloop.IOLoop.instance()
+        if process.is_alive(): # not finished yet
+            now = datetime.datetime.now()
+            delta = now - started
+            if delta.seconds < 120:
+                ioloop.add_timeout(datetime.timedelta(seconds=0.1), poll_process)
+            
+            else: # process did not finish within 2 minutes
+                logging.error('timeout waiting for the media listing process to finish')
+                
+                callback(None)
+
+        else: # finished
+            media_list = []
+            while parent_pipe.poll():
+                media_list.append(parent_pipe.recv())
+            
+            logging.debug('media listing process has returned %(count)s files' % {'count': len(media_list)})
+
+            callback(media_list)
     
-    # TODO files listed here may not belong to the given camera
+    poll_process()
     
-    return media_files
-
 
 def get_media_content(camera_config, path, media_type):
     target_dir = camera_config.get('target_dir')
index b7aeb13685c288671846396b2d559a5437da9ee0..c73497fa00b3238892e723e85073701be43fa982 100644 (file)
@@ -51,8 +51,7 @@ def start():
 
     log_file = open(motion_log_path, 'w')
     
-    process = subprocess.Popen(args, stdout=log_file, stderr=log_file, close_fds=True,
-            cwd=settings.CONF_PATH)
+    process = subprocess.Popen(args, stdout=log_file, stderr=log_file, close_fds=True, cwd=settings.CONF_PATH)
     
     # wait 2 seconds to see that the process has successfully started
     for i in xrange(20):  # @UnusedVariable
@@ -84,22 +83,22 @@ def stop():
             
             # wait 5 seconds for the process to exit
             for i in xrange(50):  # @UnusedVariable
+                os.waitpid(pid, os.WNOHANG)
                 time.sleep(0.1)
-                os.kill(pid, 0)
-            
+
             # send the KILL signal once
             os.kill(pid, signal.SIGKILL)
             
             # wait 2 seconds for the process to exit
             for i in xrange(20):  # @UnusedVariable
                 time.sleep(0.1)
-                os.kill(pid, 0)
+                os.waitpid(pid, os.WNOHANG)
                 
             # the process still did not exit
             raise Exception('could not terminate the motion process')
         
         except OSError as e:
-            if e.errno != errno.ESRCH:
+            if e.errno != errno.ECHILD:
                 raise
     
 
diff --git a/src/smbctl.py b/src/smbctl.py
deleted file mode 100644 (file)
index 28fb229..0000000
+++ /dev/null
@@ -1,16 +0,0 @@
-
-# Copyright (c) 2013 Calin Crisan
-# This file is part of motionEye.
-#
-# motionEye is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-# 
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-# 
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>. 
diff --git a/src/thumbnailer.py b/src/thumbnailer.py
new file mode 100644 (file)
index 0000000..23fefd0
--- /dev/null
@@ -0,0 +1,83 @@
+
+# Copyright (c) 2013 Calin Crisan
+# This file is part of motionEye.
+#
+# motionEye is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>. 
+
+import datetime
+import logging
+import multiprocessing
+import os
+import signal
+import tornado
+
+import mediafiles
+import settings
+
+
+_process = None
+
+
+def start():
+    if running():
+        raise Exception('thumbnailer is already running')
+
+    _run_process()
+
+
+def stop():
+    global _process
+    
+    if not running():
+        raise Exception('thumbnailer is not running')
+    
+    _process.join(timeout=10)
+    if _process.is_alive():
+        logging.error('thumbnailer process did not finish in time, killing it...')
+        os.kill(_process.pid, signal.SIGKILL)
+    
+    _process = None
+
+
+def running():
+    return _process is not None
+
+
+def _run_process():
+    global _process
+    
+    if not _process or not _process.is_alive(): # check that the previous process has finished
+        logging.debug('running thumbnailer process...')
+
+        _process = multiprocessing.Process(target=_do_next_movie_thumbail)
+        _process.start()
+
+    # schedule the next call
+    ioloop = tornado.ioloop.IOLoop.instance()
+    ioloop.add_timeout(datetime.timedelta(seconds=settings.THUMBNAILER_INTERVAL), _run_process)
+
+
+def _do_next_movie_thumbail():
+    # this will be executed in a separate subprocess
+    
+    # ignore the terminate and interrupt signals in this subprocess
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
+    signal.signal(signal.SIGTERM, signal.SIG_IGN)
+    
+    try:
+        mediafiles.make_next_movie_preview()
+         
+    except Exception as e:
+        logging.error('failed to make movie thumbnail: %(msg)s' % {
+                'msg': unicode(e)}, exc_info=True)
index 18d2564c4f50874ffc0e8f2435ae1080944c3949..21d8c75951b13434d0f868a30248a570589e73ef 100644 (file)
@@ -62,7 +62,7 @@ def get_all_versions():
         return sorted(versions, cmp=compare_versions)
 
     except Exception as e:
-        logging.error('could not get versions: %(msg)s' % {'msg': unicode(e)})
+        logging.error('could not get versions: %(msg)s' % {'msg': unicode(e)}, exc_info=True)
         
     return []
 
@@ -191,6 +191,6 @@ def perform_update(version):
         return True
     
     except Exception as e:
-        logging.error('could not perform update: %(msg)s' % {'msg': unicode(e)})
+        logging.error('could not perform update: %(msg)s' % {'msg': unicode(e)}, exc_info=True)
         
         return False
\ No newline at end of file
index b17a20d826a0fc1a86d47ac9f92fd0fcb0edad79..505579e209b391be3994b6930d6f7824e340188b 100644 (file)
@@ -88,6 +88,26 @@ def list_resolutions(device):
         logging.debug('found resolution %(width)sx%(height)s for device %(device)s' % {
                 'device': device, 'width': width, 'height': height})
     
+    if not resolutions:
+        logging.debug('no resolutions found for device %(device)s, adding the defaults' % {'device': device})
+        
+        # no resolution returned by v4l2-ctl call, add common default resolutions
+        resolutions.add((160, 120))
+        resolutions.add((320, 240))
+        resolutions.add((640, 480))
+        resolutions.add((800, 480))
+        resolutions.add((800, 600))
+        resolutions.add((1024, 576))
+        resolutions.add((1024, 768))
+        resolutions.add((1280, 720))
+        resolutions.add((1280, 800))
+        resolutions.add((1280, 960))
+        resolutions.add((1366, 768))
+        resolutions.add((1440, 900))
+        resolutions.add((1680, 1050))
+        resolutions.add((1920, 1080))
+        resolutions.add((1920, 1440))
+
     resolutions = list(sorted(resolutions, key=lambda r: (r[0], r[1])))
     _resolutions_cache[device] = resolutions