From: Calin Crisan Date: Sat, 21 Nov 2015 19:30:27 +0000 (+0200) Subject: added tasks module and got rid of the old thumbnailer mechanism X-Git-Url: http://www.vanbest.org/gitweb/?a=commitdiff_plain;h=d11049bdc0e99693688ec0eca9cfcd0c86796153;p=motioneye-debian added tasks module and got rid of the old thumbnailer mechanism --- diff --git a/extra/motioneye.conf.sample b/extra/motioneye.conf.sample index 2503a1c..ec3189d 100644 --- a/extra/motioneye.conf.sample +++ b/extra/motioneye.conf.sample @@ -46,10 +46,6 @@ motion_check_interval 10 # to remove old pictures and movies cleanup_interval 43200 -# interval in seconds at which the thumbnail mechanism runs -# (set to 0 to disable) -thumbnailer_interval 60 - # timeout in seconds to wait for response from a remote motionEye server remote_request_timeout 10 diff --git a/motioneye/cleanup.py b/motioneye/cleanup.py index 5f93ea3..6f122d3 100644 --- a/motioneye/cleanup.py +++ b/motioneye/cleanup.py @@ -25,7 +25,6 @@ from tornado.ioloop import IOLoop import mediafiles import settings -import thumbnailer _process = None @@ -66,15 +65,8 @@ def _run_process(): io_loop = IOLoop.instance() - if thumbnailer.running(): - # postpone if thumbnailer is currently running - io_loop.add_timeout(datetime.timedelta(seconds=60), _run_process) - - return - - else: - # schedule the next call - io_loop.add_timeout(datetime.timedelta(seconds=settings.CLEANUP_INTERVAL), _run_process) + # schedule the next call + io_loop.add_timeout(datetime.timedelta(seconds=settings.CLEANUP_INTERVAL), _run_process) if not running(): # check that the previous process has finished logging.debug('running cleanup process...') diff --git a/motioneye/config.py b/motioneye/config.py index e358fe8..104baf9 100644 --- a/motioneye/config.py +++ b/motioneye/config.py @@ -51,11 +51,11 @@ _LAST_OLD_CONFIG_VERSIONS = (490, '3.2.12') _KNOWN_MOTION_OPTIONS = set([ 'auto_brightness', 'brightness', 'contrast', 'emulate_motion', 'event_gap', 'ffmpeg_bps', 'ffmpeg_output_movies', 'ffmpeg_variable_bitrate', 'ffmpeg_video_codec', 'framerate', 'height', 'hue', 'lightswitch', 'locate_motion_mode', 'locate_motion_style', 'minimum_motion_frames', 'movie_filename', 'max_movie_time', 'max_mpeg_time', - 'noise_level', 'noise_tune', 'on_event_end', 'on_event_start', 'output_pictures', 'picture_filename', 'post_capture', 'pre_capture', 'quality', 'rotate', 'saturation', - 'snapshot_filename', 'snapshot_interval', 'stream_auth_method', 'stream_authentication', 'stream_localhost', 'stream_maxrate', 'stream_motion', 'stream_port', 'stream_quality', - 'target_dir', 'text_changes', 'text_double', 'text_left', 'text_right', 'threshold', 'videodevice', 'width', - 'webcam_localhost', 'webcam_port', 'webcam_maxrate', 'webcam_quality', 'webcam_motion', 'ffmpeg_cap_new', 'output_normal', 'output_motion', 'jpeg_filename', 'output_all', 'gap', 'locate', - 'netcam_url', 'netcam_userpass', 'netcam_http', 'netcam_tolerant_check', 'netcam_keepalive', 'rtsp_uses_tcp' + 'noise_level', 'noise_tune', 'on_event_end', 'on_event_start', 'on_movie_end', 'on_picture_save', 'output_pictures', 'picture_filename', 'post_capture', 'pre_capture', + 'quality', 'rotate', 'saturation', 'snapshot_filename', 'snapshot_interval', 'stream_auth_method', 'stream_authentication', 'stream_localhost', 'stream_maxrate', + 'stream_motion', 'stream_port', 'stream_quality', 'target_dir', 'text_changes', 'text_double', 'text_left', 'text_right', 'threshold', 'videodevice', 'width', + 'webcam_localhost', 'webcam_port', 'webcam_maxrate', 'webcam_quality', 'webcam_motion', 'ffmpeg_cap_new', 'output_normal', 'output_motion', 'jpeg_filename', 'output_all', + 'gap', 'locate', 'netcam_url', 'netcam_userpass', 'netcam_http', 'netcam_tolerant_check', 'netcam_keepalive', 'rtsp_uses_tcp' ]) @@ -681,7 +681,9 @@ def motion_camera_ui_to_dict(ui, old_config=None): # events 'on_event_start': '', - 'on_event_end': '' + 'on_event_end': '', + 'on_movie_end': '', + 'on_picture_save': '' } if utils.v4l2_camera(old_config): @@ -879,6 +881,14 @@ def motion_camera_ui_to_dict(ui, old_config=None): data['on_event_end'] = '; '.join(on_event_end) + # movie end + on_movie_end = ['%(script)s movie_end %%t %%f' % {'script': meyectl.find_command('relayevent')}] + data['on_movie_end'] = '; '.join(on_movie_end) + + # picture save + on_picture_save = ['%(script)s picture_save %%t %%f' % {'script': meyectl.find_command('relayevent')}] + data['on_picture_save'] = '; '.join(on_picture_save) + # additional configs for name, value in ui.iteritems(): if not name.startswith('_'): @@ -1661,6 +1671,8 @@ def _set_default_motion_camera(camera_id, data): data.setdefault('on_event_start', '') data.setdefault('on_event_end', '') + data.setdefault('on_movie_end', '') + data.setdefault('on_picture_save', '') def _set_default_simple_mjpeg_camera(camera_id, data): diff --git a/motioneye/handlers.py b/motioneye/handlers.py index 37bfc32..fa11e95 100644 --- a/motioneye/handlers.py +++ b/motioneye/handlers.py @@ -34,6 +34,7 @@ import prefs import remote import settings import smbctl +import tasker import template import update import utils @@ -41,23 +42,50 @@ import v4l2ctl class BaseHandler(RequestHandler): - def get_data(self): + def get_all_arguments(self): keys = self.request.arguments.keys() - data = dict([(key, self.get_argument(key)) for key in keys]) + arguments = dict([(key, self.get_argument(key)) for key in keys]) for key in self.request.files: files = self.request.files[key] if len(files) > 1: - data[key] = files + arguments[key] = files elif len(files) > 0: - data[key] = files[0] + arguments[key] = files[0] else: continue + + # consider the json passed in body as well + data = self.get_json() + if data and isinstance(data, dict): + arguments.update(data) + + return arguments + + def get_json(self): + if not hasattr(self, '_json'): + self._json = None + if self.request.headers.get('Content-Type', '').startswith('application/json'): + self._json = json.loads(self.request.body) - return data + return self._json + def get_argument(self, name, default=None): + DEF = {} + argument = RequestHandler.get_argument(self, name, default=DEF) + if argument is DEF: + # try to find it in json body + data = self.get_json() + if data: + argument = data.get(name, DEF) + + if argument is DEF: + argument = default + + return argument + def render(self, template_name, content_type='text/html', **context): self.set_header('Content-Type', content_type) @@ -492,7 +520,7 @@ class ConfigHandler(BaseHandler): def list(self): logging.debug('listing cameras') - proto = self.get_data().get('proto') + proto = self.get_argument('proto') if proto == 'motioneye': # remote listing def on_response(cameras=None, error=None): if error: @@ -501,10 +529,10 @@ class ConfigHandler(BaseHandler): else: self.finish_json({'cameras': cameras}) - remote.list(self.get_data(), on_response) + remote.list(self.get_all_arguments(), on_response) elif proto == 'netcam': - scheme = self.get_data().get('scheme', 'http') + scheme = self.get_argument('scheme', 'http') def on_response(cameras=None, error=None): if error: @@ -514,10 +542,10 @@ class ConfigHandler(BaseHandler): self.finish_json({'cameras': cameras}) if scheme in ['http', 'https']: - utils.test_mjpeg_url(self.get_data(), auth_modes=['basic'], allow_jpeg=True, callback=on_response) + utils.test_mjpeg_url(self.get_all_arguments(), auth_modes=['basic'], allow_jpeg=True, callback=on_response) elif config.motion_rtsp_support() and scheme == 'rtsp': - utils.test_rtsp_url(self.get_data(), callback=on_response) + utils.test_rtsp_url(self.get_all_arguments(), callback=on_response) else: on_response(error='protocol %s not supported' % scheme) @@ -530,7 +558,7 @@ class ConfigHandler(BaseHandler): else: self.finish_json({'cameras': cameras}) - utils.test_mjpeg_url(self.get_data(), auth_modes=['basic', 'digest'], allow_jpeg=False, callback=on_response) + utils.test_mjpeg_url(self.get_all_arguments(), auth_modes=['basic', 'digest'], allow_jpeg=False, callback=on_response) elif proto == 'v4l2': configured_devices = set() @@ -1372,7 +1400,7 @@ class RelayEventHandler(BaseHandler): camera_id = motionctl.thread_id_to_camera_id(thread_id) if camera_id is None: - logging.debug('ignoring event for thread id %s' % thread_id) + logging.debug('ignoring event for unknown thread id %s' % thread_id) return self.finish_json() camera_config = config.get_camera(camera_id) @@ -1389,6 +1417,17 @@ class RelayEventHandler(BaseHandler): elif event == 'stop': motionctl.set_motion_detected(camera_id, False) + + elif event == 'movie_end': + full_path = self.get_argument('filename') + + # generate preview (thumbnail) + tasker.add_task(5, mediafiles.make_movie_preview, tag='make_movie_preview(%s)' % full_path, async=True, + camera_config=camera_config, full_path=full_path) + + # upload TODO +# tasker.add_task(5, upload.upload_media_file, tag='upload_media_file(%s)' % full_path, +# camera_config=camera_config, full_path=full_path) else: logging.warn('unknown event %s' % event) diff --git a/motioneye/mediafiles.py b/motioneye/mediafiles.py index 7523870..d008ad6 100644 --- a/motioneye/mediafiles.py +++ b/motioneye/mediafiles.py @@ -42,9 +42,6 @@ import utils _PICTURE_EXTS = ['.jpg'] _MOVIE_EXTS = ['.avi', '.mp4'] -# a cache list of paths to movies without preview -_previewless_movie_files = [] - # a cache of prepared files (whose preparing time is significant) _prepared_files = {} @@ -244,43 +241,6 @@ def make_movie_preview(camera_config, full_path): return full_path + '.thumb' -def make_next_movie_preview(): - global _previewless_movie_files - - logging.debug('making preview for the next movie...') - - if _previewless_movie_files: - (camera_config, path) = _previewless_movie_files.pop(0) - - make_movie_preview(camera_config, path) - - else: - logging.debug('gathering movies without preview...') - - count = 0 - for camera_id in config.get_camera_ids(): - camera_config = config.get_camera(camera_id) - if not utils.local_motion_camera(camera_config): - continue - - target_dir = camera_config['target_dir'] - - for (full_path, st) in _list_media_files(target_dir, _MOVIE_EXTS): # @UnusedVariable - if os.path.exists(full_path + '.thumb'): - continue - - logging.debug('found a movie without preview: %(path)s' % { - 'path': full_path}) - - _previewless_movie_files.append((camera_config, full_path)) - count += 1 - - logging.debug('found %(count)d movies without preview' % {'count': count}) - - if count: - make_next_movie_preview() - - def list_media(camera_config, media_type, callback, prefix=None): target_dir = camera_config.get('target_dir') @@ -669,6 +629,10 @@ def get_media_preview(camera_config, path, media_type, width, height): if media_type == 'movie': if not os.path.exists(full_path + '.thumb'): + # at this point we expect the thumb to + # have already been created by the thumbnailer task; + # if, for some reason that's not the case, + # we create it right away if not make_movie_preview(camera_config, full_path): return None diff --git a/motioneye/relayevent.py b/motioneye/relayevent.py index f22dd93..fa48320 100644 --- a/motioneye/relayevent.py +++ b/motioneye/relayevent.py @@ -20,7 +20,7 @@ import json import logging import os.path import sys -import urllib +import urllib2 sys.path.append(os.path.join(os.path.dirname(sys.argv[0]),'src')) @@ -89,6 +89,7 @@ def get_admin_credentials(): def parse_options(parser, args): parser.add_argument('event', help='the name of the event to relay') parser.add_argument('thread_id', help='the id of the thread') + parser.add_argument('filename', nargs='?', help='the name of the file related to the event') return parser.parse_args(args) @@ -104,21 +105,32 @@ def main(parser, args): logging.debug('hello!') logging.debug('event = %s' % options.event) logging.debug('thread_id = %s' % options.thread_id) + if options.filename: + logging.debug('filename = %s' % options.filename) admin_username, admin_password = get_admin_credentials() - - path = '/_relay_event/?event=%(event)s&thread_id=%(thread_id)s&_username=%(username)s' % { - 'username': admin_username, - 'thread_id': options.thread_id, - 'event': options.event} - signature = utils.compute_signature('POST', path, '', admin_password) + data = { + '_username': admin_username, + 'thread_id': options.thread_id, + 'event': options.event + } + + if options.filename: + data['filename'] = options.filename + + path = '/_relay_event/' + body = json.dumps(data) - url = 'http://127.0.0.1:%(port)s' + path + '&_signature=' + signature + signature = utils.compute_signature('POST', path, body, admin_password) + + url = 'http://127.0.0.1:%(port)s' + path + '?_signature=' + signature url = url % {'port': settings.PORT} + request = urllib2.Request(url, data=body, headers={'Content-Type': 'application/json'}) + try: - response = urllib.urlopen(url, data='') + response = urllib2.urlopen(request) response = json.load(response) if response.get('error'): raise Exception(response['error']) diff --git a/motioneye/server.py b/motioneye/server.py index 816924a..96e0350 100644 --- a/motioneye/server.py +++ b/motioneye/server.py @@ -20,6 +20,7 @@ import datetime import logging import multiprocessing import os +import re import signal import sys import time @@ -33,6 +34,7 @@ import template _PID_FILE = 'motioneye.pid' +_CURRENT_PICTURE_REGEX = re.compile('^/picture/\d+/current') class Daemon(object): @@ -146,8 +148,11 @@ class Daemon(object): def _log_request(handler): + log_method = None + if handler.get_status() < 400: - log_method = logging.debug + if not _CURRENT_PICTURE_REGEX.match(handler.request.uri): + log_method = logging.debug elif handler.get_status() < 500: log_method = logging.warning @@ -155,9 +160,10 @@ def _log_request(handler): else: log_method = logging.error - request_time = 1000.0 * handler.request.request_time() - log_method("%d %s %.2fms", handler.get_status(), - handler._request_summary(), request_time) + if log_method: + request_time = 1000.0 * handler.request.request_time() + log_method("%d %s %.2fms", handler.get_status(), + handler._request_summary(), request_time) handler_mapping = [ (r'^/$', handlers.MainHandler), @@ -324,7 +330,7 @@ def run(): import motionctl import motioneye import smbctl - import thumbnailer + import tasker import wsswitch configure_signals() @@ -347,9 +353,8 @@ def run(): wsswitch.start() logging.info('wsswitch started') - if settings.THUMBNAILER_INTERVAL: - thumbnailer.start() - logging.info('thumbnailer started') + tasker.start() + logging.info('tasker started') if settings.MJPG_CLIENT_TIMEOUT: mjpgclient.start() @@ -372,10 +377,6 @@ def run(): logging.info('server stopped') - if thumbnailer.running(): - thumbnailer.stop() - logging.info('thumbnailer stopped') - if cleanup.running(): cleanup.stop() logging.info('cleanup stopped') diff --git a/motioneye/settings.py b/motioneye/settings.py index 5f3fb18..563d97f 100644 --- a/motioneye/settings.py +++ b/motioneye/settings.py @@ -73,10 +73,6 @@ MOTION_CHECK_INTERVAL = 10 # to remove old pictures and movies CLEANUP_INTERVAL = 43200 -# interval in seconds at which the thumbnail mechanism runs -# (set to 0 to disable) -THUMBNAILER_INTERVAL = 60 - # timeout in seconds to wait for response from a remote motionEye server REMOTE_REQUEST_TIMEOUT = 10 diff --git a/motioneye/tasker.py b/motioneye/tasker.py new file mode 100644 index 0000000..f2f6db7 --- /dev/null +++ b/motioneye/tasker.py @@ -0,0 +1,147 @@ + +# Copyright (c) 2013 Calin Crisan +# This file is part of motionEye. +# +# motionEye is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import calendar +import cPickle +import datetime +import logging +import multiprocessing +import os +import time + +from tornado.ioloop import IOLoop + +import settings + + +_INTERVAL = 10 +_STATE_FILE_NAME = 'tasks.pickle' +_MAX_TASKS = 100 +_POOL_SIZE = 2 + +_tasks = [] +_pool = None + + +def start(): + global _pool + + io_loop = IOLoop.instance() + io_loop.add_timeout(datetime.timedelta(seconds=_INTERVAL), check_tasks) + + _load() + _pool = multiprocessing.Pool(_POOL_SIZE) + + +def check_tasks(): + io_loop = IOLoop.instance() + io_loop.add_timeout(datetime.timedelta(seconds=_INTERVAL), check_tasks) + + now = time.time() + changed = False + while _tasks and _tasks[0][0] <= now: + (when, func, tag, async, params) = _tasks.pop(0) # @UnusedVariable + + logging.debug('executing task "%s"' % tag or func.func_name) + if async: + _pool.apply_async(func, kwds=params) + + else: + try: + func(**params) + + except Exception as e: + logging.error('task "%s" failed: %s' % (tag or func.func_name, e), exc_info=True) + + changed = True + + if changed: + _save() + + +def add_task(when, func, tag=None, async=False, **params): + if len(_tasks) >= _MAX_TASKS: + return logging.error('the maximum number of tasks (%d) has been reached' % _MAX_TASKS) + + if isinstance(when, int): # delay, in seconds + when += time.time() + + elif isinstance(when, datetime.timedelta): + when = time.time() + when.total_seconds() + + elif isinstance(when, datetime.datetime): + when = calendar.timegm(when.timetuple()) + + i = 0 + while i < len(_tasks) and _tasks[i][0] <= when: + i += 1 + + logging.debug('adding task "%s" in %d seconds' % (tag or func.func_name, when - time.time())) + _tasks.insert(i, (when, func, tag, async, params)) + + _save() + + +def _load(): + global _tasks + + _tasks = [] + + file_path = os.path.join(settings.CONF_PATH, _STATE_FILE_NAME) + + if os.path.exists(file_path): + logging.debug('loading tasks from "%s"...' % file_path) + + try: + file = open(file_path, 'r') + + except Exception as e: + logging.error('could not open tasks file "%s": %s' % (file_path, e)) + + return + + try: + _tasks = cPickle.load(file) + + except Exception as e: + logging.error('could not read tasks from file "%s": %s' % (file_path, e)) + + finally: + file.close() + + +def _save(): + file_path = os.path.join(settings.CONF_PATH, _STATE_FILE_NAME) + + logging.debug('saving tasks to "%s"...' % file_path) + + try: + file = open(file_path, 'w') + + except Exception as e: + logging.error('could not open tasks file "%s": %s' % (file_path, e)) + + return + + try: + cPickle.dump(_tasks, file) + + except Exception as e: + logging.error('could not save tasks to file "%s": %s'% (file_path, e)) + + finally: + file.close() diff --git a/motioneye/thumbnailer.py b/motioneye/thumbnailer.py deleted file mode 100644 index 0edd3a8..0000000 --- a/motioneye/thumbnailer.py +++ /dev/null @@ -1,90 +0,0 @@ - -# Copyright (c) 2013 Calin Crisan -# This file is part of motionEye. -# -# motionEye is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import datetime -import logging -import multiprocessing -import os -import signal - -from tornado.ioloop import IOLoop - -import cleanup -import mediafiles -import settings - - -_process = None - - -def start(): - if not settings.THUMBNAILER_INTERVAL: - return - - # schedule the first call a bit later to improve performance at startup - io_loop = IOLoop.instance() - io_loop.add_timeout(datetime.timedelta(seconds=min(settings.THUMBNAILER_INTERVAL, 30)), _run_process) - - -def stop(): - global _process - - if not running(): - _process = None - return - - if _process.is_alive(): - _process.join(timeout=10) - - if _process.is_alive(): - logging.error('thumbnailer process did not finish in time, killing it...') - os.kill(_process.pid, signal.SIGKILL) - - _process = None - - -def running(): - return _process is not None and _process.is_alive() - - -def _run_process(): - global _process - - # schedule the next call - io_loop = IOLoop.instance() - io_loop.add_timeout(datetime.timedelta(seconds=settings.THUMBNAILER_INTERVAL), _run_process) - - if not running() and not cleanup.running(): # check that the previous process has finished and that cleanup is not running - logging.debug('running thumbnailer process...') - - _process = multiprocessing.Process(target=_do_next_movie_thumbail) - _process.start() - - -def _do_next_movie_thumbail(): - # this will be executed in a separate subprocess - - # ignore the terminate and interrupt signals in this subprocess - signal.signal(signal.SIGINT, signal.SIG_IGN) - signal.signal(signal.SIGTERM, signal.SIG_IGN) - - try: - mediafiles.make_next_movie_preview() - - except Exception as e: - logging.error('failed to make movie thumbnail: %(msg)s' % { - 'msg': unicode(e)}, exc_info=True)