From 0964cc346ee5ff523eca5aecb30b0980ad8d9ff4 Mon Sep 17 00:00:00 2001 From: Calin Crisan Date: Sun, 22 Nov 2015 20:22:27 +0200 Subject: [PATCH] added support for uploading media files to Google Drive --- extra/motioneye.init-debian | 2 +- motioneye/config.py | 43 +++- motioneye/handlers.py | 91 ++++++-- motioneye/meyectl.py | 1 + motioneye/prefs.py | 1 + motioneye/remote.py | 117 +++++++--- motioneye/server.py | 17 +- motioneye/static/css/main.css | 3 +- motioneye/static/js/main.js | 64 +++++- motioneye/{tasker.py => tasks.py} | 69 +++--- motioneye/templates/main.html | 35 +-- motioneye/uploadservices.py | 362 ++++++++++++++++++++++++++++++ 12 files changed, 706 insertions(+), 99 deletions(-) rename motioneye/{tasker.py => tasks.py} (83%) create mode 100644 motioneye/uploadservices.py diff --git a/extra/motioneye.init-debian b/extra/motioneye.init-debian index 8dd7aee..660e91f 100755 --- a/extra/motioneye.init-debian +++ b/extra/motioneye.init-debian @@ -13,7 +13,7 @@ ### END INIT INFO NAME="motioneye" -PATH_BIN="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin:/usr/local/sbin" +PATH="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin:/usr/local/sbin" DAEMON="/usr/local/bin/meyectl" PIDFILE="/var/run/$NAME.pid" DESC="motionEye server" diff --git a/motioneye/config.py b/motioneye/config.py index 104baf9..5646ed5 100644 --- a/motioneye/config.py +++ b/motioneye/config.py @@ -31,6 +31,7 @@ import diskctl import powerctl import settings import update +import uploadservices import utils import v4l2ctl @@ -280,7 +281,8 @@ def get_camera(camera_id, as_lines=False): camera_config = _conf_to_dict(lines, no_convert=['@name', '@network_share_name', '@network_server', - '@network_username', '@network_password', '@storage_device']) + '@network_username', '@network_password', '@storage_device', + '@upload_server', '@upload_username', '@upload_password', '@upload_authorization_key']) if utils.local_motion_camera(camera_config): # determine the enabled status @@ -323,12 +325,16 @@ def get_camera(camera_id, as_lines=False): camera_config['netcam_keepalive'] = camera_config.pop('netcam_http') in ['1.1', 'keepalive'] _get_additional_config(camera_config, camera_id=camera_id) + + _set_default_motion_camera(camera_id, camera_config) elif utils.remote_camera(camera_config): pass elif utils.simple_mjpeg_camera(camera_config): _get_additional_config(camera_config, camera_id=camera_id) + + _set_default_simple_mjpeg_camera(camera_id, camera_config) else: # incomplete configuration logging.warn('camera config file at %s is incomplete, ignoring' % camera_config_path) @@ -631,6 +637,15 @@ def motion_camera_ui_to_dict(ui, old_config=None): '@network_share_name': ui['network_share_name'], '@network_username': ui['network_username'], '@network_password': ui['network_password'], + '@upload_enabled': ui['upload_enabled'], + '@upload_service': ui['upload_service'], + '@upload_server': ui['upload_server'], + '@upload_port': ui['upload_port'], + '@upload_method': ui['upload_method'], + '@upload_location': ui['upload_location'], + '@upload_username': ui['upload_username'], + '@upload_password': ui['upload_password'], + '@upload_authorization_key': ui['upload_authorization_key'], # text overlay 'text_left': '', @@ -767,6 +782,12 @@ def motion_camera_ui_to_dict(ui, old_config=None): else: data['target_dir'] = ui['root_directory'] + + if ui['upload_enabled'] and '@id' in old_config: + upload_settings = {k[7:]: ui[k] for k in ui.iterkeys() if k.startswith('upload_')} + service = uploadservices.get(old_config['@id'], ui['upload_service']) + service.load(upload_settings) + uploadservices.save() if ui['text_overlay']: left_text = ui['left_text'] @@ -932,6 +953,15 @@ def motion_camera_dict_to_ui(data): 'disk_used': 0, 'disk_total': 0, 'available_disks': diskctl.list_mounted_disks(), + 'upload_enabled': data['@upload_enabled'], + 'upload_service': data['@upload_service'], + 'upload_server': data['@upload_server'], + 'upload_port': data['@upload_port'], + 'upload_method': data['@upload_method'], + 'upload_location': data['@upload_location'], + 'upload_username': data['@upload_username'], + 'upload_password': data['@upload_password'], + 'upload_authorization_key': data['@upload_authorization_key'], # text overlay 'text_overlay': False, @@ -1619,7 +1649,16 @@ def _set_default_motion_camera(camera_id, data): data.setdefault('@network_username', '') data.setdefault('@network_password', '') data.setdefault('target_dir', settings.MEDIA_PATH) - + data.setdefault('@upload_enabled', False) + data.setdefault('@upload_service', 'ftp') + data.setdefault('@upload_server', '') + data.setdefault('@upload_port', '') + data.setdefault('@upload_method', 'POST') + data.setdefault('@upload_location', '') + data.setdefault('@upload_username', '') + data.setdefault('@upload_password', '') + data.setdefault('@upload_authorization_key', '') + data.setdefault('stream_localhost', False) data.setdefault('stream_port', int('808' + str(camera_id))) data.setdefault('stream_maxrate', 5) diff --git a/motioneye/handlers.py b/motioneye/handlers.py index 3dc1f33..3042f7e 100644 --- a/motioneye/handlers.py +++ b/motioneye/handlers.py @@ -34,9 +34,10 @@ import prefs import remote import settings import smbctl -import tasker +import tasks import template import update +import uploadservices import utils import v4l2ctl @@ -212,11 +213,8 @@ class ConfigHandler(BaseHandler): elif op == 'backup': self.backup() - elif op == 'test': - self.test() - elif op == 'authorize': - self.authorize() + self.authorize(camera_id) else: raise HTTPError(400, 'unknown operation') @@ -241,6 +239,9 @@ class ConfigHandler(BaseHandler): elif op == 'restore': self.restore() + elif op == 'test': + self.test(camera_id) + else: raise HTTPError(400, 'unknown operation') @@ -733,12 +734,59 @@ class ConfigHandler(BaseHandler): self.finish_json({'ok': False}) @BaseHandler.auth(admin=True) - def test(self): - pass + def test(self, camera_id): + what = self.get_argument('what') + data = self.get_all_arguments() + camera_config = config.get_camera(camera_id) + if what == 'upload_service': + service_name = data.get('service') + if not service_name: + raise HTTPError(400, 'service_name required') + + if utils.local_motion_camera(camera_config): + service = uploadservices.get(camera_id, service_name) + service.load(data) + if not service: + raise HTTPError(400, 'unknown upload service %s' % service_name) + + logging.debug('testing access to %s' % service) + result = service.test_access() + if result is True: + logging.debug('accessing %s succeeded' % service) + self.finish_json() + + else: + logging.warn('accessing %s failed' % service) + self.finish_json({'error': result}) + + elif utils.remote_camera(camera_config): + def on_response(result=None, error=None): + if result is True: + self.finish_json() + + else: + result = result or error + self.finish_json({'error': result}) + + remote.test(camera_config, data, on_response) + + else: + raise HTTPError(400, 'unknown test %s' % what) @BaseHandler.auth(admin=True) - def authorize(self): - pass + def authorize(self, camera_id): + service_name = self.get_argument('service') + if not service_name: + raise HTTPError(400, 'service_name required') + + service = uploadservices.get(camera_id, service_name) + if not service: + raise HTTPError(400, 'unknown upload service %s' % service_name) + + url = service.get_authorize_url() + + logging.debug('redirected to authorization url %s' % url) + self.redirect(url) class PictureHandler(BaseHandler): @@ -1436,15 +1484,26 @@ class RelayEventHandler(BaseHandler): motionctl.set_motion_detected(camera_id, False) elif event == 'movie_end': - full_path = self.get_argument('filename') + filename = self.get_argument('filename') # generate preview (thumbnail) - tasker.add_task(5, mediafiles.make_movie_preview, tag='make_movie_preview(%s)' % full_path, async=True, - camera_config=camera_config, full_path=full_path) - - # upload TODO -# tasker.add_task(5, upload.upload_media_file, tag='upload_media_file(%s)' % full_path, -# camera_config=camera_config, full_path=full_path) + tasks.add(5, mediafiles.make_movie_preview, tag='make_movie_preview(%s)' % filename, async=True, + camera_config=camera_config, full_path=filename) + + # upload to external service + if camera_config['@upload_enabled']: + service_name = camera_config['@upload_service'] + tasks.add(5, uploadservices.upload_media_file, tag='upload_media_file(%s)' % filename, + camera_id=camera_id, service_name=service_name, filename=filename) + + elif event == 'picture_save': + filename = self.get_argument('filename') + + # upload to external service + if camera_config['@upload_enabled']: + service_name = camera_config['@upload_service'] + tasks.add(5, uploadservices.upload_media_file, tag='upload_media_file(%s)' % filename, + camera_id=camera_id, service_name=service_name, filename=filename) else: logging.warn('unknown event %s' % event) diff --git a/motioneye/meyectl.py b/motioneye/meyectl.py index 5e5ce6f..ed0f298 100755 --- a/motioneye/meyectl.py +++ b/motioneye/meyectl.py @@ -173,6 +173,7 @@ def configure_logging(cmd, log_to_file=False): sys.exit(-1) logging.getLogger('tornado').setLevel(logging.WARN) + logging.getLogger('oauth2client').setLevel(logging.WARN) def configure_tornado(): diff --git a/motioneye/prefs.py b/motioneye/prefs.py index 793277c..f2e00fb 100644 --- a/motioneye/prefs.py +++ b/motioneye/prefs.py @@ -23,6 +23,7 @@ import settings _PREFS_FILE_NAME = 'prefs.json' + _prefs = None diff --git a/motioneye/remote.py b/motioneye/remote.py index de4daf4..d1be83a 100644 --- a/motioneye/remote.py +++ b/motioneye/remote.py @@ -28,7 +28,7 @@ import utils _DOUBLE_SLASH_REGEX = re.compile('//+') -def _make_request(scheme, host, port, username, password, path, method='GET', data=None, query=None, timeout=None): +def _make_request(scheme, host, port, username, password, path, method='GET', data=None, query=None, timeout=None, content_type=None): path = _DOUBLE_SLASH_REGEX.sub('/', path) url = '%(scheme)s://%(host)s%(port)s%(path)s' % { 'scheme': scheme, @@ -51,8 +51,12 @@ def _make_request(scheme, host, port, username, password, path, method='GET', da if timeout is None: timeout = settings.REMOTE_REQUEST_TIMEOUT - - return HTTPRequest(url, method, body=data, connect_timeout=timeout, request_timeout=timeout) + + headers = {} + if content_type: + headers['Content-Type'] = content_type + + return HTTPRequest(url, method, body=data, connect_timeout=timeout, request_timeout=timeout, headers=headers) def _callback_wrapper(callback): @@ -117,7 +121,8 @@ def list(local_config, callback): logging.debug('listing remote cameras on %(url)s' % { 'url': pretty_camera_url(local_config, camera=False)}) - request = _make_request(scheme, host, port, username, password, path + '/config/list/') + request = _make_request(scheme, host, port, username, password, + path + '/config/list/') def on_response(response): def make_camera_response(c): @@ -162,7 +167,8 @@ def get_config(local_config, callback): 'id': camera_id, 'url': pretty_camera_url(local_config)}) - request = _make_request(scheme, host, port, username, password, path + '/config/%(id)s/get/' % {'id': camera_id}) + request = _make_request(scheme, host, port, username, password, + path + '/config/%(id)s/get/' % {'id': camera_id}) def on_response(response): if response.error: @@ -207,7 +213,9 @@ def set_config(local_config, ui_config, callback): ui_config = json.dumps(ui_config) - request = _make_request(scheme, host, port, username, password, path + '/config/%(id)s/set/' % {'id': camera_id}, method='POST', data=ui_config) + request = _make_request(scheme, host, port, username, password, + path + '/config/%(id)s/set/' % {'id': camera_id}, + method='POST', data=ui_config, content_type='application/json') def on_response(response): if response.error: @@ -233,7 +241,9 @@ def set_preview(local_config, controls, callback): data = json.dumps(controls) - request = _make_request(scheme, host, port, username, password, path + '/config/%(id)s/set_preview/' % {'id': camera_id}, method='POST', data=data) + request = _make_request(scheme, host, port, username, password, + path + '/config/%(id)s/set_preview/' % {'id': camera_id}, + method='POST', data=data, content_type='application/json') def on_response(response): if response.error: @@ -250,12 +260,42 @@ def set_preview(local_config, controls, callback): http_client.fetch(request, _callback_wrapper(on_response)) -def get_current_picture(local_config, width, height, callback): +def test(local_config, data, callback): scheme, host, port, username, password, path, camera_id = _remote_params(local_config) - - logging.debug('getting current picture for remote camera %(id)s on %(url)s' % { + what = data['what'] + logging.debug('testing %(what)s on remote camera %(id)s, on %(url)s' % { + 'what': what, 'id': camera_id, 'url': pretty_camera_url(local_config)}) + + data = json.dumps(data) + + request = _make_request(scheme, host, port, username, password, + path + '/config/%(id)s/test/' % {'id': camera_id}, + method='POST', data=data, content_type='application/json') + + def on_response(response): + if response.error: + logging.error('failed to test %(what)s on remote camera %(id)s, on %(url)s: %(msg)s' % { + 'what': what, + 'id': camera_id, + 'url': pretty_camera_url(local_config), + 'msg': utils.pretty_http_error(response)}) + + return callback(error=utils.pretty_http_error(response)) + + callback() + + http_client = AsyncHTTPClient() + http_client.fetch(request, _callback_wrapper(on_response)) + + +def get_current_picture(local_config, width, height, callback): + scheme, host, port, username, password, path, camera_id = _remote_params(local_config) + +# logging.debug('getting current picture for remote camera %(id)s on %(url)s' % { +# 'id': camera_id, +# 'url': pretty_camera_url(local_config)}) query = {} @@ -265,7 +305,9 @@ def get_current_picture(local_config, width, height, callback): if height: query['height'] = str(height) - request = _make_request(scheme, host, port, username, password, path + '/picture/%(id)s/current/' % {'id': camera_id}, query=query) + request = _make_request(scheme, host, port, username, password, + path + '/picture/%(id)s/current/' % {'id': camera_id}, + query=query) def on_response(response): motion_detected = False @@ -303,8 +345,10 @@ def list_media(local_config, media_type, prefix, callback): query['prefix'] = prefix # timeout here is 10 times larger than usual - we expect a big delay when fetching the media list - request = _make_request(scheme, host, port, username, password, path + '/%(media_type)s/%(id)s/list/' % { - 'id': camera_id, 'media_type': media_type}, query=query, timeout=10 * settings.REMOTE_REQUEST_TIMEOUT) + request = _make_request(scheme, host, port, username, password, + path + '/%(media_type)s/%(id)s/list/' % { + 'id': camera_id, 'media_type': media_type}, query=query, + timeout=10 * settings.REMOTE_REQUEST_TIMEOUT) def on_response(response): if response.error: @@ -345,7 +389,8 @@ def get_media_content(local_config, filename, media_type, callback): 'filename': filename} # timeout here is 10 times larger than usual - we expect a big delay when fetching the media list - request = _make_request(scheme, host, port, username, password, path, timeout=10 * settings.REMOTE_REQUEST_TIMEOUT) + request = _make_request(scheme, host, port, username, password, + path, timeout=10 * settings.REMOTE_REQUEST_TIMEOUT) def on_response(response): if response.error: @@ -377,7 +422,8 @@ def make_zipped_content(local_config, media_type, group, callback): 'group': group} # timeout here is 100 times larger than usual - we expect a big delay - request = _make_request(scheme, host, port, username, password, prepare_path, timeout=100 * settings.REMOTE_REQUEST_TIMEOUT) + request = _make_request(scheme, host, port, username, password, + prepare_path, timeout=100 * settings.REMOTE_REQUEST_TIMEOUT) def on_response(response): if response.error: @@ -412,11 +458,12 @@ def get_zipped_content(local_config, media_type, key, group, callback): 'id': camera_id, 'url': pretty_camera_url(local_config)}) - request = _make_request(scheme, host, port, username, password, path + '/%(media_type)s/%(id)s/zipped/%(group)s/?key=%(key)s' % { - 'media_type': media_type, - 'group': group, - 'id': camera_id, - 'key': key}, + request = _make_request(scheme, host, port, username, password, + path + '/%(media_type)s/%(id)s/zipped/%(group)s/?key=%(key)s' % { + 'media_type': media_type, + 'group': group, + 'id': camera_id, + 'key': key}, timeout=10 * settings.REMOTE_REQUEST_TIMEOUT) def on_response(response): @@ -454,7 +501,8 @@ def make_timelapse_movie(local_config, framerate, interval, group, callback): 'framerate': framerate, 'group': group} - request = _make_request(scheme, host, port, username, password, path, timeout=100 * settings.REMOTE_REQUEST_TIMEOUT) + request = _make_request(scheme, host, port, username, password, + path, timeout=100 * settings.REMOTE_REQUEST_TIMEOUT) def on_response(response): if response.error: @@ -491,9 +539,10 @@ def check_timelapse_movie(local_config, group, callback): 'id': camera_id, 'url': pretty_camera_url(local_config)}) - request = _make_request(scheme, host, port, username, password, path + '/picture/%(id)s/timelapse/%(group)s/?check=true' % { - 'id': camera_id, - 'group': group}) + request = _make_request(scheme, host, port, username, password, + path + '/picture/%(id)s/timelapse/%(group)s/?check=true' % { + 'id': camera_id, + 'group': group}) def on_response(response): if response.error: @@ -527,10 +576,11 @@ def get_timelapse_movie(local_config, key, group, callback): 'id': camera_id, 'url': pretty_camera_url(local_config)}) - request = _make_request(scheme, host, port, username, password, path + '/picture/%(id)s/timelapse/%(group)s/?key=%(key)s' % { - 'id': camera_id, - 'group': group, - 'key': key}, + request = _make_request(scheme, host, port, username, password, + path + '/picture/%(id)s/timelapse/%(group)s/?key=%(key)s' % { + 'id': camera_id, + 'group': group, + 'key': key}, timeout=10 * settings.REMOTE_REQUEST_TIMEOUT) def on_response(response): @@ -573,7 +623,8 @@ def get_media_preview(local_config, filename, media_type, width, height, callbac if height: query['height'] = str(height) - request = _make_request(scheme, host, port, username, password, path, query=query) + request = _make_request(scheme, host, port, username, password, + path, query=query) def on_response(response): if response.error: @@ -604,7 +655,9 @@ def del_media_content(local_config, filename, media_type, callback): 'id': camera_id, 'filename': filename} - request = _make_request(scheme, host, port, username, password, path, method='POST', data='{}', timeout=settings.REMOTE_REQUEST_TIMEOUT) + request = _make_request(scheme, host, port, username, password, + path, method='POST', data='{}', + timeout=settings.REMOTE_REQUEST_TIMEOUT, content_type='application/json') def on_response(response): if response.error: @@ -635,7 +688,9 @@ def del_media_group(local_config, group, media_type, callback): 'id': camera_id, 'group': group} - request = _make_request(scheme, host, port, username, password, path, method='POST', data='{}', timeout=settings.REMOTE_REQUEST_TIMEOUT) + request = _make_request(scheme, host, port, username, password, path, + method='POST', data='{}', + timeout=settings.REMOTE_REQUEST_TIMEOUT, content_type='application/json') def on_response(response): if response.error: diff --git a/motioneye/server.py b/motioneye/server.py index 8a8b8d2..f379ba5 100644 --- a/motioneye/server.py +++ b/motioneye/server.py @@ -168,8 +168,8 @@ def _log_request(handler): handler_mapping = [ (r'^/$', handlers.MainHandler), (r'^/config/main/(?Pset|get)/?$', handlers.ConfigHandler), - (r'^/config/(?P\d+)/(?Pget|set|rem|set_preview)/?$', handlers.ConfigHandler), - (r'^/config/(?Padd|list|backup|restore|test|authorize)/?$', handlers.ConfigHandler), + (r'^/config/(?P\d+)/(?Pget|set|rem|set_preview|test|authorize)/?$', handlers.ConfigHandler), + (r'^/config/(?Padd|list|backup|restore)/?$', handlers.ConfigHandler), (r'^/picture/(?P\d+)/(?Pcurrent|list|frame)/?$', handlers.PictureHandler), (r'^/picture/(?P\d+)/(?Pdownload|preview|delete)/(?P.+?)/?$', handlers.PictureHandler), (r'^/picture/(?P\d+)/(?Pzipped|timelapse|delete_all)/(?P.*?)/?$', handlers.PictureHandler), @@ -330,7 +330,8 @@ def run(): import motionctl import motioneye import smbctl - import tasker + import tasks + import uploadservices import wsswitch configure_signals() @@ -353,8 +354,11 @@ def run(): wsswitch.start() logging.info('wsswitch started') - tasker.start() - logging.info('tasker started') + tasks.start() + logging.info('tasks started') + + uploadservices.load() + logging.info('upload services loaded') if settings.MJPG_CLIENT_TIMEOUT: mjpgclient.start() @@ -377,6 +381,9 @@ def run(): logging.info('server stopped') + tasks.stop() + logging.info('tasks stopped') + if cleanup.running(): cleanup.stop() logging.info('cleanup stopped') diff --git a/motioneye/static/css/main.css b/motioneye/static/css/main.css index 2c82405..c0dbc0d 100644 --- a/motioneye/static/css/main.css +++ b/motioneye/static/css/main.css @@ -375,7 +375,8 @@ div.normal-button { div.update-button, div.backup-button, -div.restore-button { +div.restore-button, +div.test-button { background: #317CAD; } diff --git a/motioneye/static/js/main.js b/motioneye/static/js/main.js index 2a43652..5174eea 100644 --- a/motioneye/static/js/main.js +++ b/motioneye/static/js/main.js @@ -495,6 +495,18 @@ String.prototype.format = function () { return text; }; + + /* various */ + +function authorizeUpload() { + var service = $('#uploadServiceSelect').val(); + var cameraId = $('#cameraSelect').val(); + var url = basePath + 'config/' + cameraId + '/authorize/?service=' + service; + url = addAuthParams('GET', url); + + window.open(url, '_blank'); +} + /* UI initialization */ @@ -1352,6 +1364,15 @@ function cameraUi2Dict() { 'network_username': $('#networkUsernameEntry').val(), 'network_password': $('#networkPasswordEntry').val(), 'root_directory': $('#rootDirectoryEntry').val(), + 'upload_enabled': $('#uploadEnabledSwitch')[0].checked, + 'upload_service': $('#uploadServiceSelect').val(), + 'upload_server': $('#uploadServerEntry').val(), + 'upload_port': $('#uploadPortEntry').val(), + 'upload_method': $('#uploadMethodSelect').val(), + 'upload_location': $('#uploadLocationEntry').val(), + 'upload_username': $('#uploadUsernameEntry').val(), + 'upload_password': $('#uploadPasswordEntry').val(), + 'upload_authorization_key': $('#uploadAuthorizationKeyEntry').val(), /* text overlay */ 'text_overlay': $('#textOverlaySwitch')[0].checked, @@ -1623,12 +1644,22 @@ function dict2CameraUi(dict) { if (dict['disk_total'] != 0) { percent = parseInt(dict['disk_used'] * 100 / dict['disk_total']); } - + $('#diskUsageProgressBar').each(function () { this.setProgress(percent); this.setText((dict['disk_used'] / 1073741824).toFixed(1) + '/' + (dict['disk_total'] / 1073741824).toFixed(1) + ' GB (' + percent + '%)'); }); markHideIfNull('disk_used', 'diskUsageProgressBar'); + $('#uploadEnabledSwitch')[0].checked = dict['upload_enabled']; markHideIfNull('upload_enabled', 'uploadEnabledSwitch'); + $('#uploadServiceSelect').val(dict['upload_service']); markHideIfNull('upload_service', 'uploadServiceSelect'); + $('#uploadServerEntry').val(dict['upload_server']); markHideIfNull('upload_server', 'uploadServerEntry'); + $('#uploadPortEntry').val(dict['upload_port']); markHideIfNull('upload_port', 'uploadPortEntry'); + $('#uploadMethodSelect').val(dict['upload_method']); markHideIfNull('upload_method', 'uploadMethodSelect'); + $('#uploadLocationEntry').val(dict['upload_location']); markHideIfNull('upload_location', 'uploadLocationEntry'); + $('#uploadUsernameEntry').val(dict['upload_username']); markHideIfNull('upload_username', 'uploadUsernameEntry'); + $('#uploadPasswordEntry').val(dict['upload_password']); markHideIfNull('upload_password', 'uploadPasswordEntry'); + $('#uploadAuthorizationKeyEntry').val(dict['upload_authorization_key']); markHideIfNull('upload_authorization_key', 'uploadAuthorizationKeyEntry'); + /* text overlay */ $('#textOverlaySwitch')[0].checked = dict['text_overlay']; markHideIfNull('text_overlay', 'textOverlaySwitch'); $('#leftTextSelect').val(dict['left_text']); markHideIfNull('left_text', 'leftTextSelect'); @@ -2272,6 +2303,34 @@ function doRestore() { }); } +function doTestUpload() { + showModalDialog('
', null, null, true); + + var data = { + what: 'upload_service', + service: $('#uploadServiceSelect').val(), + server: $('#uploadServerEntry').val(), + port: $('#uploadPortEntry').val(), + method: $('#uploadMethodSelect').val(), + location: $('#uploadLocationEntry').val(), + username: $('#uploadUsernameEntry').val(), + password: $('#uploadPasswordEntry').val(), + authorization_key: $('#uploadAuthorizationKeyEntry').val() + }; + + var cameraId = $('#cameraSelect').val(); + + ajax('POST', basePath + 'config/' + cameraId + '/test/', data, function (data) { + hideModalDialog(); /* progress */ + if (data.error) { + showErrorMessage('Accessing the upload service failed: ' + data.error + '!'); + } + else { + showPopupMessage('Accessing the upload service succeeded!', 'info'); + } + }); +} + function doDownloadZipped(cameraId, groupKey) { showModalDialog('
', null, null, true); ajax('GET', basePath + 'picture/' + cameraId + '/zipped/' + groupKey + '/', null, function (data) { @@ -4022,6 +4081,9 @@ $(document).ready(function () { $('div#backupButton').click(doBackup); $('div#restoreButton').click(doRestore); + /* test buttons */ + $('div#uploadTestButton').click(doTestUpload); + /* prevent scroll events on settings div from propagating TODO this does not actually work */ $('div.settings').mousewheel(function (e, d) { var t = $(this); diff --git a/motioneye/tasker.py b/motioneye/tasks.py similarity index 83% rename from motioneye/tasker.py rename to motioneye/tasks.py index f2f6db7..85ca7b1 100644 --- a/motioneye/tasker.py +++ b/motioneye/tasks.py @@ -41,15 +41,53 @@ def start(): global _pool io_loop = IOLoop.instance() - io_loop.add_timeout(datetime.timedelta(seconds=_INTERVAL), check_tasks) + io_loop.add_timeout(datetime.timedelta(seconds=_INTERVAL), _check_tasks) + + def init_pool_process(): + import signal + + signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGTERM, signal.SIG_IGN) _load() - _pool = multiprocessing.Pool(_POOL_SIZE) + _pool = multiprocessing.Pool(_POOL_SIZE, initializer=init_pool_process) + + +def stop(): + global _pool + + #_pool.terminate() + _pool = None + + +def add(when, func, tag=None, async=False, **params): + if len(_tasks) >= _MAX_TASKS: + return logging.error('the maximum number of tasks (%d) has been reached' % _MAX_TASKS) + + now = time.time() + + if isinstance(when, int): # delay, in seconds + when += now + + elif isinstance(when, datetime.timedelta): + when = now + when.total_seconds() + + elif isinstance(when, datetime.datetime): + when = calendar.timegm(when.timetuple()) + + i = 0 + while i < len(_tasks) and _tasks[i][0] <= when: + i += 1 + logging.debug('adding task "%s" in %d seconds' % (tag or func.func_name, when - now)) + _tasks.insert(i, (when, func, tag, async, params)) + + _save() -def check_tasks(): + +def _check_tasks(): io_loop = IOLoop.instance() - io_loop.add_timeout(datetime.timedelta(seconds=_INTERVAL), check_tasks) + io_loop.add_timeout(datetime.timedelta(seconds=_INTERVAL), _check_tasks) now = time.time() changed = False @@ -73,29 +111,6 @@ def check_tasks(): _save() -def add_task(when, func, tag=None, async=False, **params): - if len(_tasks) >= _MAX_TASKS: - return logging.error('the maximum number of tasks (%d) has been reached' % _MAX_TASKS) - - if isinstance(when, int): # delay, in seconds - when += time.time() - - elif isinstance(when, datetime.timedelta): - when = time.time() + when.total_seconds() - - elif isinstance(when, datetime.datetime): - when = calendar.timegm(when.timetuple()) - - i = 0 - while i < len(_tasks) and _tasks[i][0] <= when: - i += 1 - - logging.debug('adding task "%s" in %d seconds' % (tag or func.func_name, when - time.time())) - _tasks.insert(i, (when, func, tag, async, params)) - - _save() - - def _load(): global _tasks diff --git a/motioneye/templates/main.html b/motioneye/templates/main.html index 0c07200..1a9c2e4 100644 --- a/motioneye/templates/main.html +++ b/motioneye/templates/main.html @@ -344,12 +344,12 @@ Upload Service ? @@ -364,21 +364,21 @@ ? - - Location - - ? - Method - ? + + Location + + ? + Username @@ -389,20 +389,25 @@ ? - + Authorization Key - - ? + + ? ? + + +
Test Service
+ ? + {% for config in camera_sections.get('storage', {}).get('configs', []) %} {{config_item(config)}} {% endfor %} diff --git a/motioneye/uploadservices.py b/motioneye/uploadservices.py new file mode 100644 index 0000000..2e427ca --- /dev/null +++ b/motioneye/uploadservices.py @@ -0,0 +1,362 @@ + +# Copyright (c) 2013 Calin Crisan +# This file is part of motionEye. +# +# motionEye is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import httplib2 +import json +import logging +import mimetypes +import os.path +import urllib +import urllib2 + +from oauth2client.client import OAuth2WebServerFlow, Credentials + +import settings + + +_STATE_FILE_NAME = 'uploadservices.json' + +_services = {} + + +class UploadService(object): + NAME = 'base' + + def __init__(self, **kwargs): + pass + + def __str__(self): + return self.NAME + + def get_authorize_url(self): + return '/' + + def test_access(self): + return True + + def upload_file(self, filename): + self.debug('uploading file "%s" to %s' % (filename, self)) + + try: + st = os.stat(filename) + + except Exception as e: + msg = 'failed to open file "%s": %s' % (filename, e) + self.error(msg) + raise Exception(msg) + + if st.st_size > self.MAX_FILE_SIZE: + msg = 'file "%s" is too large (%sMB/%sMB)' % (filename, st.st_size / 1024 / 1024, self.MAX_FILE_SIZE / 1024 / 1024) + self.error(msg) + raise Exception(msg) + + try: + f = open(filename) + + except Exception as e: + msg = 'failed to open file "%s": %s' % (filename, e) + self.error(msg) + raise Exception(msg) + + data = f.read() + self.debug('size of "%s" is %.1fMB' % (filename, len(data) / 1024.0 / 1024)) + + mime_type = mimetypes.guess_type(filename)[0] or 'image/jpeg' + self.debug('mime type of "%s" is "%s"' % (filename, mime_type)) + + self.upload_data(filename, mime_type, data) + + def upload_data(self, filename, mime_type, data): + pass + + def dump(self): + return {} + + def load(self, data): + pass + + def log(self, level, *args, **kwargs): + logging.log(level, *args, **kwargs) + + def debug(self, *args, **kwargs): + self.log(logging.DEBUG, *args, **kwargs) + + def info(self, *args, **kwargs): + self.log(logging.INFO, *args, **kwargs) + + def error(self, *args, **kwargs): + self.log(logging.ERROR, *args, **kwargs) + + @staticmethod + def get_service_classes(): + return {c.NAME: c for c in UploadService.__subclasses__()} + + +class GoogleDrive(UploadService): + NAME = 'gdrive' + CLIENT_ID = '349038943026-m16svdadjrqc0c449u4qv71v1m1niu5o.apps.googleusercontent.com' + CLIENT_NOT_SO_SECRET = 'jjqbWmICpA0GvbhsJB3okX7s' + SCOPE = 'https://www.googleapis.com/auth/drive' + CHILDREN_URL = 'https://www.googleapis.com/drive/v2/files/%(parent_id)s/children?q=%(query)s' + CHILDREN_QUERY = "'%(parent_id)s' in parents and title = '%(child_name)s'" + UPLOAD_URL = 'https://www.googleapis.com/upload/drive/v2/files?uploadType=multipart' + BOUNDARY = 'motioneye_multipart_boundary' + MAX_FILE_SIZE = 128 * 1024 * 1024 # 128 MB + + def __init__(self, location=None, authorization_key=None, credentials=None, **kwargs): + self._location = location + self._authorization_key = authorization_key + self._credentials = credentials + self._folder_id = None + + def get_authorize_url(self): + flow = OAuth2WebServerFlow(client_id=self.CLIENT_ID, client_secret=self.CLIENT_NOT_SO_SECRET, + scope='https://www.googleapis.com/auth/drive', redirect_uri='urn:ietf:wg:oauth:2.0:oob') + + return flow.step1_get_authorize_url() + + def test_access(self): + try: + self._folder_id = None + self._get_folder_id() + return True + + except Exception as e: + return str(e) + + def upload_data(self, filename, mime_type, data): + metadata = { + 'title': filename, + 'parents': [{'id': self._get_folder_id()}] + } + + body = ['--' + self.BOUNDARY] + body.append('Content-Type: application/json; charset=UTF-8') + body.append('') + body.append(json.dumps(metadata)) + body.append('') + + body.append('--' + self.BOUNDARY) + body.append('Content-Type: %s' % mime_type) + body.append('') + body.append('') + body = '\r\n'.join(body) + body += data + body += '\r\n--%s--' % self.BOUNDARY + + headers = { + 'Content-Type': 'multipart/related; boundary="%s"' % self.BOUNDARY, + 'Content-Length': len(body) + } + + self._request(self.UPLOAD_URL, body, headers) + + def dump(self): + return { + 'location': self._location, + 'credentials': self._credentials and json.loads(self._credentials.to_json()), + 'authorization_key': self._authorization_key, + 'folder_id': self._folder_id + } + + def load(self, data): + if 'location' in data: + self._location = data['location'] + self._folder_id = None # invalidate the folder + if 'credentials' in data: + self._credentials = Credentials.new_from_json(json.dumps(data['credentials'])) + if 'authorization_key' in data: + self._authorization_key = data['authorization_key'] + if 'folder_id' in data: + self._folder_id = data['folder_id'] + + def _get_folder_id(self): + if not self._folder_id: + self.debug('finding folder id for location "%s"' % self._location) + self._folder_id = self._get_folder_id_by_path(self._location) + save() + + return self._folder_id + + def _get_folder_id_by_path(self, path): + path = [p.strip() for p in path.split('/') if p.strip()] + + parent_id = 'root' + for name in path: + parent_id = self._get_folder_id_by_name(parent_id, name) + + return parent_id + + def _get_folder_id_by_name(self, parent_id, child_name): + query = self.CHILDREN_QUERY % {'parent_id': parent_id, 'child_name': child_name} + query = urllib.quote(query) + url = self.CHILDREN_URL % {'parent_id': parent_id, 'query': query} + response = self._request(url) + try: + response = json.loads(response) + + except Exception: + self.error("response doesn't seem to be a valid json") + raise + + items = response.get('items') + if not items: + msg = 'folder with name "%s" could not be found' % child_name + self.error(msg) + raise Exception(msg) + + return items[0]['id'] + + def _request(self, url, body=None, headers=None): + if not self._authorization_key: + msg = 'missing authorization key' + self.error(msg) + raise Exception(msg) + + if not self._credentials: + self.debug('requesting access token') + flow = self._get_oauth2_flow() + try: + self._credentials = flow.step2_exchange(self._authorization_key) + save() + + except Exception as e: + self.error('failed to obtain access token: %s' % e) + raise + + headers = headers or {} + headers['Authorization'] = 'Bearer %s' % self._credentials.access_token + + self.debug('requesting %s' % url) + request = urllib2.Request(url, data=body, headers=headers) + try: + response = urllib2.urlopen(request) + + except urllib2.HTTPError as e: + if e.code == 403: # unauthorized, access token may have expired + try: + self.debug('access token might have expired, refreshing it') + self._credentials.refresh(httplib2.Http()) + save() + + except Exception as e: + self.error('refreshing access token failed') + raise + + else: + self.error('request failed: %s' % e) + raise + + except Exception as e: + self.error('request failed: %s' % e) + raise + + return response.read() + + def _get_oauth2_flow(self): + return OAuth2WebServerFlow(client_id=self.CLIENT_ID, client_secret=self.CLIENT_NOT_SO_SECRET, + scope=self.SCOPE, redirect_uri='urn:ietf:wg:oauth:2.0:oob') + + +def get(camera_id, name, create=True): + camera_id = str(camera_id) + service = _services.get(camera_id, {}).get(name) + if not service and create: + classes = UploadService.get_service_classes() + cls = classes.get(name) + if cls: + logging.debug('creating upload service %s for camera with id %s' % (name, camera_id)) + service = cls() + _services.setdefault(camera_id, {})[name] = service + + return service + + +def load(): + file_path = os.path.join(settings.CONF_PATH, _STATE_FILE_NAME) + + if os.path.exists(file_path): + logging.debug('loading upload services state from "%s"...' % file_path) + + try: + file = open(file_path, 'r') + + except Exception as e: + logging.error('could not open upload services state file "%s": %s' % (file_path, e)) + + return + + try: + data = json.load(file) + + except Exception as e: + return logging.error('could not read upload services state from file "%s": %s'(file_path, e)) + + finally: + file.close() + + for camera_id, d in data.iteritems(): + for name, state in d.iteritems(): + camera_services = _services.setdefault(camera_id, {}) + cls = UploadService.get_service_classes().get(name) + if cls: + service = cls() + service.load(state) + + camera_services[name] = service + + logging.debug('loaded upload service "%s" for camera with id "%s"' % (name, camera_id)) + + +def save(): + file_path = os.path.join(settings.CONF_PATH, _STATE_FILE_NAME) + + logging.debug('saving upload services state to "%s"...' % file_path) + + try: + file = open(file_path, 'w') + + except Exception as e: + logging.error('could not open upload services state file "%s": %s' % (file_path, e)) + + return + + data = {} + for camera_id, camera_services in _services.iteritems(): + for name, service in camera_services.iteritems(): + data.setdefault(camera_id, {})[name] = service.dump() + + try: + json.dump(data, file, sort_keys=True, indent=4) + + except Exception as e: + logging.error('could not save upload services state to file "%s": %s'(file_path, e)) + + finally: + file.close() + + +def upload_media_file(camera_id, service_name, filename): + service = get(camera_id, service_name, create=False) + if not service: + return logging.error('service "%s" not initialized for camera with id %s' % (service_name, camera_id)) + + try: + service.upload_file(filename) + + except Exception as e: + logging.error('failed to upload file "%s" with service %s: %s' % (filename, service, e)) -- 2.39.5