From: Jan-Pascal van Best <janpascal@vanbest.org>
Date: Wed, 10 Feb 2016 22:12:36 +0000 (+0100)
Subject: Run background stream listener is separate process, start it with runstream.py
X-Git-Tag: v1.0~2
X-Git-Url: http://www.vanbest.org/gitweb/?a=commitdiff_plain;h=03a0c6605a96b4d9e4e09bf49cd81ba501475bd9;p=tweet-django-debian

Run background stream listener is separate process, start it with runstream.py
---

diff --git a/runstream.py b/runstream.py
new file mode 100644
index 0000000..6c532a3
--- /dev/null
+++ b/runstream.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
+import os
+import sys
+import time
+import logging
+
+import django
+
+logger = logging.getLogger(__name__)
+
+if __name__ == "__main__":
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tweet_django.settings")
+    django.setup()
+
+    from tweet import streamrunner
+    streamrunner.run_stream()
diff --git a/tweet/__init__.py b/tweet/__init__.py
index 012103f..5785fa0 100644
--- a/tweet/__init__.py
+++ b/tweet/__init__.py
@@ -1,30 +1 @@
 # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
-
-import threading
-import time
-
-import  django.apps
-
-_started = False
-
-def startup_thread():
-    global _started
-
-    if _started:
-        return
-    _started = True
-    print("Waiting until app starts..")
-    # Wait until the app started before accessing the database
-    while not django.apps.apps.ready:
-        time.sleep(1)
-    
-    # Only import streamrunner here, because it imports models which won't
-    # be available earlier
-    from . import streamrunner
-    from .models import Settings
-    settings = Settings.get()
-    streamrunner.start_stream(settings.stream_terms)
-
-print("Starting startup thread...")
-thread = threading.Thread(target=startup_thread)
-thread.start()
diff --git a/tweet/streamrunner.py b/tweet/streamrunner.py
index b3f7985..1177100 100644
--- a/tweet/streamrunner.py
+++ b/tweet/streamrunner.py
@@ -8,7 +8,7 @@ from django.conf import settings
 
 from twython import TwythonStreamer
 
-from .models import Tweet
+from .models import Tweet, Settings
 from .utils import ExcelExporter
 
 logger = logging.getLogger(__name__)
@@ -40,7 +40,7 @@ class Streamer(TwythonStreamer):
 def start_stream(terms):
     global _twitter
     global _thread
-    print("Starting Twitter stream...")
+    logger.info("Starting Twitter stream for {}...".format(terms))
     if len(terms)==0:
         logger.warning("No terms given for twitter stream, not starting stream...")
         return
@@ -48,7 +48,7 @@ def start_stream(terms):
             settings.TWEET_OAUTH_CONSUMER_SECRET, 
             settings.TWEET_OAUTH_ACCESS_TOKEN,
             settings.TWEET_OAUTH_ACCESS_TOKEN_SECRET)
-    _thread = threading.Thread(target=_twitter.statuses.filter,
+    _thread = threading.Thread(target=_twitter.statuses.filter, name="Stream thread",
             kwargs={'track':",".join(terms)}, 
             daemon=True)
     _thread.start()
@@ -61,6 +61,27 @@ def close_stream():
         _twitter.disconnect()
         _twitter = None
 
+def run_stream():
+    logger.info("Starting run_stream()")
+    settings = Settings.get()
+    terms = settings.stream_terms
+    start_stream(terms)
+    while True:
+        time.sleep(5)
+        settings.refresh_from_db()
+        if terms != settings.stream_terms:
+            logger.debug("Stream terms updated in database config, restarting stream...")
+            terms = settings.stream_terms
+            close_stream()
+            # Make sure stream stopped properly
+            time.sleep(5)
+            start_stream(terms)
+
+            logger.info('Running threads:')
+            for t in threading.enumerate():
+                logger.info("    {}".format(t.name))
+
+
 def export_tweets(filename):
     exporter = ExcelExporter(filename)
     tweets = Tweet.objects.all()
diff --git a/tweet/views.py b/tweet/views.py
index afbc736..041a0bf 100644
--- a/tweet/views.py
+++ b/tweet/views.py
@@ -144,13 +144,10 @@ def edit_stream_terms(request):
                 terms.append(request.POST[key])
         logger.info("New terms: {}".format(terms))
         settings.stream_terms = terms
+        # Separate process will pick this up
         settings.save()
 
-        streamrunner.close_stream()
-        # Make sure stream stopped properly
-        time.sleep(5)
-        streamrunner.start_stream(terms)
-        messages.success(request, 'Twitter stream started for {}'.format(terms))
+        messages.success(request, 'Twitter stream terms updated: {}'.format(terms))
 
         return HttpResponseRedirect(reverse('tweet:list_stream'))
     terms = Settings.get().stream_terms