Improvements in url fetch retry mechanism

- Implement exponential back-off for URL fetch retries
- Use back-off mechanism also for RTL EPG source

diff --git a/src/main/java/org/vanbest/xmltv/AbstractEPGSource.java b/src/main/java/org/vanbest/xmltv/AbstractEPGSource.java
index 1ca2cd3a5578cf331de019dd67e3885631c6f98e..aa41845ab6160274b3d8640702f6239fa6b43302 100644 (file)
--- a/src/main/java/org/vanbest/xmltv/AbstractEPGSource.java
+++ b/src/main/java/org/vanbest/xmltv/AbstractEPGSource.java
@@ -50,10 +50,10 @@ public abstract class AbstractEPGSource implements EPGSource {
        }
 
        protected String fetchURL(URL url) throws Exception {
-               Thread.sleep(config.niceMilliseconds);
                StringBuffer buf = new StringBuffer();
                boolean done = false;
                for (int count = 0; !done; count++) {
+                       Thread.sleep(config.niceMilliseconds*(1<<count));
                        try {
                                BufferedReader reader = new BufferedReader(
                                                new InputStreamReader(url.openStream()));
@@ -133,4 +133,4 @@ public abstract class AbstractEPGSource implements EPGSource {
                return result;
        }
 
-}
\ No newline at end of file
+}

diff --git a/src/main/java/org/vanbest/xmltv/RTL.java b/src/main/java/org/vanbest/xmltv/RTL.java
index 246ea8884d523ca69e7af7d681df3dad040c9781..0c439b5d8ae2ca9092d5f4c6b81b2792585f2625 100644 (file)
--- a/src/main/java/org/vanbest/xmltv/RTL.java
+++ b/src/main/java/org/vanbest/xmltv/RTL.java
@@ -98,6 +98,32 @@ public class RTL extends AbstractEPGSource implements EPGSource {
                return NAME;\r
        }\r
 \r
+\r
+       private Document fetchXML(URL url) throws Exception {\r
+               Document xml = null;\r
+               boolean done = false;\r
+               for (int count = 0; !done; count++) {\r
+                       Thread.sleep(config.niceMilliseconds*(1<<count));\r
+                       try {\r
+                               xml = DocumentBuilderFactory.newInstance()\r
+                                       .newDocumentBuilder().parse(url.openStream());\r
+                               done = true;\r
+                       } catch (Exception e) {\r
+                               if (!config.quiet) {\r
+                                       logger.warn("Error fetching from url " + url + ", count="\r
+                                                       + count);\r
+                               }\r
+                               if (count >= MAX_FETCH_TRIES) {\r
+                                       stats.fetchErrors++;\r
+                                       logger.debug("Error getting data from url", e);\r
+                                       throw new Exception("Error getting data from url "\r
+                                                       + url, e);\r
+                               }\r
+                      }\r
+                }\r
+                return xml;\r
+        }\r
+\r
        public List<Channel> getChannels() {\r
                List<Channel> result = new ArrayList<Channel>(10);\r
 \r
@@ -109,11 +135,11 @@ public class RTL extends AbstractEPGSource implements EPGSource {
                }\r
                Document xml = null;\r
                try {\r
-                       xml = DocumentBuilderFactory.newInstance().newDocumentBuilder()\r
-                                       .parse(url.openStream());\r
+                       xml = fetchXML(url);\r
                } catch (Exception e) {\r
-                       logger.error("Exception reading info from " + url\r
+                       logger.error("Exception reading RTL channel listing from " + url\r
                                        + " and transforming to XML", e);\r
+                        return result;\r
                }\r
                Element root = xml.getDocumentElement();\r
                String json = root.getTextContent();\r
@@ -165,9 +191,7 @@ public class RTL extends AbstractEPGSource implements EPGSource {
        private void fetchDetail(Programme prog, DateStatus dateStatus, String id)\r
                        throws Exception {\r
                URL url = detailUrl(id);\r
-               Thread.sleep(config.niceMilliseconds);\r
-               Document xml = DocumentBuilderFactory.newInstance()\r
-                               .newDocumentBuilder().parse(url.openStream());\r
+               Document xml = fetchXML(url);\r
                Element root = xml.getDocumentElement();\r
                if (root.hasAttributes()) {\r
                        logger.warn("Unknown attributes for RTL detail root node");\r
@@ -333,11 +357,7 @@ public class RTL extends AbstractEPGSource implements EPGSource {
                                channelMap.put(c.id, c);\r
                }\r
                URL url = programmeUrl(day);\r
-               // String xmltext = fetchURL(url);\r
-               // System.out.println(xmltext);\r
-               Thread.sleep(config.niceMilliseconds);\r
-               Document xml = DocumentBuilderFactory.newInstance()\r
-                               .newDocumentBuilder().parse(url.openStream());\r
+               Document xml = fetchXML(url);\r
                Element root = xml.getDocumentElement();\r
                Date date = new SimpleDateFormat("yyyy-MM-dd").parse(root\r
                                .getAttribute("date"));\r

diff --git a/src/test/java/org/vanbest/xmltv/RTLTest.java b/src/test/java/org/vanbest/xmltv/RTLTest.java
index e1cef3290b58abe7eb363db1b3b4471957e127ce..69b64e012900392ed35018d93968c2bb71b28fe4 100644 (file)
--- a/src/test/java/org/vanbest/xmltv/RTLTest.java
+++ b/src/test/java/org/vanbest/xmltv/RTLTest.java
@@ -67,6 +67,7 @@ public class RTLTest {
                for (Channel c : channels) {
                        if (c.defaultName().equals("RTL 4")) {
                                rtl4 = c;
+                                break;
                        }
                }
                assertNotNull("Should be able to find RTL 4 channel", rtl4);