From: Jan-Pascal van Best <janpascal@vanbest.org>
Date: Thu, 3 Jan 2013 14:00:52 +0000 (+0100)
Subject: Improvements in url fetch retry mechanism
X-Git-Tag: 1.1.0~3
X-Git-Url: http://www.vanbest.org/gitweb/?a=commitdiff_plain;h=099a0fe5a403390931259b9a4e171ac0d7cf5eee;p=tv_grab_nl_java

Improvements in url fetch retry mechanism

- Implement exponential back-off for URL fetch retries
- Use back-off mechanism also for RTL EPG source
---

diff --git a/src/main/java/org/vanbest/xmltv/AbstractEPGSource.java b/src/main/java/org/vanbest/xmltv/AbstractEPGSource.java
index 1ca2cd3..aa41845 100644
--- a/src/main/java/org/vanbest/xmltv/AbstractEPGSource.java
+++ b/src/main/java/org/vanbest/xmltv/AbstractEPGSource.java
@@ -50,10 +50,10 @@ public abstract class AbstractEPGSource implements EPGSource {
 	}
 
 	protected String fetchURL(URL url) throws Exception {
-		Thread.sleep(config.niceMilliseconds);
 		StringBuffer buf = new StringBuffer();
 		boolean done = false;
 		for (int count = 0; !done; count++) {
+			Thread.sleep(config.niceMilliseconds*(1<<count));
 			try {
 				BufferedReader reader = new BufferedReader(
 						new InputStreamReader(url.openStream()));
@@ -133,4 +133,4 @@ public abstract class AbstractEPGSource implements EPGSource {
 		return result;
 	}
 
-}
\ No newline at end of file
+}
diff --git a/src/main/java/org/vanbest/xmltv/RTL.java b/src/main/java/org/vanbest/xmltv/RTL.java
index 246ea88..0c439b5 100644
--- a/src/main/java/org/vanbest/xmltv/RTL.java
+++ b/src/main/java/org/vanbest/xmltv/RTL.java
@@ -98,6 +98,32 @@ public class RTL extends AbstractEPGSource implements EPGSource {
 		return NAME;
 	}
 
+
+	private Document fetchXML(URL url) throws Exception {
+		Document xml = null;
+		boolean done = false;
+		for (int count = 0; !done; count++) {
+			Thread.sleep(config.niceMilliseconds*(1<<count));
+           		try {
+                          	xml = DocumentBuilderFactory.newInstance()
+                                  	.newDocumentBuilder().parse(url.openStream());
+                          	done = true;
+                  	} catch (Exception e) {
+				if (!config.quiet) {
+					logger.warn("Error fetching from url " + url + ", count="
+							+ count);
+				}
+				if (count >= MAX_FETCH_TRIES) {
+					stats.fetchErrors++;
+					logger.debug("Error getting data from url", e);
+					throw new Exception("Error getting data from url "
+							+ url, e);
+				}
+         	       }
+                }
+                return xml;
+        }
+
 	public List<Channel> getChannels() {
 		List<Channel> result = new ArrayList<Channel>(10);
 
@@ -109,11 +135,11 @@ public class RTL extends AbstractEPGSource implements EPGSource {
 		}
 		Document xml = null;
 		try {
-			xml = DocumentBuilderFactory.newInstance().newDocumentBuilder()
-					.parse(url.openStream());
+			xml = fetchXML(url);
 		} catch (Exception e) {
-			logger.error("Exception reading info from " + url
+			logger.error("Exception reading RTL channel listing from " + url
 					+ " and transforming to XML", e);
+                        return result;
 		}
 		Element root = xml.getDocumentElement();
 		String json = root.getTextContent();
@@ -165,9 +191,7 @@ public class RTL extends AbstractEPGSource implements EPGSource {
 	private void fetchDetail(Programme prog, DateStatus dateStatus, String id)
 			throws Exception {
 		URL url = detailUrl(id);
-		Thread.sleep(config.niceMilliseconds);
-		Document xml = DocumentBuilderFactory.newInstance()
-				.newDocumentBuilder().parse(url.openStream());
+		Document xml = fetchXML(url);
 		Element root = xml.getDocumentElement();
 		if (root.hasAttributes()) {
 			logger.warn("Unknown attributes for RTL detail root node");
@@ -333,11 +357,7 @@ public class RTL extends AbstractEPGSource implements EPGSource {
 				channelMap.put(c.id, c);
 		}
 		URL url = programmeUrl(day);
-		// String xmltext = fetchURL(url);
-		// System.out.println(xmltext);
-		Thread.sleep(config.niceMilliseconds);
-		Document xml = DocumentBuilderFactory.newInstance()
-				.newDocumentBuilder().parse(url.openStream());
+		Document xml = fetchXML(url);
 		Element root = xml.getDocumentElement();
 		Date date = new SimpleDateFormat("yyyy-MM-dd").parse(root
 				.getAttribute("date"));
diff --git a/src/test/java/org/vanbest/xmltv/RTLTest.java b/src/test/java/org/vanbest/xmltv/RTLTest.java
index e1cef32..69b64e0 100644
--- a/src/test/java/org/vanbest/xmltv/RTLTest.java
+++ b/src/test/java/org/vanbest/xmltv/RTLTest.java
@@ -67,6 +67,7 @@ public class RTLTest {
 		for (Channel c : channels) {
 			if (c.defaultName().equals("RTL 4")) {
 				rtl4 = c;
+                                break;
 			}
 		}
 		assertNotNull("Should be able to find RTL 4 channel", rtl4);