From: Jan-Pascal van Best <janpascal@vanbest.org>
Date: Mon, 9 Apr 2012 08:17:33 +0000 (+0200)
Subject: For analysis, RTL class can now dump CSV file of rtl.nl details
X-Git-Tag: 0.9.1~3
X-Git-Url: http://www.vanbest.org/gitweb/?a=commitdiff_plain;h=c04a0816395cb97e3d7843c50359edd967c5f6b4;p=tv_grab_nl_java

For analysis, RTL class can now dump CSV file of rtl.nl details
---

diff --git a/src/main/java/org/vanbest/xmltv/Config.java b/src/main/java/org/vanbest/xmltv/Config.java
index 60ef333..7ed397b 100644
--- a/src/main/java/org/vanbest/xmltv/Config.java
+++ b/src/main/java/org/vanbest/xmltv/Config.java
@@ -38,6 +38,7 @@ import org.apache.commons.io.FileUtils;
 public class Config {
 	//constants
 	public static final int LOG_INFO = 0x0001;
+	public static final int LOG_DEBUG = 0x0005;
 	public static final int LOG_JSON = 0x0100;
 	private final static int LOG_PROGRAMME_INFO = 0x0200;
 	private final static int CURRENT_FILE_FORMAT=4;
diff --git a/src/main/java/org/vanbest/xmltv/ProgrammeCache.java b/src/main/java/org/vanbest/xmltv/ProgrammeCache.java
index 0a32bb0..55272bb 100644
--- a/src/main/java/org/vanbest/xmltv/ProgrammeCache.java
+++ b/src/main/java/org/vanbest/xmltv/ProgrammeCache.java
@@ -64,7 +64,7 @@ public class ProgrammeCache {
 			db = null;
 			if (!config.quiet) {
 				System.out.println("Unable to open cache database, proceeding without cache");
-				e.printStackTrace();
+				if (config.logLevel>=Config.LOG_DEBUG) e.printStackTrace();
 			}
         }
         boolean recreateTable = false;
@@ -93,7 +93,7 @@ public class ProgrammeCache {
 				if (!config.quiet) {
 					System.out.println("Got SQL exception when trying to find current database schema");
 					System.out.flush();
-					e.printStackTrace();
+					if (config.logLevel>=Config.LOG_DEBUG) e.printStackTrace();
 					System.out.flush();
 				}
 				recreateTable = true;
@@ -121,7 +121,7 @@ public class ProgrammeCache {
 					if (!config.quiet) {
 						System.out.println("Unable to create cache database, proceeding without cache");
 						System.out.flush();
-						e.printStackTrace();
+						if (config.logLevel>=Config.LOG_DEBUG) e.printStackTrace();
 						System.out.flush();
 					}
 					db = null;
@@ -137,7 +137,7 @@ public class ProgrammeCache {
 				if (!config.quiet) {
 					System.out.println("Unable to prepare statements, proceeding without cache");
 					System.out.flush();
-					e.printStackTrace();
+					if (config.logLevel>=Config.LOG_DEBUG) e.printStackTrace();
 					System.out.flush();
 				}
 				db = null;
@@ -162,7 +162,8 @@ public class ProgrammeCache {
 			}
 		} catch (SQLException e) {
 			if (!config.quiet) {
-				e.printStackTrace();
+				System.out.println("Error fetching programme ("+source+","+id+") from cache");
+				if (config.logLevel>=Config.LOG_DEBUG) e.printStackTrace();
 			}
 			return null;
 		}
@@ -192,8 +193,8 @@ public class ProgrammeCache {
 				System.out.println("Weird, cache database update statement affected " + count + " rows");
 			}
 		} catch (SQLException e) {
-			// TODO Auto-generated catch block
-			e.printStackTrace();
+			System.out.println("Error writing programme ("+source+","+id+") to cache");
+			if (config.logLevel>=Config.LOG_DEBUG) e.printStackTrace();
 		}
 	}
 
@@ -221,8 +222,7 @@ public class ProgrammeCache {
 				System.out.println("Cleared " + count + " entries from cache");
 			}
 		} catch (SQLException e) {
-			// TODO Auto-generated catch block
-			e.printStackTrace();
+			if (config.logLevel>=Config.LOG_DEBUG) e.printStackTrace();
 		}
 	}
 
@@ -237,7 +237,7 @@ public class ProgrammeCache {
 				db.close();
 			} catch (SQLException e) {
 				// TODO Auto-generated catch block
-				e.printStackTrace();
+				if (config.logLevel>=Config.LOG_DEBUG) e.printStackTrace();
 			}
 		}
 	}
diff --git a/src/main/java/org/vanbest/xmltv/RTL.java b/src/main/java/org/vanbest/xmltv/RTL.java
index 164a196..0341af6 100644
--- a/src/main/java/org/vanbest/xmltv/RTL.java
+++ b/src/main/java/org/vanbest/xmltv/RTL.java
@@ -1,12 +1,15 @@
 package org.vanbest.xmltv;
 
+import java.io.BufferedOutputStream;
 import java.io.BufferedReader;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.io.PrintWriter;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.sql.Connection;
@@ -64,6 +67,10 @@ public class RTL extends AbstractEPGSource implements EPGSource  {
 	
 	String[] xmlKeys = {"zendernr", "pgmsoort", "genre", "bijvnwlanden", "ondertiteling", "begintijd", "titel", 
 			"site_path", "wwwadres", "presentatie", "omroep", "eindtijd", "inhoud", "tt_inhoud", "alginhoud", "afl_titel", "kijkwijzer" };
+	Map<String,Integer> xmlKeyMap = new HashMap<String,Integer>();
+	
+	static boolean debug = false;
+	PrintWriter debugWriter;
 		
 	class RTLException extends Exception {
 		public RTLException(String s) {
@@ -73,6 +80,11 @@ public class RTL extends AbstractEPGSource implements EPGSource  {
 	
 	public RTL(int sourceId, Config config) {
 		super(sourceId, config);
+		if(debug) {
+			for(int i=0; i<xmlKeys.length; i++) {
+				xmlKeyMap.put(xmlKeys[i], i);
+			}
+		}
 	}
 	
 	public String getName() {
@@ -172,8 +184,16 @@ public class RTL extends AbstractEPGSource implements EPGSource  {
 			}
 			// we have a uitzending_data_item node
 			NodeList subnodes = n.getChildNodes();
+			String[] result = new String[xmlKeys.length];
 			for( int j=0; j<subnodes.getLength(); j++) {
 				try {
+					if (debug) {
+						Node sub = subnodes.item(j);
+						String key = ((Element)sub).getTagName();
+						int index = xmlKeyMap.get(key);
+						String value = "\"" + sub.getTextContent().replaceAll("\\s", " ") + "\"";
+						result[index] = value;
+					}
 					handleNode(prog, date, subnodes.item(j));
 				} catch (RTLException e) {
 					System.out.println(e.getMessage());
@@ -183,6 +203,13 @@ public class RTL extends AbstractEPGSource implements EPGSource  {
 					continue;
 				}
 			}
+			if (debug) {
+				for(int k=0; k<result.length; k++) {
+					debugWriter.print(result[k]);
+					debugWriter.print(",");
+				}
+				debugWriter.println();
+			}
 		}
 	}
 
@@ -204,6 +231,7 @@ public class RTL extends AbstractEPGSource implements EPGSource  {
 		}
 		Element e = (Element)n;
 		String tag = e.getTagName();
+
 		if (e.getTextContent().isEmpty()) {
 			return;
 		}
@@ -213,7 +241,7 @@ public class RTL extends AbstractEPGSource implements EPGSource  {
 			prog.endTime = parseTime(date, e.getTextContent());
 		} else if (tag.equals("omroep")) {
 		} else if (tag.equals("kijkwijzer")) {
-			System.out.println("Kijkwijzer: \"" + e.getTextContent() + "\"");
+			//System.out.println("Kijkwijzer: \"" + e.getTextContent() + "\"");
 		} else if (tag.equals("presentatie")) {
 			// A; A en B; A, B, C en D
 			String[] presentatoren = e.getTextContent().split(", | en ");
@@ -281,8 +309,9 @@ public class RTL extends AbstractEPGSource implements EPGSource  {
 				String starttime = p.getString(0);
 				String title = p.getString(1);
 				String programme_id = p.getString(2);
-				String quark1 = p.getString(3);
-				String quark2 = p.getString(4);
+				String genre_id = p.getString(3); // 1 = amusement, etc
+				String quark2 = p.getString(4); // 0 of 1, movie flag?
+				debugWriter.print("\""+id+"\",\""+starttime+"\",\""+title+"\",\""+genre_id+"\",\""+quark2+"\",");
 				Programme prog = cache.get(getId(), programme_id);
 				if (prog == null) {
 					stats.cacheMisses++;
@@ -335,10 +364,25 @@ public class RTL extends AbstractEPGSource implements EPGSource  {
 
 	/**
 	 * @param args
+	 * @throws FileNotFoundException 
 	 */
-	public static void main(String[] args) {
+	public static void main(String[] args) throws FileNotFoundException {
+		debug = true;
 		Config config = Config.getDefaultConfig();
+		config.niceMilliseconds = 50;
 		RTL rtl = new RTL(2, config);
+		if (debug) {
+			rtl.cache.clear();
+			System.out.println("Writing CSV to rtl.csv");
+			rtl.debugWriter = new PrintWriter( new BufferedOutputStream(new FileOutputStream("rtl.csv")));
+			rtl.debugWriter.print("\"zender\",\"starttime\",\"title\",\"quark1\",\"quark2\",");
+			for(int k=0; k<rtl.xmlKeys.length; k++) {
+				rtl.debugWriter.print(rtl.xmlKeys[k]);
+				rtl.debugWriter.print(",");
+			}
+			rtl.debugWriter.println();
+		}
+
 		try {
 			List<Channel> channels = rtl.getChannels();
 			System.out.println("Channels: " + channels);
@@ -350,9 +394,11 @@ public class RTL extends AbstractEPGSource implements EPGSource  {
 			writer.writeStartElement("tv");
 			for(Channel c: channels) {c.serialize(writer);}
 			writer.flush();
-			List<Programme> programmes = rtl.getProgrammes(channels.subList(6, 9), 0);
-			//List<Programme> programmes = rtl.getProgrammes(channels, 0, true);
-			for(Programme p: programmes) {p.serialize(writer);}
+			//List<Programme> programmes = rtl.getProgrammes(channels.subList(6, 9), 0);
+			for(int day=0; day<10; day++) {
+				List<Programme> programmes = rtl.getProgrammes(channels, day);
+				for(Programme p: programmes) {p.serialize(writer);}
+			}
 			writer.writeEndElement();
 			writer.writeEndDocument();
 			writer.flush();
@@ -362,6 +408,8 @@ public class RTL extends AbstractEPGSource implements EPGSource  {
 				System.out.println("Number of programmes fetched: " + stats.cacheMisses);
 				System.out.println("Number of fetch errors: " + stats.fetchErrors);
 			}
+			rtl.debugWriter.flush();
+			rtl.debugWriter.close();
 			rtl.close();
 		} catch (Exception e) {
 			// TODO Auto-generated catch block