From: JP <jp@here>
Date: Mon, 19 Mar 2012 14:44:01 +0000 (+0100)
Subject: Introduce --log-level command line option;
X-Git-Tag: 0.5~10
X-Git-Url: http://www.vanbest.org/gitweb/?a=commitdiff_plain;h=38305e538576f225e3acbca218e3eafda6fd0bfc;p=tv_grab_nl_java

Introduce --log-level command line option;
Also fetch programme details from /programma/<ID>/, halfway done,
recognise 'teletekst' but no output to xmltv format yet. To be done:
breedbeeld, HD, any other flags?
---

diff --git a/tv_grab_nl_java/src/org/vanbest/xmltv/Config.java b/tv_grab_nl_java/src/org/vanbest/xmltv/Config.java
index f62f944..f112293 100644
--- a/tv_grab_nl_java/src/org/vanbest/xmltv/Config.java
+++ b/tv_grab_nl_java/src/org/vanbest/xmltv/Config.java
@@ -23,6 +23,13 @@ public class Config {
 	public Map<String, String> cattrans;
 	protected File cacheFile;
 	boolean quiet = false;
+	public int logLevel = LOG_DEFAULT;
+	
+	public static final int LOG_INFO = 0x0001;
+	public static final int LOG_JSON = 0x0100;
+	private static final int LOG_PROGRAMME_INFO = 0x0200;
+	
+	public static int LOG_DEFAULT = LOG_INFO;
 	
 	private Config() {
 	}
@@ -170,6 +177,14 @@ public class Config {
 		}
 		return result;
 	}
+	
+	public boolean logJSON() {
+		return (logLevel & LOG_JSON) != 0;
+	}
+
+	public boolean logProgrammes() {
+		return (logLevel & LOG_PROGRAMME_INFO) != 0;
+	}
 
 }
 
diff --git a/tv_grab_nl_java/src/org/vanbest/xmltv/Main.java b/tv_grab_nl_java/src/org/vanbest/xmltv/Main.java
index 6ec1b1a..6c86d09 100644
--- a/tv_grab_nl_java/src/org/vanbest/xmltv/Main.java
+++ b/tv_grab_nl_java/src/org/vanbest/xmltv/Main.java
@@ -148,6 +148,7 @@ public class Main {
 		options.addOption("f", "config-file", true, "Configuration file location");
 		options.addOption("h", "cache", true, "Cache file location");
 		options.addOption("e", "help", false, "Show this help");
+		options.addOption("l", "log-level", true, "Set log level (0x0100=JSON)");
 		//options.addOption("p", "preferredmethod", false, "Show preferred method");
 
 		CommandLine line = null;
@@ -179,6 +180,9 @@ public class Main {
 		if (line.hasOption("o")) {
 			this.outputWriter = new PrintStream( new FileOutputStream(line.getOptionValue("o")));
 		}
+		if (line.hasOption("o")) {
+			config.logLevel = Integer.parseInt(line.getOptionValue("l"));
+		}
 		if (line.hasOption("h")) {
 			config.cacheFile = new File(line.getOptionValue("h"));
 		}
diff --git a/tv_grab_nl_java/src/org/vanbest/xmltv/ProgrammeDetails.java b/tv_grab_nl_java/src/org/vanbest/xmltv/ProgrammeDetails.java
index 69c7484..bbc5728 100644
--- a/tv_grab_nl_java/src/org/vanbest/xmltv/ProgrammeDetails.java
+++ b/tv_grab_nl_java/src/org/vanbest/xmltv/ProgrammeDetails.java
@@ -15,6 +15,7 @@ public class ProgrammeDetails implements Serializable {
 	String acteursnamen_rolverdeling;
 	String regisseur;
 	String zender_id;
+	public boolean teletekst = false;
 	
 	public void fixup(Programme p, boolean quiet) {
 		this.titel = org.apache.commons.lang.StringEscapeUtils.unescapeHtml(titel);
diff --git a/tv_grab_nl_java/src/org/vanbest/xmltv/TvGids.java b/tv_grab_nl_java/src/org/vanbest/xmltv/TvGids.java
index 5612b7e..bb53ea3 100644
--- a/tv_grab_nl_java/src/org/vanbest/xmltv/TvGids.java
+++ b/tv_grab_nl_java/src/org/vanbest/xmltv/TvGids.java
@@ -15,6 +15,8 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import net.sf.ezmorph.MorpherRegistry;
 import net.sf.ezmorph.ObjectMorpher;
@@ -29,6 +31,7 @@ public class TvGids {
 	static String channels_url="http://www.tvgids.nl/json/lists/channels.php";
 	static String programme_base_url="http://www.tvgids.nl/json/lists/programs.php";
 	static String detail_base_url = "http://www.tvgids.nl/json/lists/program.php";
+	static String html_detail_base_url = "http://www.tvgids.nl/programma/";
 
 	Config config;
 	ProgrammeCache cache;
@@ -68,7 +71,7 @@ public class TvGids {
 		cache.close();
 	}
 
-	static public List<Channel> getChannels() {
+	public List<Channel> getChannels() {
 		List<Channel> result = new ArrayList<Channel>(10);
 		URL url = null;
 		try {
@@ -90,6 +93,7 @@ public class TvGids {
 			e.printStackTrace();
 		}
 
+		if (config.logJSON()) System.out.println(json.toString());
 		JSONArray jsonArray = JSONArray.fromObject( json.toString() );  
 		// System.out.println( jsonArray );  
 		
@@ -128,13 +132,20 @@ public class TvGids {
 		return new URL(s.toString());
 	}
 	
-	public static URL detailUrl(String id) throws Exception {
+	public static URL JSONDetailUrl(String id) throws Exception {
 		StringBuilder s = new StringBuilder(detail_base_url);
 		s.append("?id=");
 		s.append(id);
 		return new URL(s.toString());
 	}
 		
+	public static URL HTMLDetailUrl(String id) throws Exception {
+		StringBuilder s = new StringBuilder(html_detail_base_url);
+		s.append(id);
+		s.append("/");
+		return new URL(s.toString());
+	}
+		
 	public Set<Programme> getProgrammes(List<Channel> channels, int day, boolean fetchDetails) throws Exception {
 		Set<Programme> result = new HashSet<Programme>();
 		URL url = programmeUrl(channels, day);
@@ -144,16 +155,11 @@ public class TvGids {
 		
 		for( Channel c: channels) {
 			JSON ps = (JSON) jsonObject.get(""+c.id);
-			//System.out.println( ps );
 			if ( ps.isArray() ) {
 				JSONArray programs = (JSONArray) ps;
 				for( int i=0; i<programs.size(); i++ ) {
 					JSONObject programme = programs.getJSONObject(i);
-					Programme p = (Programme) JSONObject.toBean(programme, Programme.class);
-					p.fixup(config);
-					if (fetchDetails) {
-						fillDetails(p);
-					}
+					Programme p = programmeFromJSON(programme, fetchDetails);
 					p.channel = c;
 					result.add( p );
 				}
@@ -161,11 +167,7 @@ public class TvGids {
 				JSONObject programs = (JSONObject) ps;
 				for( Object o: programs.keySet() ) {
 					JSONObject programme = programs.getJSONObject(o.toString());
-					Programme p = (Programme) JSONObject.toBean(programme, Programme.class);
-					p.fixup(config);
-					if (fetchDetails) {
-						fillDetails(p);
-					}
+					Programme p = programmeFromJSON(programme, fetchDetails);
 					p.channel = c;
 					result.add( p );
 				}
@@ -175,27 +177,76 @@ public class TvGids {
 		return result;
 	}
 	
-	private JSONObject fetchJSON(URL url) throws Exception {
+	private Programme programmeFromJSON(JSONObject programme, boolean fetchDetails) throws Exception {
+		Programme p = (Programme) JSONObject.toBean(programme, Programme.class);
+		p.fixup(config);
+		if (fetchDetails) {
+			fillDetails(p);
+		}
+		if(config.logProgrammes()) {
+			System.out.println(p.toString());
+		}
+		return p;
+	}
+
+	private String fetchURL(URL url) throws Exception {
 		Thread.sleep(config.niceMilliseconds);
-		StringBuffer json = new StringBuffer();
+		StringBuffer buf = new StringBuffer();
 		try {
 			BufferedReader reader = new BufferedReader( new InputStreamReader( url.openStream()));
 			String s;
-			while ((s = reader.readLine()) != null) json.append(s);
+			while ((s = reader.readLine()) != null) buf.append(s);
 		} catch (IOException e) {
 			fetchErrors++;
 			throw new Exception("Error getting program data from url " + url, e);
 		}
-		return JSONObject.fromObject( json.toString() );  
+		return buf.toString();  
+	}
+
+	private JSONObject fetchJSON(URL url) throws Exception {
+		String json = fetchURL(url);
+		if (config.logJSON()) System.out.println(json);
+		return JSONObject.fromObject( json );  
 	}
 
 	private void fillDetails(Programme p) throws Exception {
 		p.details = cache.getDetails(p.db_id);
 		if ( p.details == null ) {
 			cacheMisses++;
-			URL url = detailUrl(p.db_id);
+			
+			URL url = JSONDetailUrl(p.db_id);
 			JSONObject json = fetchJSON(url);
 			p.details = (ProgrammeDetails) JSONObject.toBean(json, ProgrammeDetails.class);
+			
+			url = HTMLDetailUrl(p.db_id);
+			String clob=fetchURL(url);
+			//System.out.println("clob:");
+			//System.out.println(clob);
+			Pattern progInfoPattern = Pattern.compile("prog-info-content.*prog-info-footer", Pattern.DOTALL);
+			Matcher m = progInfoPattern.matcher(clob);
+			if (m.find()) {
+				String progInfo = m.group();
+				//System.out.println("progInfo");
+				//System.out.println(progInfo);
+				Pattern infoLinePattern = Pattern.compile("<li><strong>(.*?):</strong>(.*?)</li>");
+				Matcher m2 = infoLinePattern.matcher(progInfo);
+				while (m2.find()) {
+					//System.out.println("    infoLine: " + m2.group());
+					//System.out.println("         key: " + m2.group(1));
+					//System.out.println("       value: " + m2.group(2));
+					String key = m2.group(1);
+					String value = m2.group(2);
+					switch(key.toLowerCase()) {
+					case "bijzonderheden":
+						if (value.toLowerCase().contains("teletekst")) {
+							p.details.teletekst = true;
+						}
+						break;
+					}
+					
+				}
+			}
+			
 			p.details.fixup(p, config.quiet);
 			cache.add(p.db_id, p.details);
 		} else {