Introduce --log-level command line option;

Also fetch programme details from /programma/<ID>/, halfway done,
recognise 'teletekst' but no output to xmltv format yet. To be done:
breedbeeld, HD, any other flags?

diff --git a/tv_grab_nl_java/src/org/vanbest/xmltv/Config.java b/tv_grab_nl_java/src/org/vanbest/xmltv/Config.java
index f62f94492d9c42dedd3544570a768fb70779c52b..f112293a5fae57138c5bac2dcdb1b94f3b9fa041 100644 (file)
--- a/tv_grab_nl_java/src/org/vanbest/xmltv/Config.java
+++ b/tv_grab_nl_java/src/org/vanbest/xmltv/Config.java
@@ -23,6 +23,13 @@ public class Config {
        public Map<String, String> cattrans;
        protected File cacheFile;
        boolean quiet = false;
+       public int logLevel = LOG_DEFAULT;
+       
+       public static final int LOG_INFO = 0x0001;
+       public static final int LOG_JSON = 0x0100;
+       private static final int LOG_PROGRAMME_INFO = 0x0200;
+       
+       public static int LOG_DEFAULT = LOG_INFO;
        
        private Config() {
        }
@@ -170,6 +177,14 @@ public class Config {
                }
                return result;
        }
+       
+       public boolean logJSON() {
+               return (logLevel & LOG_JSON) != 0;
+       }
+
+       public boolean logProgrammes() {
+               return (logLevel & LOG_PROGRAMME_INFO) != 0;
+       }
 
 }
 

diff --git a/tv_grab_nl_java/src/org/vanbest/xmltv/Main.java b/tv_grab_nl_java/src/org/vanbest/xmltv/Main.java
index 6ec1b1ab0d2a669eb5825dac9daf5def1504c45e..6c86d09ae2976ea4bc32061595e834899bc3ff21 100644 (file)
--- a/tv_grab_nl_java/src/org/vanbest/xmltv/Main.java
+++ b/tv_grab_nl_java/src/org/vanbest/xmltv/Main.java
@@ -148,6 +148,7 @@ public class Main {
                options.addOption("f", "config-file", true, "Configuration file location");
                options.addOption("h", "cache", true, "Cache file location");
                options.addOption("e", "help", false, "Show this help");
+               options.addOption("l", "log-level", true, "Set log level (0x0100=JSON)");
                //options.addOption("p", "preferredmethod", false, "Show preferred method");
 
                CommandLine line = null;
@@ -179,6 +180,9 @@ public class Main {
                if (line.hasOption("o")) {
                        this.outputWriter = new PrintStream( new FileOutputStream(line.getOptionValue("o")));
                }
+               if (line.hasOption("o")) {
+                       config.logLevel = Integer.parseInt(line.getOptionValue("l"));
+               }
                if (line.hasOption("h")) {
                        config.cacheFile = new File(line.getOptionValue("h"));
                }

diff --git a/tv_grab_nl_java/src/org/vanbest/xmltv/ProgrammeDetails.java b/tv_grab_nl_java/src/org/vanbest/xmltv/ProgrammeDetails.java
index 69c7484ca1be7ffa820daa6aa1db9510be8d2940..bbc5728d6fc859ca2e102e2d1fb0518f0acacd99 100644 (file)
--- a/tv_grab_nl_java/src/org/vanbest/xmltv/ProgrammeDetails.java
+++ b/tv_grab_nl_java/src/org/vanbest/xmltv/ProgrammeDetails.java
@@ -15,6 +15,7 @@ public class ProgrammeDetails implements Serializable {
        String acteursnamen_rolverdeling;
        String regisseur;
        String zender_id;
+       public boolean teletekst = false;
        
        public void fixup(Programme p, boolean quiet) {
                this.titel = org.apache.commons.lang.StringEscapeUtils.unescapeHtml(titel);

diff --git a/tv_grab_nl_java/src/org/vanbest/xmltv/TvGids.java b/tv_grab_nl_java/src/org/vanbest/xmltv/TvGids.java
index 5612b7ea4a80fc7d11abdf31ec5134c24a93fdfe..bb53ea3286f166e8c5e827490b901a21088df1d2 100644 (file)
--- a/tv_grab_nl_java/src/org/vanbest/xmltv/TvGids.java
+++ b/tv_grab_nl_java/src/org/vanbest/xmltv/TvGids.java
@@ -15,6 +15,8 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import net.sf.ezmorph.MorpherRegistry;
 import net.sf.ezmorph.ObjectMorpher;
@@ -29,6 +31,7 @@ public class TvGids {
        static String channels_url="http://www.tvgids.nl/json/lists/channels.php";
        static String programme_base_url="http://www.tvgids.nl/json/lists/programs.php";
        static String detail_base_url = "http://www.tvgids.nl/json/lists/program.php";
+       static String html_detail_base_url = "http://www.tvgids.nl/programma/";
 
        Config config;
        ProgrammeCache cache;
@@ -68,7 +71,7 @@ public class TvGids {
                cache.close();
        }
 
-       static public List<Channel> getChannels() {
+       public List<Channel> getChannels() {
                List<Channel> result = new ArrayList<Channel>(10);
                URL url = null;
                try {
@@ -90,6 +93,7 @@ public class TvGids {
                        e.printStackTrace();
                }
 
+               if (config.logJSON()) System.out.println(json.toString());
                JSONArray jsonArray = JSONArray.fromObject( json.toString() );  
                // System.out.println( jsonArray );  
                
@@ -128,13 +132,20 @@ public class TvGids {
                return new URL(s.toString());
        }
        
-       public static URL detailUrl(String id) throws Exception {
+       public static URL JSONDetailUrl(String id) throws Exception {
                StringBuilder s = new StringBuilder(detail_base_url);
                s.append("?id=");
                s.append(id);
                return new URL(s.toString());
        }
                
+       public static URL HTMLDetailUrl(String id) throws Exception {
+               StringBuilder s = new StringBuilder(html_detail_base_url);
+               s.append(id);
+               s.append("/");
+               return new URL(s.toString());
+       }
+               
        public Set<Programme> getProgrammes(List<Channel> channels, int day, boolean fetchDetails) throws Exception {
                Set<Programme> result = new HashSet<Programme>();
                URL url = programmeUrl(channels, day);
@@ -144,16 +155,11 @@ public class TvGids {
                
                for( Channel c: channels) {
                        JSON ps = (JSON) jsonObject.get(""+c.id);
-                       //System.out.println( ps );
                        if ( ps.isArray() ) {
                                JSONArray programs = (JSONArray) ps;
                                for( int i=0; i<programs.size(); i++ ) {
                                        JSONObject programme = programs.getJSONObject(i);
-                                       Programme p = (Programme) JSONObject.toBean(programme, Programme.class);
-                                       p.fixup(config);
-                                       if (fetchDetails) {
-                                               fillDetails(p);
-                                       }
+                                       Programme p = programmeFromJSON(programme, fetchDetails);
                                        p.channel = c;
                                        result.add( p );
                                }
@@ -161,11 +167,7 @@ public class TvGids {
                                JSONObject programs = (JSONObject) ps;
                                for( Object o: programs.keySet() ) {
                                        JSONObject programme = programs.getJSONObject(o.toString());
-                                       Programme p = (Programme) JSONObject.toBean(programme, Programme.class);
-                                       p.fixup(config);
-                                       if (fetchDetails) {
-                                               fillDetails(p);
-                                       }
+                                       Programme p = programmeFromJSON(programme, fetchDetails);
                                        p.channel = c;
                                        result.add( p );
                                }
@@ -175,27 +177,76 @@ public class TvGids {
                return result;
        }
        
-       private JSONObject fetchJSON(URL url) throws Exception {
+       private Programme programmeFromJSON(JSONObject programme, boolean fetchDetails) throws Exception {
+               Programme p = (Programme) JSONObject.toBean(programme, Programme.class);
+               p.fixup(config);
+               if (fetchDetails) {
+                       fillDetails(p);
+               }
+               if(config.logProgrammes()) {
+                       System.out.println(p.toString());
+               }
+               return p;
+       }
+
+       private String fetchURL(URL url) throws Exception {
                Thread.sleep(config.niceMilliseconds);
-               StringBuffer json = new StringBuffer();
+               StringBuffer buf = new StringBuffer();
                try {
                        BufferedReader reader = new BufferedReader( new InputStreamReader( url.openStream()));
                        String s;
-                       while ((s = reader.readLine()) != null) json.append(s);
+                       while ((s = reader.readLine()) != null) buf.append(s);
                } catch (IOException e) {
                        fetchErrors++;
                        throw new Exception("Error getting program data from url " + url, e);
                }
-               return JSONObject.fromObject( json.toString() );  
+               return buf.toString();  
+       }
+
+       private JSONObject fetchJSON(URL url) throws Exception {
+               String json = fetchURL(url);
+               if (config.logJSON()) System.out.println(json);
+               return JSONObject.fromObject( json );  
        }
 
        private void fillDetails(Programme p) throws Exception {
                p.details = cache.getDetails(p.db_id);
                if ( p.details == null ) {
                        cacheMisses++;
-                       URL url = detailUrl(p.db_id);
+                       
+                       URL url = JSONDetailUrl(p.db_id);
                        JSONObject json = fetchJSON(url);
                        p.details = (ProgrammeDetails) JSONObject.toBean(json, ProgrammeDetails.class);
+                       
+                       url = HTMLDetailUrl(p.db_id);
+                       String clob=fetchURL(url);
+                       //System.out.println("clob:");
+                       //System.out.println(clob);
+                       Pattern progInfoPattern = Pattern.compile("prog-info-content.*prog-info-footer", Pattern.DOTALL);
+                       Matcher m = progInfoPattern.matcher(clob);
+                       if (m.find()) {
+                               String progInfo = m.group();
+                               //System.out.println("progInfo");
+                               //System.out.println(progInfo);
+                               Pattern infoLinePattern = Pattern.compile("<li><strong>(.*?):</strong>(.*?)</li>");
+                               Matcher m2 = infoLinePattern.matcher(progInfo);
+                               while (m2.find()) {
+                                       //System.out.println("    infoLine: " + m2.group());
+                                       //System.out.println("         key: " + m2.group(1));
+                                       //System.out.println("       value: " + m2.group(2));
+                                       String key = m2.group(1);
+                                       String value = m2.group(2);
+                                       switch(key.toLowerCase()) {
+                                       case "bijzonderheden":
+                                               if (value.toLowerCase().contains("teletekst")) {
+                                                       p.details.teletekst = true;
+                                               }
+                                               break;
+                                       }
+                                       
+                               }
+                       }
+                       
                        p.details.fixup(p, config.quiet);
                        cache.add(p.db_id, p.details);
                } else {