From: Jan-Pascal van Best Date: Sat, 10 Mar 2012 21:25:17 +0000 (+0100) Subject: Start implementing xmltv grabber interface X-Git-Tag: 0.2~15 X-Git-Url: http://www.vanbest.org/gitweb/?a=commitdiff_plain;h=3b37650407de07c3da71e73f4c681bdf5aa4534c;p=tv_grab_nl_java Start implementing xmltv grabber interface --- diff --git a/tv_grab_nl_java/pom.xml b/tv_grab_nl_java/pom.xml index 390fb9d..59dfdab 100644 --- a/tv_grab_nl_java/pom.xml +++ b/tv_grab_nl_java/pom.xml @@ -10,5 +10,15 @@ 2.4 jdk15 + + org.apache.commons + commons-io + 1.3.2 + + + commons-cli + commons-cli + 1.2 + \ No newline at end of file diff --git a/tv_grab_nl_java/src/org/vanbest/xmltv/Channel.java b/tv_grab_nl_java/src/org/vanbest/xmltv/Channel.java index f414491..97b31b6 100644 --- a/tv_grab_nl_java/src/org/vanbest/xmltv/Channel.java +++ b/tv_grab_nl_java/src/org/vanbest/xmltv/Channel.java @@ -12,4 +12,8 @@ public class Channel { public String toString() { return "id: " + id + "; name: " + name + "; shortName: " + shortName; } + + public String getChannelId() { + return id+".tvgids.nl"; + } } diff --git a/tv_grab_nl_java/src/org/vanbest/xmltv/Main.java b/tv_grab_nl_java/src/org/vanbest/xmltv/Main.java index 281d8ba..6c85c0f 100644 --- a/tv_grab_nl_java/src/org/vanbest/xmltv/Main.java +++ b/tv_grab_nl_java/src/org/vanbest/xmltv/Main.java @@ -1,35 +1,93 @@ package org.vanbest.xmltv; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Set; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.Parser; + public class Main { /** * @param args */ - public static void main(String[] args) { - List channels = TvGids.getChannels(); - System.out.println(channels); + + public static void test() { + TvGids gids = new TvGids(); + + List channels = gids.getChannels(); try { - System.out.println(TvGids.programmeUrl(channels, 0)); + List myChannels = channels; // .subList(0, 2); + Set programmes = new HashSet(); + for( Channel c: myChannels ) { + ArrayList cs = new ArrayList(2); + cs.add(c); + Set p = gids.getProgrammes(cs, 0, true); + programmes.addAll( p ); + } - List myChannels = channels.subList(0, 2); - Set programmes = TvGids.getProgrammes(myChannels, 0, true); - - System.out.println( programmes ); - - XmlTvWriter writer = new XmlTvWriter(System.out); + XmlTvWriter writer = new XmlTvWriter(new FileOutputStream("/tmp/tv_grab_nl_java.xml")); writer.writeChannels(myChannels); writer.writePrograms(programmes); - System.out.flush(); + writer.close(); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); + } finally { + try { + gids.close(); + } catch (FileNotFoundException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + } + + public static void main(String[] args) { + Options options = new Options(); + options.addOption("d", "description", false, "Display a description to identify this grabber"); + options.addOption("c", "capablities", false, "Show grabber capabilities"); + options.addOption("q", "quiet", false, "Be quiet"); + options.addOption("o", "output", true, "Set xlmtv output filename"); + options.addOption("d", "days", true, "Number of days to grab"); + options.addOption("s", "offset", true, "Start day for grabbing (0=today)"); + options.addOption("n", "configure", false, "Interactive configuration"); + options.addOption("f", "config-file", true, "Configuration file location"); + options.addOption("h", "cache", true, "Cache file location"); + //options.addOption("p", "preferredmethod", false, "Show preferred method"); + + CommandLine line = null; + try { + line = new GnuParser().parse(options, args); + } catch (ParseException e) { + // TODO Auto-generated catch block + e.printStackTrace(); } + if (line.hasOption("d")) { + System.out.println("tv_grab_nl_java is a parser for Dutch TV listings using the tvgids.nl JSON interface"); + System.exit(0); + } + if (line.hasOption("c")) { + System.out.println("baseline"); + System.out.println("manualconfig"); + System.out.println("cache"); + // System.out.println("preferredmethod"); + System.exit(0); + } } diff --git a/tv_grab_nl_java/src/org/vanbest/xmltv/Programme.java b/tv_grab_nl_java/src/org/vanbest/xmltv/Programme.java index bebc3f0..3420402 100644 --- a/tv_grab_nl_java/src/org/vanbest/xmltv/Programme.java +++ b/tv_grab_nl_java/src/org/vanbest/xmltv/Programme.java @@ -67,12 +67,12 @@ public class Programme { this.datum_end = datum_end; } - public boolean isIs_hightlight() { - return is_hightlight; + public boolean isIs_highlight() { + return is_highlight; } - public void setIs_hightlight(boolean is_hightlight) { - this.is_hightlight = is_hightlight; + public void setIs_highlight(boolean is_highlight) { + this.is_highlight = is_highlight; } public String getHighlight_afbeelding() { @@ -83,6 +83,14 @@ public class Programme { this.highlight_afbeelding = highlight_afbeelding; } + public String getHighlight_content() { + return highlight_content; + } + + public void setHighlight_content(String highlight_content) { + this.highlight_content = highlight_content; + } + String db_id; String titel; String genre; @@ -91,8 +99,9 @@ public class Programme { String artikel_id; Date datum_start; Date datum_end; - boolean is_hightlight; + boolean is_highlight; String highlight_afbeelding; + String highlight_content; ProgrammeDetails details = null; Channel channel = null; diff --git a/tv_grab_nl_java/src/org/vanbest/xmltv/ProgrammeCache.java b/tv_grab_nl_java/src/org/vanbest/xmltv/ProgrammeCache.java new file mode 100644 index 0000000..f30c1be --- /dev/null +++ b/tv_grab_nl_java/src/org/vanbest/xmltv/ProgrammeCache.java @@ -0,0 +1,49 @@ +package org.vanbest.xmltv; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.io.FileUtils; + +public class ProgrammeCache { + + static String cacheDir = "/tmp/tv_grab_nl_java"; + + private File cacheFile = new File(cacheDir); + private Map cache; + + public ProgrammeCache() { + if (cacheFile.canRead()) { + try { + cache = (Map) new ObjectInputStream( new FileInputStream( cacheFile ) ).readObject(); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + cache = new HashMap(); + } + } else { + cache = new HashMap(); + } + // FileUtils.forceMkdir(root); + } + + public ProgrammeDetails getDetails(String id) { + return cache.get(id); + } + + public void add(String id, ProgrammeDetails d) { + cache.put(id, d); + } + + public void close() throws FileNotFoundException, IOException { + new ObjectOutputStream( new FileOutputStream(cacheFile)).writeObject(cache); + } +} diff --git a/tv_grab_nl_java/src/org/vanbest/xmltv/ProgrammeDetails.java b/tv_grab_nl_java/src/org/vanbest/xmltv/ProgrammeDetails.java index 7dc2783..b4138a9 100644 --- a/tv_grab_nl_java/src/org/vanbest/xmltv/ProgrammeDetails.java +++ b/tv_grab_nl_java/src/org/vanbest/xmltv/ProgrammeDetails.java @@ -1,6 +1,8 @@ package org.vanbest.xmltv; -public class ProgrammeDetails { +import java.io.Serializable; + +public class ProgrammeDetails implements Serializable { String db_id; String titel; String datum; diff --git a/tv_grab_nl_java/src/org/vanbest/xmltv/TvGids.java b/tv_grab_nl_java/src/org/vanbest/xmltv/TvGids.java index 9aca65c..89db0d5 100644 --- a/tv_grab_nl_java/src/org/vanbest/xmltv/TvGids.java +++ b/tv_grab_nl_java/src/org/vanbest/xmltv/TvGids.java @@ -1,6 +1,7 @@ package org.vanbest.xmltv; import java.io.BufferedReader; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStreamReader; import java.net.MalformedURLException; @@ -15,6 +16,7 @@ import java.util.Map; import java.util.Set; import net.sf.ezmorph.object.DateMorpher; +import net.sf.json.JSON; import net.sf.json.JSONArray; import net.sf.json.JSONObject; import net.sf.json.util.JSONUtils; @@ -25,6 +27,22 @@ public class TvGids { static String programme_base_url="http://www.tvgids.nl/json/lists/programs.php"; static String detail_base_url = "http://www.tvgids.nl/json/lists/program.php"; + ProgrammeCache cache; + boolean initialised = false; + + public TvGids() { + cache = new ProgrammeCache(); + if ( ! initialised ) { + String[] formats = {"yyyy-MM-dd HH:mm:ss"}; + JSONUtils.getMorpherRegistry().registerMorpher( new DateMorpher(formats, new Locale("nl"))); + initialised = true; + } + } + + public void close() throws FileNotFoundException, IOException { + cache.close(); + } + static public List getChannels() { List result = new ArrayList(10); URL url = null; @@ -34,22 +52,18 @@ public class TvGids { // TODO Auto-generated catch block e.printStackTrace(); } - /* + StringBuffer json = new StringBuffer(); try { BufferedReader reader = new BufferedReader( new InputStreamReader( url.openStream())); String s; - while ((s = reader.readLine()) != null) { - json.append(s); - } + while ((s = reader.readLine()) != null) json.append(s); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } - */ - String json = "[{'id':'1','name':'Nederland 1','name_short':'Ned 1'},{'id':'2','name':'Nederland 2','name_short':'Ned 2'},{'id':'3','name':'Nederland 3','name_short':'Ned 3'},{'id':'4','name':'RTL 4','name_short':'RTL 4'},{'id':'31','name':'RTL 5','name_short':'RTL 5'},{'id':'46','name':'RTL 7','name_short':'RTL 7'},{'id':'92','name':'RTL 8','name_short':'RTL 8'},{'id':'36','name':'SBS 6','name_short':'SBS 6'},{'id':'37','name':'NET 5','name_short':'NET 5'},{'id':'34','name':'Veronica','name_short':'Veronica'},{'id':'29','name':'Discovery Channel','name_short':'Discovery'},{'id':'18','name':'National Geographic','name_short':'NGC'},{'id':'84','name':'Het Gesprek','name_short':'HetGesprek'},{'id':'406','name':'NostalgieNet','name_short':'Nost Net'},{'id':'5','name':'Eén','name_short':'Eén'},{'id':'6','name':'KETNET/Canvas','name_short':'KET/Can'},{'id':'25','name':'MTV','name_short':'MTV'},{'id':'405','name':'TLC','name_short':'tlc'},{'id':'91','name':'Comedy Central','name_short':'Com. Centr.'},{'id':'49','name':'VTM','name_short':'VTM'},{'id':'59','name':'2BE','name_short':'2BE'},{'id':'89','name':'Nickelodeon','name_short':'Nick'},{'id':'60','name':'VT4','name_short':'VT4'},{'id':'90','name':'BVN','name_short':'BVN'},{'id':'404','name':'FOXlife','name_short':'FOXlife'},{'id':'7','name':'BBC 1','name_short':'BBC 1'},{'id':'8','name':'BBC 2','name_short':'BBC 2'},{'id':'86','name':'BBC World','name_short':'BBC W'},{'id':'9','name':'ARD','name_short':'ARD'},{'id':'10','name':'ZDF','name_short':'ZDF'},{'id':'13','name':'NDR Fernsehen','name_short':'NDR'},{'id':'14','name':'Südwest Fernsehen','name_short':'SWF'},{'id':'12','name':'WDR Fernsehen','name_short':'WDR'},{'id':'50','name':'3Sat','name_short':'3Sat'},{'id':'28','name':'Sat 1','name_short':'Sat 1'},{'id':'11','name':'RTL','name_short':'RTL'},{'id':'58','name':'PRO 7','name_short':'PRO 7'},{'id':'15','name':'RTBF La 1','name_short':'RTBF La1'},{'id':'16','name':'RTBF La 2','name_short':'RTBF La2'},{'id':'17','name':'TV 5','name_short':'TV 5'},{'id':'27','name':'Rai Uno','name_short':'Rai Uno'},{'id':'32','name':'TRT int.','name_short':'TRT'},{'id':'40','name':'AT 5','name_short':'AT 5'},{'id':'24','name':'Film 1 Premiere','name_short':'Film1 Prem.'},{'id':'39','name':'Film 1 Family','name_short':'Film1 Fam.'},{'id':'107','name':'Film 1 Festival','name_short':'Film1 Fest.'},{'id':'35','name':'TMF','name_short':'TMF'},{'id':'73','name':'Mezzo','name_short':'Mezzo'},{'id':'21','name':'Cartoon Network','name_short':'Cart. Net.'},{'id':'26','name':'CNN','name_short':'CNN'},{'id':'19','name':'Eurosport','name_short':'Eurosport'},{'id':'99','name':'Sport1','name_short':'Sport1'},{'id':'20','name':'TCM','name_short':'TCM'},{'id':'65','name':'Animal Planet','name_short':'Animal Pl.'},{'id':'87','name':'TV E','name_short':'TVE'},{'id':'38','name':'ARTE','name_short':'ARTE'},{'id':'103','name':'RTV Noord-Holland','name_short':'RTV N-H'},{'id':'100','name':'RTV Utrecht','name_short':'Utrecht'},{'id':'101','name':'RTV West','name_short':'RTV West'},{'id':'102','name':'RTV Rijnmond','name_short':'Rijnmond'},{'id':'104','name':'BBC Entertainment','name_short':'BBC E'},{'id':'105','name':'Private Spice','name_short':'Private Sp.'},{'id':'93','name':'13TH STREET','name_short':'13TH ST'},{'id':'94','name':'Syfy','name_short':'Syfy'},{'id':'109','name':'Omrop Fryslân','name_short':'Fryslân'},{'id':'112','name':'Omroep Gelderland','name_short':'Gelderland'},{'id':'115','name':'L1 TV','name_short':'L1 TV'},{'id':'110','name':'RTV Drenthe','name_short':'Drenthe'},{'id':'113','name':'Omroep Flevoland','name_short':'Flevoland'},{'id':'116','name':'Omroep Zeeland','name_short':'Zeeland'},{'id':'108','name':'RTV Noord','name_short':'RTV Noord'},{'id':'111','name':'RTV Oost','name_short':'RTV Oost'},{'id':'114','name':'Omroep Brabant','name_short':'Brabant'},{'id':'67','name':'Consumenten 24','name_short':'Cons24'},{'id':'81','name':'HollandDoc 24','name_short':'HolDoc24'},{'id':'314','name':'Journaal 24','name_short':'Journaal24'},{'id':'316','name':'Best 24','name_short':'Best24'},{'id':'401','name':'Playboy TV','name_short':'Playboy'},{'id':'306','name':'Discovery Science','name_short':'Disc. Sc.'},{'id':'403','name':'Goed TV','name_short':'Goed TV'},{'id':'303','name':'Hallmark','name_short':'Hallmark'},{'id':'300','name':'BBC 3','name_short':'BBC 3'},{'id':'310','name':'3voor12 Portal','name_short':''},{'id':'64','name':'Familie 24','name_short':'Fam24'},{'id':'69','name':'Sterren 24','name_short':'Sterren24'},{'id':'82','name':'Geschiedenis 24','name_short':'Gesch24'},{'id':'312','name':'Nick Jr.','name_short':'Nick Jr.'},{'id':'311','name':'Disney XD','name_short':'Disney XD'},{'id':'317','name':'Comedy Family','name_short':'Com. Fam.'},{'id':'402','name':'Adult Channel','name_short':'Adult'},{'id':'305','name':'Discovery World','name_short':'Disc. World'},{'id':'308','name':'3voor12 Central','name_short':''},{'id':'148','name':'Eredivisie Live','name_short':'Eredivisie'},{'id':'66','name':'HumorTV 24','name_short':'Humor24'},{'id':'70','name':'Cultura 24','name_short':'Cult24'},{'id':'83','name':'3voor12','name_short':''},{'id':'313','name':'Boomerang','name_short':'Boomerang'},{'id':'315','name':'Zone Reality','name_short':'ZoneReality'},{'id':'400','name':'Hustler TV','name_short':'Hustler'},{'id':'307','name':'Discovery Travel & Living','name_short':'Disc T&L'},{'id':'304','name':'MGM','name_short':'MGM'},{'id':'301','name':'BBC 4','name_short':'BBC 4'},{'id':'309','name':'3voor12 On Stage','name_short':''},{'id':'33','name':'Spirit 24','name_short':'Spirit24'}]"; JSONArray jsonArray = JSONArray.fromObject( json.toString() ); // System.out.println( jsonArray ); @@ -92,37 +106,46 @@ public class TvGids { s.append(id); return new URL(s.toString()); } - - static public Set getProgrammes(List channels, int day, boolean fetchDetails) throws Exception { + public Set getProgrammes(List channels, int day, boolean fetchDetails) throws Exception { Set result = new HashSet(); URL url = programmeUrl(channels, day); JSONObject jsonObject = fetchJSON(url); System.out.println( jsonObject ); - String[] formats = {"yyyy-MM-dd HH:mm:ss"}; - JSONUtils.getMorpherRegistry().registerMorpher( new DateMorpher(formats, new Locale("nl"))); - - for( Channel i: channels) { - JSONObject programs = jsonObject.getJSONObject(""+i.id); - System.out.println( programs ); - - for( Object o: programs.keySet() ) { - JSONObject programme = programs.getJSONObject(o.toString()); - Programme p = (Programme) JSONObject.toBean(programme, Programme.class); - if (fetchDetails) { - p.details = getDetails(p.db_id); + for( Channel c: channels) { + JSON ps = (JSON) jsonObject.get(""+c.id); + System.out.println( ps ); + if ( ps.isArray() ) { + JSONArray programs = (JSONArray) ps; + for( int i=0; i result = new HashSet(); + private ProgrammeDetails getDetails(String db_id) throws Exception { + ProgrammeDetails d = cache.getDetails(db_id); + if ( d != null ) { + return d; + } URL url = detailUrl(db_id); JSONObject json = fetchJSON(url); //System.out.println( json ); - ProgrammeDetails d = (ProgrammeDetails) JSONObject.toBean(json, ProgrammeDetails.class); + d = (ProgrammeDetails) JSONObject.toBean(json, ProgrammeDetails.class); + cache.add(db_id, d); return d; } } diff --git a/tv_grab_nl_java/src/org/vanbest/xmltv/XmlTvWriter.java b/tv_grab_nl_java/src/org/vanbest/xmltv/XmlTvWriter.java index 80e9f35..879e92f 100644 --- a/tv_grab_nl_java/src/org/vanbest/xmltv/XmlTvWriter.java +++ b/tv_grab_nl_java/src/org/vanbest/xmltv/XmlTvWriter.java @@ -49,7 +49,7 @@ public class XmlTvWriter { public void writeChannels(List channels) throws XMLStreamException { for(Channel c: channels) { writer.writeStartElement("channel"); - writer.writeAttribute("id", ""+c.id); + writer.writeAttribute("id", c.getChannelId()); writer.writeStartElement("display-name"); writer.writeAttribute("lang", "nl"); writer.writeCharacters(c.name); @@ -59,46 +59,34 @@ public class XmlTvWriter { } } - /* - * - * - -Mozart - Così fan tutte - - -Opera, opgenomen in 2006 in het Amsterdamse Muziektheater als onderdeel van de -eigentijdse enscenering van de drie Da Ponte opera's met o.a. Sally Matthews en -Maite Beaumont. - - -Arts/Culture - - - - */ - public void writePrograms(Collection programs) throws XMLStreamException { DateFormat df = new SimpleDateFormat("yyyyMMddHHmmss Z"); for(Programme p: programs) { writer.writeStartElement("programme"); writer.writeAttribute("start", df.format(p.datum_start)); writer.writeAttribute("stop", df.format(p.datum_end)); - writer.writeAttribute("channel", ""+p.channel.id); + writer.writeAttribute("channel", ""+p.channel.getChannelId()); + writer.writeCharacters("\n"); writer.writeStartElement("title"); writer.writeAttribute("lang", "nl"); writer.writeCharacters(p.titel); writer.writeEndElement(); + writer.writeCharacters("\n"); - writer.writeStartElement("desc"); - writer.writeAttribute("lang", "nl"); - writer.writeCharacters(p.details.synop); - writer.writeEndElement(); + if(p.details.synop != null && ! p.details.synop.isEmpty()) { + writer.writeStartElement("desc"); + writer.writeAttribute("lang", "nl"); + writer.writeCharacters(p.details.synop); + writer.writeEndElement(); + writer.writeCharacters("\n"); + } writer.writeStartElement("category"); writer.writeAttribute("lang", "en"); writer.writeCharacters(p.genre); // soort? FIXME translation to mythtv categories writer.writeEndElement(); + writer.writeCharacters("\n"); writer.writeEndElement(); writer.writeCharacters("\n");