From bae38118085b1fc34d7aa38ed73c32881a2864a1 Mon Sep 17 00:00:00 2001 From: Jan-Pascal van Best Date: Tue, 3 Apr 2012 22:16:26 +0200 Subject: [PATCH] Experimental feature, make it possible to configure RTL channels and fetch them from rtl.nl. --- .../org/vanbest/xmltv/AbstractEPGSource.java | 4 +- src/main/java/org/vanbest/xmltv/Channel.java | 33 +-------- src/main/java/org/vanbest/xmltv/Config.java | 7 +- .../java/org/vanbest/xmltv/EPGSource.java | 19 ++--- .../org/vanbest/xmltv/EPGSourceFactory.java | 56 ++++++++++++++ src/main/java/org/vanbest/xmltv/Main.java | 73 +++++++++++++++---- .../org/vanbest/xmltv/ProgrammeCache.java | 30 ++++---- src/main/java/org/vanbest/xmltv/RTL.java | 44 ++++++++--- src/main/java/org/vanbest/xmltv/TvGids.java | 22 ++++-- 9 files changed, 193 insertions(+), 95 deletions(-) create mode 100644 src/main/java/org/vanbest/xmltv/EPGSourceFactory.java diff --git a/src/main/java/org/vanbest/xmltv/AbstractEPGSource.java b/src/main/java/org/vanbest/xmltv/AbstractEPGSource.java index 424ed0b..6dcf076 100644 --- a/src/main/java/org/vanbest/xmltv/AbstractEPGSource.java +++ b/src/main/java/org/vanbest/xmltv/AbstractEPGSource.java @@ -24,11 +24,11 @@ public abstract class AbstractEPGSource implements EPGSource { cache = new ProgrammeCache(config); } - public List getProgrammes(Channel channel, int day, boolean fetchDetails) + public List getProgrammes(Channel channel, int day) throws Exception { ArrayList list = new ArrayList(2); list.add(channel); - return getProgrammes(list, day, fetchDetails); + return getProgrammes(list, day); } @Override diff --git a/src/main/java/org/vanbest/xmltv/Channel.java b/src/main/java/org/vanbest/xmltv/Channel.java index 650ae5e..e1725b8 100644 --- a/src/main/java/org/vanbest/xmltv/Channel.java +++ b/src/main/java/org/vanbest/xmltv/Channel.java @@ -1,10 +1,8 @@ package org.vanbest.xmltv; import java.util.ArrayList; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import javax.xml.stream.XMLStreamException; @@ -18,12 +16,6 @@ public class Channel { protected boolean enabled = true; int source; - public final static int CHANNEL_SOURCE_TVGIDS=1; - public final static int CHANNEL_SOURCE_RTL=2; - private final static String[] CHANNEL_SOURCE_NAMES={"tvgids.nl", "rtl.nl"}; - private static Map channelSourceNameMap = new HashMap(); - - protected Channel(int source, String id) { this.id = id; this.source = source; @@ -50,34 +42,13 @@ public class Channel { } public String getXmltvChannelId() { - switch (source) { - case CHANNEL_SOURCE_TVGIDS: - case CHANNEL_SOURCE_RTL: - default: - return id+"."+getSourceName(); - } + return id+"."+getSourceName(); } - public static String getChannelSourceName(int id) { - return CHANNEL_SOURCE_NAMES[id-1]; - } - public String getSourceName() { - return getChannelSourceName(source); + return EPGSourceFactory.getChannelSourceName(source); } - public static int getChannelSourceId(String name) { - if (channelSourceNameMap.isEmpty()) { - int i=1; - for (String s: CHANNEL_SOURCE_NAMES) { - channelSourceNameMap.put(s, i); - i++; - } - } - return channelSourceNameMap.get(name); - - } - public void serialize(XMLStreamWriter writer) throws XMLStreamException { writer.writeStartElement("channel"); writer.writeAttribute("id", getXmltvChannelId()); diff --git a/src/main/java/org/vanbest/xmltv/Config.java b/src/main/java/org/vanbest/xmltv/Config.java index c5bfcc5..0707120 100644 --- a/src/main/java/org/vanbest/xmltv/Config.java +++ b/src/main/java/org/vanbest/xmltv/Config.java @@ -57,6 +57,7 @@ public class Config { // command-line options boolean quiet = false; public int logLevel = LOG_DEFAULT; + boolean fetchDetails = true; String project_version; String build_time; @@ -213,13 +214,13 @@ public class Config { // System.out.println("Adding channel " + parts + " in file format " + fileformat); switch(fileformat) { case 0: - c = Channel.getChannel(Channel.CHANNEL_SOURCE_TVGIDS, parts.get(1), parts.get(2)); + c = Channel.getChannel(EPGSourceFactory.CHANNEL_SOURCE_TVGIDS, parts.get(1), parts.get(2)); if (parts.size()>3) { c.addIcon(parts.get(3)); } break; case 1: - c = Channel.getChannel(Channel.CHANNEL_SOURCE_TVGIDS, parts.get(1), parts.get(3)); + c = Channel.getChannel(EPGSourceFactory.CHANNEL_SOURCE_TVGIDS, parts.get(1), parts.get(3)); if (parts.size()>4) { c.addIcon(parts.get(4)); } @@ -239,7 +240,7 @@ public class Config { if (fileformat==2) { source = Integer.parseInt(parts.get(1)); } else { - source = Channel.getChannelSourceId(parts.get(1)); + source = EPGSourceFactory.getChannelSourceId(parts.get(1)); } c = Channel.getChannel(source, parts.get(2), parts.get(4)); if (parts.size()>5) { diff --git a/src/main/java/org/vanbest/xmltv/EPGSource.java b/src/main/java/org/vanbest/xmltv/EPGSource.java index aa6a697..fd5a497 100644 --- a/src/main/java/org/vanbest/xmltv/EPGSource.java +++ b/src/main/java/org/vanbest/xmltv/EPGSource.java @@ -6,25 +6,22 @@ import java.util.List; import java.util.Set; public interface EPGSource { - public class Stats { int fetchErrors = 0; int cacheHits = 0; int cacheMisses = 0; } - public abstract void close() throws FileNotFoundException, IOException; - - public abstract List getChannels(); + public int getId(); + public String getName(); + public List getChannels(); // Convenience method - public abstract List getProgrammes(Channel channel, int day, - boolean fetchDetails) throws Exception; - - public abstract List getProgrammes(List channels, - int day, boolean fetchDetails) throws Exception; + public List getProgrammes(Channel channel, int day) throws Exception; + public List getProgrammes(List channels, int day) throws Exception; - public abstract Stats getStats(); + public Stats getStats(); public void clearCache(); -} \ No newline at end of file + public void close() throws FileNotFoundException, IOException; +} diff --git a/src/main/java/org/vanbest/xmltv/EPGSourceFactory.java b/src/main/java/org/vanbest/xmltv/EPGSourceFactory.java new file mode 100644 index 0000000..9634b17 --- /dev/null +++ b/src/main/java/org/vanbest/xmltv/EPGSourceFactory.java @@ -0,0 +1,56 @@ +package org.vanbest.xmltv; + +import java.util.HashMap; +import java.util.Map; + +public class EPGSourceFactory { + + public final static int CHANNEL_SOURCE_TVGIDS=1; + public final static int CHANNEL_SOURCE_RTL=2; + + private final static int[] CHANNEL_IDS={CHANNEL_SOURCE_TVGIDS, CHANNEL_SOURCE_RTL}; + private final static String[] CHANNEL_SOURCE_NAMES={"tvgids.nl", "rtl.nl"}; + private static Map channelSourceNameMap = new HashMap(); + + private EPGSourceFactory() { + } + + public static EPGSourceFactory newInstance() { + return new EPGSourceFactory(); + } + + public EPGSource createEPGSource(int source, Config config) { + switch(source) { + case EPGSourceFactory.CHANNEL_SOURCE_RTL: + return new RTL(config, false); + case EPGSourceFactory.CHANNEL_SOURCE_TVGIDS: + return new TvGids(config); + default: + return null; + } + } + + public EPGSource createEPGSource(String source, Config config) { + int sourceId = EPGSourceFactory.getChannelSourceId(source); + return createEPGSource(sourceId, config); + } + + public static String getChannelSourceName(int id) { + return CHANNEL_SOURCE_NAMES[id-1]; + } + + public static int getChannelSourceId(String name) { + if (channelSourceNameMap.isEmpty()) { + int i=1; + for (String s: EPGSourceFactory.CHANNEL_SOURCE_NAMES) { + EPGSourceFactory.channelSourceNameMap.put(s, i); + i++; + } + } + return EPGSourceFactory.channelSourceNameMap.get(name); + } + + public int[] getAll() { + return CHANNEL_IDS; + } +} diff --git a/src/main/java/org/vanbest/xmltv/Main.java b/src/main/java/org/vanbest/xmltv/Main.java index be8ea28..08278d2 100644 --- a/src/main/java/org/vanbest/xmltv/Main.java +++ b/src/main/java/org/vanbest/xmltv/Main.java @@ -26,6 +26,7 @@ import java.io.InputStreamReader; import java.io.PrintStream; import java.io.PrintWriter; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -68,18 +69,21 @@ public class Main { System.out.println("tv_grab_nl_java comes with ABSOLUTELY NO WARRANTY. It is free software, and you are welcome to redistribute it"); System.out.println("under certain conditions; `tv_grab_nl_java --license' for details."); } + public void run() throws FactoryConfigurationError, Exception { if (!config.quiet) { showHeader(); - System.out.println("Fetching programme data for " + this.days + " starting from day " + this.offset); + System.out.println("Fetching programme data for " + this.days + " days starting from day " + this.offset); int enabledCount = 0; for(Channel c: config.channels) { if (c.enabled) enabledCount++; } System.out.println("... from " + enabledCount + " channels"); System.out.println("... using cache at " + config.cacheDbHandle); } - EPGSource gids = new TvGids(config); - if (clearCache) gids.clearCache(); + Map guides = new HashMap(); + EPGSourceFactory factory = EPGSourceFactory.newInstance(); + //EPGSource gids = new TvGids(config); + //if (clearCache) gids.clearCache(); XMLStreamWriter writer = XMLOutputFactory.newInstance().createXMLStreamWriter(outputWriter); writer.writeStartDocument(); @@ -91,6 +95,7 @@ public class Main { writer.writeAttribute("source-info-url", "http://tvgids.nl/"); writer.writeAttribute("source-info-name", "TvGids.nl"); writer.writeAttribute("generator-info-name", "tv_grab_nl_java release "+config.project_version + ", built " + config.build_time); + writer.writeCharacters(System.getProperty("line.separator")); for(Channel c: config.channels) if (c.enabled) c.serialize(writer); @@ -99,7 +104,11 @@ public class Main { for(Channel c: config.channels) { if (!c.enabled) continue; if (!config.quiet) System.out.print("."); - List programmes = gids.getProgrammes(c, day, true); + if(!guides.containsKey(c.source)) { + guides.put(c.source, factory.createEPGSource(c.source, config)); + if (clearCache) guides.get(c.source).clearCache(); + } + List programmes = guides.get(c.source).getProgrammes(c, day); for (Programme p: programmes) p.serialize(writer); writer.flush(); } @@ -112,7 +121,9 @@ public class Main { writer.close(); try { - gids.close(); + for(int source: guides.keySet()) { + guides.get(source).close(); + } } catch (FileNotFoundException e) { // TODO Auto-generated catch block e.printStackTrace(); @@ -122,7 +133,13 @@ public class Main { } if (!config.quiet) { - EPGSource.Stats stats = gids.getStats(); + EPGSource.Stats stats = new EPGSource.Stats(); + for(int source: guides.keySet()) { + EPGSource.Stats part = guides.get(source).getStats(); + stats.cacheHits += part.cacheHits; + stats.cacheMisses += part.cacheMisses; + stats.fetchErrors += part.fetchErrors; + } System.out.println("Number of programmes from cache: " + stats.cacheHits); System.out.println("Number of programmes fetched: " + stats.cacheMisses); System.out.println("Number of fetch errors: " + stats.fetchErrors); @@ -132,16 +149,6 @@ public class Main { public void configure() throws IOException { showHeader(); - EPGSource gids = new TvGids(config); - - Set oldChannels = new HashSet(); - for (Channel c: config.channels) { - if (c.enabled) { - oldChannels.add(c.source+"::"+c.id); - } - } - List channels = gids.getChannels(); - BufferedReader reader = new BufferedReader(new InputStreamReader(System.in)); System.out.print("Delay between each request to the server (in milliseconds, default="+config.niceMilliseconds+"):"); @@ -163,6 +170,40 @@ public class Main { // public String cacheDbHandle; // public String cacheDbUser; // public String cacheDbPassword; + + + EPGSourceFactory factory = EPGSourceFactory.newInstance(); + int[] sources = factory.getAll(); + + System.out.println("Please select the TV programme information sources to use"); + List guides = new ArrayList(); + for(int source: sources) { + EPGSource guide = factory.createEPGSource(source, config); + System.out.print(" Use \"" + guide.getName() + "\" (Y/N):"); + while(true) { + String s = reader.readLine().toLowerCase(); + if ( s.startsWith("y")) { + guides.add(guide); + break; + } else if (s.startsWith("n")) { + break; + } + } + } + + //EPGSource gids = new TvGids(config); + + Set oldChannels = new HashSet(); + for (Channel c: config.channels) { + if (c.enabled) { + oldChannels.add(c.source+"::"+c.id); + } + } + + List channels = new ArrayList(); + for(EPGSource guide: guides) { + channels.addAll(guide.getChannels()); + } boolean all = false; boolean none = false; diff --git a/src/main/java/org/vanbest/xmltv/ProgrammeCache.java b/src/main/java/org/vanbest/xmltv/ProgrammeCache.java index cdfdbff..8a8ae80 100644 --- a/src/main/java/org/vanbest/xmltv/ProgrammeCache.java +++ b/src/main/java/org/vanbest/xmltv/ProgrammeCache.java @@ -51,12 +51,12 @@ public class ProgrammeCache { try { db = DriverManager.getConnection(config.cacheDbHandle, config.cacheDbUser, config.cacheDbPassword); Statement stat = db.createStatement(); - stat.execute("CREATE TABLE IF NOT EXISTS cache (id VARCHAR(64) PRIMARY KEY, date DATE, programme OTHER)"); + stat.execute("CREATE TABLE IF NOT EXISTS cache (source INTEGER, id VARCHAR(64), date DATE, programme OTHER, PRIMARY KEY (source,id))"); stat.close(); - getStatement = db.prepareStatement("SELECT programme FROM cache WHERE id=?"); - putStatement = db.prepareStatement("INSERT INTO cache VALUES (?,?,?)"); - removeStatement = db.prepareStatement("DELETE FROM cache WHERE id=?"); + getStatement = db.prepareStatement("SELECT programme FROM cache WHERE source=? AND id=?"); + putStatement = db.prepareStatement("INSERT INTO cache VALUES (?,?,?,?)"); + removeStatement = db.prepareStatement("DELETE FROM cache WHERE source=? AND id=?"); clearStatement = db.prepareStatement("DELETE FROM cache"); } catch (SQLException e) { db = null; @@ -67,17 +67,18 @@ public class ProgrammeCache { } } - public Programme get(String id) { + public Programme get(int source, String id) { if (db==null) return null; try { - getStatement.setString(1, id); + getStatement.setInt(1, source); + getStatement.setString(2, id); ResultSet r = getStatement.executeQuery(); if (!r.next()) return null; // not found try { Programme result = (Programme) r.getObject("programme"); return result; } catch (java.sql.SQLDataException e) { - removeCacheEntry(id); + removeCacheEntry(source, id); return null; } } catch (SQLException e) { @@ -88,9 +89,10 @@ public class ProgrammeCache { } } - private void removeCacheEntry(String id) { + private void removeCacheEntry(int source, String id) { try { - removeStatement.setString(1, id); + removeStatement.setInt(1, source); + removeStatement.setString(2, id); removeStatement.execute(); } catch (SQLException e) { // TODO Auto-generated catch block @@ -98,12 +100,14 @@ public class ProgrammeCache { } } - public void put(String id, Programme prog) { + public void put(int source, String id, Programme prog) { if (db == null) return; try { - putStatement.setString(1, id); - putStatement.setDate(2, new java.sql.Date(prog.startTime.getTime())); - putStatement.setObject(3, prog); + putStatement.setInt(1, source); + putStatement.setString(2, id); + putStatement.setDate(3, new java.sql.Date(prog.startTime.getTime())); + putStatement.setObject(4, prog); + //System.out.println(putStatement.toString()); int count = putStatement.executeUpdate(); if (count!=1 && !config.quiet) { System.out.println("Weird, cache database update statement affected " + count + " rows"); diff --git a/src/main/java/org/vanbest/xmltv/RTL.java b/src/main/java/org/vanbest/xmltv/RTL.java index 0881bcf..a8b30f9 100644 --- a/src/main/java/org/vanbest/xmltv/RTL.java +++ b/src/main/java/org/vanbest/xmltv/RTL.java @@ -18,6 +18,8 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; +import java.util.Collections; +import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -113,6 +115,14 @@ public class RTL extends AbstractEPGSource implements EPGSource { } } + public int getId() { + return EPGSourceFactory.CHANNEL_SOURCE_RTL; + } + + public String getName() { + return EPGSourceFactory.getChannelSourceName(getId()); + } + public List getChannels() { List result = new ArrayList(10); @@ -139,10 +149,21 @@ public class RTL extends AbstractEPGSource implements EPGSource { String name = (String) j.get(0); String icon = icon_url+id+".gif"; - Channel c = Channel.getChannel(Channel.CHANNEL_SOURCE_RTL, id, name, icon); + Channel c = Channel.getChannel(EPGSourceFactory.CHANNEL_SOURCE_RTL, id, name, icon); result.add(c); } + Collections.sort(result, new Comparator() { + public int compare(Channel o1, Channel o2) { + if (o1.source==o2.source) { + int c1=Integer.parseInt(o1.id); + int c2=Integer.parseInt(o2.id); + return (c1==c2 ? 0 : ((c1 getProgrammes(List channels, int day, - boolean fetchDetails) throws Exception { + public List getProgrammes(List channels, int day) throws Exception { List result = new LinkedList(); Map channelMap = new HashMap(); for(Channel c: channels) { - if (c.enabled && c.source==Channel.CHANNEL_SOURCE_RTL) channelMap.put(c.id, c); + if (c.enabled && c.source==EPGSourceFactory.CHANNEL_SOURCE_RTL) channelMap.put(c.id, c); } URL url = programmeUrl(day); //String xmltext = fetchURL(url); @@ -309,18 +329,18 @@ public class RTL extends AbstractEPGSource implements EPGSource { Document xml = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(url.openStream()); Element root = xml.getDocumentElement(); Date date = new SimpleDateFormat("yyyy-MM-dd").parse(root.getAttribute("date")); - System.out.println("date: " + date); + //System.out.println("date: " + date); String json = root.getTextContent(); - System.out.println("json: " + json); + //System.out.println("json: " + json); JSONObject o = JSONObject.fromObject( json ); for( Object k: o.keySet()) { String id = genericChannelId(k.toString()); if(!channelMap.containsKey(id)) { - if (!config.quiet) System.out.println("Skipping programmes for channel " + id); + //if (!config.quiet) System.out.println("Skipping programmes for channel " + id); continue; } JSONArray j = (JSONArray) o.get(k); - System.out.println(k.toString()+": "+j.toString()); + //System.out.println(k.toString()+": "+j.toString()); //System.out.println("Channel name:" + j.get(0)); for (int i=1; i programmes = rtl.getProgrammes(channels.subList(6, 9), 0, true); + List programmes = rtl.getProgrammes(channels.subList(6, 9), 0); //List programmes = rtl.getProgrammes(channels, 0, true); for(Programme p: programmes) {p.serialize(writer);} writer.writeEndElement(); diff --git a/src/main/java/org/vanbest/xmltv/TvGids.java b/src/main/java/org/vanbest/xmltv/TvGids.java index 70fcf1b..3668db2 100644 --- a/src/main/java/org/vanbest/xmltv/TvGids.java +++ b/src/main/java/org/vanbest/xmltv/TvGids.java @@ -62,6 +62,14 @@ public class TvGids extends AbstractEPGSource implements EPGSource { super(config); } + public int getId() { + return EPGSourceFactory.CHANNEL_SOURCE_TVGIDS; + } + + public String getName() { + return EPGSourceFactory.getChannelSourceName(getId()); + } + public static URL programmeUrl(List channels, int day) throws Exception { StringBuilder s = new StringBuilder(programme_base_url); if (channels.size() < 1) { @@ -158,7 +166,7 @@ public class TvGids extends AbstractEPGSource implements EPGSource { int id = zender.getInt("id"); String name = org.apache.commons.lang.StringEscapeUtils.unescapeHtml(zender.getString("name")); String icon = "http://tvgidsassets.nl/img/channels/53x27/" + id + ".png"; - Channel c = Channel.getChannel(Channel.CHANNEL_SOURCE_TVGIDS, Integer.toString(id), name, icon); + Channel c = Channel.getChannel(EPGSourceFactory.CHANNEL_SOURCE_TVGIDS, Integer.toString(id), name, icon); result.add(c); } @@ -175,7 +183,7 @@ public class TvGids extends AbstractEPGSource implements EPGSource { * @see org.vanbest.xmltv.EPGSource#getProgrammes(java.util.List, int, boolean) */ @Override - public List getProgrammes(List channels, int day, boolean fetchDetails) throws Exception { + public List getProgrammes(List channels, int day) throws Exception { List result = new ArrayList(); URL url = programmeUrl(channels, day); @@ -188,7 +196,7 @@ public class TvGids extends AbstractEPGSource implements EPGSource { JSONArray programs = (JSONArray) ps; for( int i=0; iMAX_PROGRAMMES_PER_DAY) break; JSONObject programme = programs.getJSONObject(o.toString()); - Programme p = programmeFromJSON(programme, fetchDetails); + Programme p = programmeFromJSON(programme, config.fetchDetails); p.channel = c.getXmltvChannelId(); result.add(p); count++; @@ -226,7 +234,7 @@ public class TvGids extends AbstractEPGSource implements EPGSource { */ private Programme programmeFromJSON(JSONObject programme, boolean fetchDetails) throws Exception { String id = programme.getString("db_id"); - Programme result = cache.get(id); + Programme result = cache.get(getId(), id); boolean cached = (result != null); if (result == null) { stats.cacheMisses++; @@ -267,7 +275,7 @@ public class TvGids extends AbstractEPGSource implements EPGSource { } if (!cached) { // FIXME where to do this? - cache.put(id, result); + cache.put(getId(), id, result); } if(config.logProgrammes()) { System.out.println(result.toString()); @@ -449,7 +457,7 @@ public class TvGids extends AbstractEPGSource implements EPGSource { List my_channels = channels.subList(0,15); for(Channel c: my_channels) {c.serialize(writer);} writer.flush(); - List programmes = gids.getProgrammes(my_channels, 2, true); + List programmes = gids.getProgrammes(my_channels, 2); for(Programme p: programmes) {p.serialize(writer);} writer.writeEndElement(); writer.writeEndDocument(); -- 2.39.5