From: Jan-Pascal van Best Date: Sat, 31 Mar 2012 10:49:33 +0000 (+0200) Subject: Implemented generic Programme cache using hsqldbl; fixed reading enabled/disabled... X-Git-Tag: 0.9.0~9 X-Git-Url: http://www.vanbest.org/gitweb/?a=commitdiff_plain;h=3865e75ad2ffed407b1dc71eb0a5943efd9126a5;p=tv_grab_nl_java Implemented generic Programme cache using hsqldbl; fixed reading enabled/disabled channels from config file; put source strings in config file; using caching with RTL grabber --- diff --git a/.gitignore b/.gitignore index 9f114ba..3f450ce 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /rtl.xml /target /testdb.* +/tv_grab_nl_java.db.properties diff --git a/src/main/java/org/vanbest/xmltv/AbstractEPGSource.java b/src/main/java/org/vanbest/xmltv/AbstractEPGSource.java index 09d2437..ef40f5b 100644 --- a/src/main/java/org/vanbest/xmltv/AbstractEPGSource.java +++ b/src/main/java/org/vanbest/xmltv/AbstractEPGSource.java @@ -13,14 +13,14 @@ import org.vanbest.xmltv.EPGSource.Stats; public abstract class AbstractEPGSource implements EPGSource { protected Config config; - protected ProgrammeCache cache; + protected TvGidsProgrammeCache cache; protected Stats stats = new Stats(); public static final int MAX_FETCH_TRIES=5; public AbstractEPGSource(Config config) { this.config = config; - cache = new ProgrammeCache(config.cacheFile); + cache = new TvGidsProgrammeCache(config.cacheFile); } public Set getProgrammes(Channel channel, int day, boolean fetchDetails) diff --git a/src/main/java/org/vanbest/xmltv/Channel.java b/src/main/java/org/vanbest/xmltv/Channel.java index b26ba9d..650ae5e 100644 --- a/src/main/java/org/vanbest/xmltv/Channel.java +++ b/src/main/java/org/vanbest/xmltv/Channel.java @@ -1,8 +1,10 @@ package org.vanbest.xmltv; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import javax.xml.stream.XMLStreamException; @@ -18,6 +20,9 @@ public class Channel { public final static int CHANNEL_SOURCE_TVGIDS=1; public final static int CHANNEL_SOURCE_RTL=2; + private final static String[] CHANNEL_SOURCE_NAMES={"tvgids.nl", "rtl.nl"}; + private static Map channelSourceNameMap = new HashMap(); + protected Channel(int source, String id) { this.id = id; @@ -47,12 +52,30 @@ public class Channel { public String getXmltvChannelId() { switch (source) { case CHANNEL_SOURCE_TVGIDS: - return id+".tvgids.nl"; case CHANNEL_SOURCE_RTL: - return id; default: - return id; + return id+"."+getSourceName(); + } + } + + public static String getChannelSourceName(int id) { + return CHANNEL_SOURCE_NAMES[id-1]; + } + + public String getSourceName() { + return getChannelSourceName(source); + } + + public static int getChannelSourceId(String name) { + if (channelSourceNameMap.isEmpty()) { + int i=1; + for (String s: CHANNEL_SOURCE_NAMES) { + channelSourceNameMap.put(s, i); + i++; + } } + return channelSourceNameMap.get(name); + } public void serialize(XMLStreamWriter writer) throws XMLStreamException { diff --git a/src/main/java/org/vanbest/xmltv/Config.java b/src/main/java/org/vanbest/xmltv/Config.java index 86c88b2..3a9544d 100644 --- a/src/main/java/org/vanbest/xmltv/Config.java +++ b/src/main/java/org/vanbest/xmltv/Config.java @@ -40,6 +40,9 @@ public class Config { public List channels; public Map cattrans; protected File cacheFile; + public String cacheDbHandle; + public String cacheDbUser; + public String cacheDbPassword; boolean quiet = false; public int logLevel = LOG_DEFAULT; @@ -49,7 +52,7 @@ public class Config { public static int LOG_DEFAULT = LOG_INFO; - private final static int CURRENT_FILE_FORMAT=2; + private final static int CURRENT_FILE_FORMAT=3; String project_version; @@ -70,6 +73,9 @@ public class Config { result.cattrans = getDefaultCattrans(); result.cacheFile = defaultCacheFile(); result.niceMilliseconds = 500; + result.cacheDbHandle = "jdbc:hsqldb:file:cachedb"; // FIXME in userdir + result.cacheDbUser = "SA"; + result.cacheDbPassword = ""; return result; } @@ -125,8 +131,8 @@ public class Config { out.println("nice-time-milliseconds: " + niceMilliseconds); for(Channel c: channels) { // FIXME: handle multiple channels names, icons and urls - out.print("channel: " + c.source + - ":" + c.id + + out.print("channel: " + c.getSourceName() + + ": " + c.id + ": " + (c.enabled?"enabled":"disabled") + ": " + escape(c.defaultName())); if (!c.icons.isEmpty()) { @@ -219,7 +225,14 @@ public class Config { } break; case 2: - c = Channel.getChannel(Integer.parseInt(parts.get(1)), parts.get(2), parts.get(4)); + case 3: + int source; + if (fileformat==2) { + source = Integer.parseInt(parts.get(1)); + } else { + source = Channel.getChannelSourceId(parts.get(1)); + } + c = Channel.getChannel(source, parts.get(2), parts.get(4)); if (parts.size()>5) { c.addIcon(parts.get(5)); } diff --git a/src/main/java/org/vanbest/xmltv/Main.java b/src/main/java/org/vanbest/xmltv/Main.java index 76e291b..d05f132 100644 --- a/src/main/java/org/vanbest/xmltv/Main.java +++ b/src/main/java/org/vanbest/xmltv/Main.java @@ -112,7 +112,9 @@ public class Main { Set oldChannels = new HashSet(); for (Channel c: config.channels) { - oldChannels.add(c.id); + if (c.enabled) { + oldChannels.add(c.source+"::"+c.id); + } } List channels = gids.getChannels(); @@ -121,7 +123,7 @@ public class Main { boolean none = false; boolean keep = false; for (Channel c: channels) { - boolean selected = oldChannels.contains(c.id); + boolean selected = oldChannels.contains(c.source+"::"+c.id); System.out.print("add channel " + c.id + " (" + c.defaultName() + ") [[y]es,[n]o,[a]ll,[none],[k]eep selection (default=" + (selected?"yes":"no") + ")] "); if (keep) { c.enabled = selected; diff --git a/src/main/java/org/vanbest/xmltv/Programme.java b/src/main/java/org/vanbest/xmltv/Programme.java index 08fbc49..0bcb6c5 100644 --- a/src/main/java/org/vanbest/xmltv/Programme.java +++ b/src/main/java/org/vanbest/xmltv/Programme.java @@ -1,5 +1,6 @@ package org.vanbest.xmltv; +import java.io.Serializable; import java.net.URL; import java.text.DateFormat; import java.text.SimpleDateFormat; @@ -11,8 +12,8 @@ import java.util.concurrent.TimeUnit; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamWriter; -public class Programme { - class Title { +public class Programme implements Serializable { + class Title implements Serializable { String title; String lang; public Title(String title, String lang) { @@ -20,11 +21,11 @@ public class Programme { this.lang = lang; } } - class Actor { + class Actor implements Serializable { String name; String role; } - class Credits { + class Credits implements Serializable { List directors; List actors; List writers; @@ -36,30 +37,30 @@ public class Programme { List commentators; List guests; } - class Length { + class Length implements Serializable { TimeUnit unit; int count; } - class Icon { + class Icon implements Serializable { URL url; int width; int height; } - class Episode { + class Episode implements Serializable { String episode; String system; // onscreen or xmltv_ns } - class Video { + class Video implements Serializable { boolean present; boolean colour; String aspect; // eg. 16:9, 4:3 String quality; // eg. 'HDTV', '800x600'. } - class Audio { + class Audio implements Serializable { boolean present; String stereo; // 'mono','stereo','dolby','dolby digital','bilingual' or 'surround'. } - class Subtitle { + class Subtitle implements Serializable { String type; // teletext | onscreen | deaf-signed Title language; } @@ -69,7 +70,7 @@ public class Programme { public Date vpsStart; public String showview; public String videoplus; - public Channel channel; // required + public String channel; // required xmltvid of the associated channel public String clumpidx; public List titles; // at least one @@ -170,7 +171,7 @@ public class Programme { writer.writeStartElement("programme"); if(startTime != null) writer.writeAttribute("start", df.format(startTime)); if(endTime != null) writer.writeAttribute("stop", df.format(endTime)); - if(channel != null) writer.writeAttribute("channel", ""+channel.id); + if(channel != null) writer.writeAttribute("channel", ""+channel); writeTitleList(titles,"title",writer); writeTitleList(secondaryTitles,"sub-title", writer); if(credits != null) { diff --git a/src/main/java/org/vanbest/xmltv/ProgrammeCache.java b/src/main/java/org/vanbest/xmltv/ProgrammeCache.java index 926b0c2..6368707 100644 --- a/src/main/java/org/vanbest/xmltv/ProgrammeCache.java +++ b/src/main/java/org/vanbest/xmltv/ProgrammeCache.java @@ -25,50 +25,101 @@ import java.io.InputStream; import java.io.InvalidClassException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; +import java.sql.Connection; +import java.sql.Date; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; import java.util.HashMap; import java.util.Map; +import java.util.Properties; import org.apache.commons.io.FileUtils; public class ProgrammeCache { - private File cacheFile; - private Map<String,TvGidsProgrammeDetails> cache; + private Connection db; + private Config config; + private PreparedStatement getStatement; + private PreparedStatement putStatement; - public ProgrammeCache(File cacheFile) { - this.cacheFile = cacheFile; - if (cacheFile.canRead()) { - try { - cache = (Map<String,TvGidsProgrammeDetails>) new ObjectInputStream( new FileInputStream( cacheFile ) ).readObject(); - } catch (InvalidClassException e) { - // TODO Auto-generated catch block - cache = new HashMap<String,TvGidsProgrammeDetails>(); - } catch (ClassNotFoundException e) { - // TODO Auto-generated catch block - cache = new HashMap<String,TvGidsProgrammeDetails>(); - } catch (FileNotFoundException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - cache = new HashMap<String,TvGidsProgrammeDetails>(); - } catch (IOException e) { - // TODO Auto-generated catch block + public ProgrammeCache(Config config) { + this.config = config; + try { + db = DriverManager.getConnection(config.cacheDbHandle, config.cacheDbUser, config.cacheDbPassword); + Statement stat = db.createStatement(); + stat.execute("CREATE TABLE IF NOT EXISTS cache (id VARCHAR(64) PRIMARY KEY, date DATE, programme OTHER)"); + stat.close(); + + getStatement = db.prepareStatement("SELECT programme FROM cache WHERE id=?"); + putStatement = db.prepareStatement("INSERT INTO cache VALUES (?,?,?)"); + } catch (SQLException e) { + db = null; + if (!config.quiet) { + System.out.println("Unable to open cache database, proceeding without cache"); e.printStackTrace(); - cache = new HashMap<String,TvGidsProgrammeDetails>(); } - } else { - cache = new HashMap<String,TvGidsProgrammeDetails>(); } - // FileUtils.forceMkdir(root); } - public TvGidsProgrammeDetails getDetails(String id) { - return cache.get(id); + public Programme get(String id) { + if (db==null) return null; + try { + getStatement.setString(1, id); + ResultSet r = getStatement.executeQuery(); + if (!r.next()) return null; // not found + return (Programme) r.getObject("programme"); + } catch (SQLException e) { + if (!config.quiet) { + e.printStackTrace(); + } + return null; + } } - public void add(String id, TvGidsProgrammeDetails d) { - cache.put(id, d); + public void put(String id, Programme prog) { + if (db == null) return; + try { + putStatement.setString(1, id); + putStatement.setDate(2, new java.sql.Date(prog.startTime.getTime())); + putStatement.setObject(3, prog); + int count = putStatement.executeUpdate(); + if (count!=1 && !config.quiet) { + System.out.println("Weird, cache database update statement affected " + count + " rows"); + } + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } } - + + public void cleanup() { + Statement stat; + try { + stat = db.createStatement(); + int count = stat.executeUpdate("DELETE FROM cache WHERE date<CURRENT_DATE - 3 DAY"); + if (!config.quiet) { + System.out.println("Purged " + count + " old entries from cache"); + } + stat.close(); + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + public void close() throws FileNotFoundException, IOException { - new ObjectOutputStream( new FileOutputStream(cacheFile)).writeObject(cache); + cleanup(); + if (db != null) { + try { + getStatement.close(); + putStatement.close(); + db.close(); + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } } } diff --git a/src/main/java/org/vanbest/xmltv/RTL.java b/src/main/java/org/vanbest/xmltv/RTL.java index 69fff39..86d8495 100644 --- a/src/main/java/org/vanbest/xmltv/RTL.java +++ b/src/main/java/org/vanbest/xmltv/RTL.java @@ -58,7 +58,7 @@ public class RTL extends AbstractEPGSource implements EPGSource { private static final String detail_url="http://www.rtl.nl/active/epg_data/uitzending_data/"; private static final String icon_url="http://www.rtl.nl/service/gids/components/vaste_componenten/"; private static final String xmltv_channel_suffix = ".rtl.nl"; - private static final int MAX_PROGRAMMES_PER_DAY = 200000; + private static final int MAX_PROGRAMMES_PER_DAY = 20; private Connection db; @@ -66,6 +66,7 @@ public class RTL extends AbstractEPGSource implements EPGSource { "site_path", "wwwadres", "presentatie", "omroep", "eindtijd", "inhoud", "tt_inhoud", "alginhoud", "afl_titel", "kijkwijzer" }; Map<String,Integer> xmlKeyMap = new HashMap<String,Integer>(); + private ProgrammeCache cache; class RTLException extends Exception { public RTLException(String s) { @@ -75,6 +76,7 @@ public class RTL extends AbstractEPGSource implements EPGSource { public RTL(Config config, boolean useDB) { super(config); + cache = new ProgrammeCache(config); try { if (useDB) { Properties dbProp = new Properties(); @@ -261,9 +263,7 @@ public class RTL extends AbstractEPGSource implements EPGSource { } else if (tag.equals("omroep")) { } else if (tag.equals("kijkwijzer")) { } else if (tag.equals("presentatie")) { - // A - // A en B - // A, B, C en D + // A; A en B; A, B, C en D String[] presentatoren = e.getTextContent().split(", | en "); for(String pres:presentatoren) { prog.addPresenter(pres); @@ -299,7 +299,7 @@ public class RTL extends AbstractEPGSource implements EPGSource { List<Programme> result = new LinkedList<Programme>(); Map<String,Channel> channelMap = new HashMap<String,Channel>(); for(Channel c: channels) { - if (c.enabled) channelMap.put(c.id, c); + if (c.enabled && c.source==Channel.CHANNEL_SOURCE_RTL) channelMap.put(c.id, c); } URL url = programmeUrl(day); //String xmltext = fetchURL(url); @@ -328,18 +328,30 @@ public class RTL extends AbstractEPGSource implements EPGSource { String programme_id = p.getString(2); String quark1 = p.getString(3); String quark2 = p.getString(4); - Programme prog = new Programme(); - prog.addTitle(title); - Date start = parseTime(date, starttime); - prog.startTime = start; - prog.channel = channelMap.get(id); - fetchDetail(prog, date, programme_id); + Programme prog = cache.get(programme_id); + if (prog == null) { + stats.cacheMisses++; + prog = new Programme(); + prog.addTitle(title); + prog.startTime = parseTime(date, starttime); + prog.channel = channelMap.get(id).getXmltvChannelId(); + if (fetchDetails) { + fetchDetail(prog, date, programme_id); + } + cache.put(programme_id, prog); + } else { + stats.cacheHits++; + } result.add(prog); } } return result; } + public void close() throws FileNotFoundException, IOException { + super.close(); + cache.close(); + } private Date parseTime(Date date, String time) { Calendar result = Calendar.getInstance(); result.setTime(date); @@ -381,12 +393,18 @@ public class RTL extends AbstractEPGSource implements EPGSource { writer.writeStartElement("tv"); for(Channel c: channels) {c.serialize(writer);} writer.flush(); - //List<Programme> programmes = rtl.getProgrammes1(channels.subList(6, 9), 0, true); - List<Programme> programmes = rtl.getProgrammes1(channels, 0, true); + List<Programme> programmes = rtl.getProgrammes1(channels.subList(6, 9), 0, true); + //List<Programme> programmes = rtl.getProgrammes1(channels, 0, true); for(Programme p: programmes) {p.serialize(writer);} writer.writeEndElement(); writer.writeEndDocument(); writer.flush(); + if (!config.quiet) { + EPGSource.Stats stats = rtl.getStats(); + System.out.println("Number of programmes from cache: " + stats.cacheHits); + System.out.println("Number of programmes fetched: " + stats.cacheMisses); + System.out.println("Number of fetch errors: " + stats.fetchErrors); + } rtl.close(); } catch (Exception e) { // TODO Auto-generated catch block diff --git a/src/main/java/org/vanbest/xmltv/TvGidsProgrammeCache.java b/src/main/java/org/vanbest/xmltv/TvGidsProgrammeCache.java new file mode 100644 index 0000000..90afc63 --- /dev/null +++ b/src/main/java/org/vanbest/xmltv/TvGidsProgrammeCache.java @@ -0,0 +1,74 @@ +package org.vanbest.xmltv; + +/* + Copyright (c) 2012 Jan-Pascal van Best <janpascal@vanbest.org> + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + The full license text can be found in the LICENSE file. +*/ + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InvalidClassException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.io.FileUtils; + +public class TvGidsProgrammeCache { + private File cacheFile; + private Map<String,TvGidsProgrammeDetails> cache; + + public TvGidsProgrammeCache(File cacheFile) { + this.cacheFile = cacheFile; + if (cacheFile.canRead()) { + try { + cache = (Map<String,TvGidsProgrammeDetails>) new ObjectInputStream( new FileInputStream( cacheFile ) ).readObject(); + } catch (InvalidClassException e) { + // TODO Auto-generated catch block + cache = new HashMap<String,TvGidsProgrammeDetails>(); + } catch (ClassNotFoundException e) { + // TODO Auto-generated catch block + cache = new HashMap<String,TvGidsProgrammeDetails>(); + } catch (FileNotFoundException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + cache = new HashMap<String,TvGidsProgrammeDetails>(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + cache = new HashMap<String,TvGidsProgrammeDetails>(); + } + } else { + cache = new HashMap<String,TvGidsProgrammeDetails>(); + } + // FileUtils.forceMkdir(root); + } + + public TvGidsProgrammeDetails getDetails(String id) { + return cache.get(id); + } + + public void add(String id, TvGidsProgrammeDetails d) { + cache.put(id, d); + } + + public void close() throws FileNotFoundException, IOException { + new ObjectOutputStream( new FileOutputStream(cacheFile)).writeObject(cache); + } +} diff --git a/src/main/java/org/vanbest/xmltv/XmlTvWriter.java b/src/main/java/org/vanbest/xmltv/XmlTvWriter.java index b2cb893..e8ea472 100644 --- a/src/main/java/org/vanbest/xmltv/XmlTvWriter.java +++ b/src/main/java/org/vanbest/xmltv/XmlTvWriter.java @@ -59,7 +59,6 @@ public class XmlTvWriter { for(Channel c: channels) { if (!c.enabled) continue; c.serialize(writer); - writeln(); } }