From 3865e75ad2ffed407b1dc71eb0a5943efd9126a5 Mon Sep 17 00:00:00 2001
From: Jan-Pascal van Best <janpascal@vanbest.org>
Date: Sat, 31 Mar 2012 12:49:33 +0200
Subject: [PATCH] Implemented generic Programme cache using hsqldbl; fixed
reading enabled/disabled channels from config file; put source strings in
config file; using caching with RTL grabber
---
.gitignore | 1 +
.../org/vanbest/xmltv/AbstractEPGSource.java | 4 +-
src/main/java/org/vanbest/xmltv/Channel.java | 29 ++++-
src/main/java/org/vanbest/xmltv/Config.java | 21 +++-
src/main/java/org/vanbest/xmltv/Main.java | 6 +-
.../java/org/vanbest/xmltv/Programme.java | 25 ++--
.../org/vanbest/xmltv/ProgrammeCache.java | 109 +++++++++++++-----
src/main/java/org/vanbest/xmltv/RTL.java | 44 ++++---
.../vanbest/xmltv/TvGidsProgrammeCache.java | 74 ++++++++++++
.../java/org/vanbest/xmltv/XmlTvWriter.java | 1 -
10 files changed, 248 insertions(+), 66 deletions(-)
create mode 100644 src/main/java/org/vanbest/xmltv/TvGidsProgrammeCache.java
diff --git a/.gitignore b/.gitignore
index 9f114ba..3f450ce 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,4 @@
/rtl.xml
/target
/testdb.*
+/tv_grab_nl_java.db.properties
diff --git a/src/main/java/org/vanbest/xmltv/AbstractEPGSource.java b/src/main/java/org/vanbest/xmltv/AbstractEPGSource.java
index 09d2437..ef40f5b 100644
--- a/src/main/java/org/vanbest/xmltv/AbstractEPGSource.java
+++ b/src/main/java/org/vanbest/xmltv/AbstractEPGSource.java
@@ -13,14 +13,14 @@ import org.vanbest.xmltv.EPGSource.Stats;
public abstract class AbstractEPGSource implements EPGSource {
protected Config config;
- protected ProgrammeCache cache;
+ protected TvGidsProgrammeCache cache;
protected Stats stats = new Stats();
public static final int MAX_FETCH_TRIES=5;
public AbstractEPGSource(Config config) {
this.config = config;
- cache = new ProgrammeCache(config.cacheFile);
+ cache = new TvGidsProgrammeCache(config.cacheFile);
}
public Set<TvGidsProgramme> getProgrammes(Channel channel, int day, boolean fetchDetails)
diff --git a/src/main/java/org/vanbest/xmltv/Channel.java b/src/main/java/org/vanbest/xmltv/Channel.java
index b26ba9d..650ae5e 100644
--- a/src/main/java/org/vanbest/xmltv/Channel.java
+++ b/src/main/java/org/vanbest/xmltv/Channel.java
@@ -1,8 +1,10 @@
package org.vanbest.xmltv;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import javax.xml.stream.XMLStreamException;
@@ -18,6 +20,9 @@ public class Channel {
public final static int CHANNEL_SOURCE_TVGIDS=1;
public final static int CHANNEL_SOURCE_RTL=2;
+ private final static String[] CHANNEL_SOURCE_NAMES={"tvgids.nl", "rtl.nl"};
+ private static Map<String,Integer> channelSourceNameMap = new HashMap<String,Integer>();
+
protected Channel(int source, String id) {
this.id = id;
@@ -47,12 +52,30 @@ public class Channel {
public String getXmltvChannelId() {
switch (source) {
case CHANNEL_SOURCE_TVGIDS:
- return id+".tvgids.nl";
case CHANNEL_SOURCE_RTL:
- return id;
default:
- return id;
+ return id+"."+getSourceName();
+ }
+ }
+
+ public static String getChannelSourceName(int id) {
+ return CHANNEL_SOURCE_NAMES[id-1];
+ }
+
+ public String getSourceName() {
+ return getChannelSourceName(source);
+ }
+
+ public static int getChannelSourceId(String name) {
+ if (channelSourceNameMap.isEmpty()) {
+ int i=1;
+ for (String s: CHANNEL_SOURCE_NAMES) {
+ channelSourceNameMap.put(s, i);
+ i++;
+ }
}
+ return channelSourceNameMap.get(name);
+
}
public void serialize(XMLStreamWriter writer) throws XMLStreamException {
diff --git a/src/main/java/org/vanbest/xmltv/Config.java b/src/main/java/org/vanbest/xmltv/Config.java
index 86c88b2..3a9544d 100644
--- a/src/main/java/org/vanbest/xmltv/Config.java
+++ b/src/main/java/org/vanbest/xmltv/Config.java
@@ -40,6 +40,9 @@ public class Config {
public List<Channel> channels;
public Map<String, String> cattrans;
protected File cacheFile;
+ public String cacheDbHandle;
+ public String cacheDbUser;
+ public String cacheDbPassword;
boolean quiet = false;
public int logLevel = LOG_DEFAULT;
@@ -49,7 +52,7 @@ public class Config {
public static int LOG_DEFAULT = LOG_INFO;
- private final static int CURRENT_FILE_FORMAT=2;
+ private final static int CURRENT_FILE_FORMAT=3;
String project_version;
@@ -70,6 +73,9 @@ public class Config {
result.cattrans = getDefaultCattrans();
result.cacheFile = defaultCacheFile();
result.niceMilliseconds = 500;
+ result.cacheDbHandle = "jdbc:hsqldb:file:cachedb"; // FIXME in userdir
+ result.cacheDbUser = "SA";
+ result.cacheDbPassword = "";
return result;
}
@@ -125,8 +131,8 @@ public class Config {
out.println("nice-time-milliseconds: " + niceMilliseconds);
for(Channel c: channels) {
// FIXME: handle multiple channels names, icons and urls
- out.print("channel: " + c.source +
- ":" + c.id +
+ out.print("channel: " + c.getSourceName() +
+ ": " + c.id +
": " + (c.enabled?"enabled":"disabled") +
": " + escape(c.defaultName()));
if (!c.icons.isEmpty()) {
@@ -219,7 +225,14 @@ public class Config {
}
break;
case 2:
- c = Channel.getChannel(Integer.parseInt(parts.get(1)), parts.get(2), parts.get(4));
+ case 3:
+ int source;
+ if (fileformat==2) {
+ source = Integer.parseInt(parts.get(1));
+ } else {
+ source = Channel.getChannelSourceId(parts.get(1));
+ }
+ c = Channel.getChannel(source, parts.get(2), parts.get(4));
if (parts.size()>5) {
c.addIcon(parts.get(5));
}
diff --git a/src/main/java/org/vanbest/xmltv/Main.java b/src/main/java/org/vanbest/xmltv/Main.java
index 76e291b..d05f132 100644
--- a/src/main/java/org/vanbest/xmltv/Main.java
+++ b/src/main/java/org/vanbest/xmltv/Main.java
@@ -112,7 +112,9 @@ public class Main {
Set<String> oldChannels = new HashSet<String>();
for (Channel c: config.channels) {
- oldChannels.add(c.id);
+ if (c.enabled) {
+ oldChannels.add(c.source+"::"+c.id);
+ }
}
List<Channel> channels = gids.getChannels();
@@ -121,7 +123,7 @@ public class Main {
boolean none = false;
boolean keep = false;
for (Channel c: channels) {
- boolean selected = oldChannels.contains(c.id);
+ boolean selected = oldChannels.contains(c.source+"::"+c.id);
System.out.print("add channel " + c.id + " (" + c.defaultName() + ") [[y]es,[n]o,[a]ll,[none],[k]eep selection (default=" + (selected?"yes":"no") + ")] ");
if (keep) {
c.enabled = selected;
diff --git a/src/main/java/org/vanbest/xmltv/Programme.java b/src/main/java/org/vanbest/xmltv/Programme.java
index 08fbc49..0bcb6c5 100644
--- a/src/main/java/org/vanbest/xmltv/Programme.java
+++ b/src/main/java/org/vanbest/xmltv/Programme.java
@@ -1,5 +1,6 @@
package org.vanbest.xmltv;
+import java.io.Serializable;
import java.net.URL;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
@@ -11,8 +12,8 @@ import java.util.concurrent.TimeUnit;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamWriter;
-public class Programme {
- class Title {
+public class Programme implements Serializable {
+ class Title implements Serializable {
String title;
String lang;
public Title(String title, String lang) {
@@ -20,11 +21,11 @@ public class Programme {
this.lang = lang;
}
}
- class Actor {
+ class Actor implements Serializable {
String name;
String role;
}
- class Credits {
+ class Credits implements Serializable {
List<String> directors;
List<Actor> actors;
List<String> writers;
@@ -36,30 +37,30 @@ public class Programme {
List<String> commentators;
List<String> guests;
}
- class Length {
+ class Length implements Serializable {
TimeUnit unit;
int count;
}
- class Icon {
+ class Icon implements Serializable {
URL url;
int width;
int height;
}
- class Episode {
+ class Episode implements Serializable {
String episode;
String system; // onscreen or xmltv_ns
}
- class Video {
+ class Video implements Serializable {
boolean present;
boolean colour;
String aspect; // eg. 16:9, 4:3
String quality; // eg. 'HDTV', '800x600'.
}
- class Audio {
+ class Audio implements Serializable {
boolean present;
String stereo; // 'mono','stereo','dolby','dolby digital','bilingual' or 'surround'.
}
- class Subtitle {
+ class Subtitle implements Serializable {
String type; // teletext | onscreen | deaf-signed
Title language;
}
@@ -69,7 +70,7 @@ public class Programme {
public Date vpsStart;
public String showview;
public String videoplus;
- public Channel channel; // required
+ public String channel; // required xmltvid of the associated channel
public String clumpidx;
public List<Title> titles; // at least one
@@ -170,7 +171,7 @@ public class Programme {
writer.writeStartElement("programme");
if(startTime != null) writer.writeAttribute("start", df.format(startTime));
if(endTime != null) writer.writeAttribute("stop", df.format(endTime));
- if(channel != null) writer.writeAttribute("channel", ""+channel.id);
+ if(channel != null) writer.writeAttribute("channel", ""+channel);
writeTitleList(titles,"title",writer);
writeTitleList(secondaryTitles,"sub-title", writer);
if(credits != null) {
diff --git a/src/main/java/org/vanbest/xmltv/ProgrammeCache.java b/src/main/java/org/vanbest/xmltv/ProgrammeCache.java
index 926b0c2..6368707 100644
--- a/src/main/java/org/vanbest/xmltv/ProgrammeCache.java
+++ b/src/main/java/org/vanbest/xmltv/ProgrammeCache.java
@@ -25,50 +25,101 @@ import java.io.InputStream;
import java.io.InvalidClassException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
+import java.sql.Connection;
+import java.sql.Date;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
import java.util.HashMap;
import java.util.Map;
+import java.util.Properties;
import org.apache.commons.io.FileUtils;
public class ProgrammeCache {
- private File cacheFile;
- private Map<String,TvGidsProgrammeDetails> cache;
+ private Connection db;
+ private Config config;
+ private PreparedStatement getStatement;
+ private PreparedStatement putStatement;
- public ProgrammeCache(File cacheFile) {
- this.cacheFile = cacheFile;
- if (cacheFile.canRead()) {
- try {
- cache = (Map<String,TvGidsProgrammeDetails>) new ObjectInputStream( new FileInputStream( cacheFile ) ).readObject();
- } catch (InvalidClassException e) {
- // TODO Auto-generated catch block
- cache = new HashMap<String,TvGidsProgrammeDetails>();
- } catch (ClassNotFoundException e) {
- // TODO Auto-generated catch block
- cache = new HashMap<String,TvGidsProgrammeDetails>();
- } catch (FileNotFoundException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- cache = new HashMap<String,TvGidsProgrammeDetails>();
- } catch (IOException e) {
- // TODO Auto-generated catch block
+ public ProgrammeCache(Config config) {
+ this.config = config;
+ try {
+ db = DriverManager.getConnection(config.cacheDbHandle, config.cacheDbUser, config.cacheDbPassword);
+ Statement stat = db.createStatement();
+ stat.execute("CREATE TABLE IF NOT EXISTS cache (id VARCHAR(64) PRIMARY KEY, date DATE, programme OTHER)");
+ stat.close();
+
+ getStatement = db.prepareStatement("SELECT programme FROM cache WHERE id=?");
+ putStatement = db.prepareStatement("INSERT INTO cache VALUES (?,?,?)");
+ } catch (SQLException e) {
+ db = null;
+ if (!config.quiet) {
+ System.out.println("Unable to open cache database, proceeding without cache");
e.printStackTrace();
- cache = new HashMap<String,TvGidsProgrammeDetails>();
}
- } else {
- cache = new HashMap<String,TvGidsProgrammeDetails>();
}
- // FileUtils.forceMkdir(root);
}
- public TvGidsProgrammeDetails getDetails(String id) {
- return cache.get(id);
+ public Programme get(String id) {
+ if (db==null) return null;
+ try {
+ getStatement.setString(1, id);
+ ResultSet r = getStatement.executeQuery();
+ if (!r.next()) return null; // not found
+ return (Programme) r.getObject("programme");
+ } catch (SQLException e) {
+ if (!config.quiet) {
+ e.printStackTrace();
+ }
+ return null;
+ }
}
- public void add(String id, TvGidsProgrammeDetails d) {
- cache.put(id, d);
+ public void put(String id, Programme prog) {
+ if (db == null) return;
+ try {
+ putStatement.setString(1, id);
+ putStatement.setDate(2, new java.sql.Date(prog.startTime.getTime()));
+ putStatement.setObject(3, prog);
+ int count = putStatement.executeUpdate();
+ if (count!=1 && !config.quiet) {
+ System.out.println("Weird, cache database update statement affected " + count + " rows");
+ }
+ } catch (SQLException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
}
-
+
+ public void cleanup() {
+ Statement stat;
+ try {
+ stat = db.createStatement();
+ int count = stat.executeUpdate("DELETE FROM cache WHERE date<CURRENT_DATE - 3 DAY");
+ if (!config.quiet) {
+ System.out.println("Purged " + count + " old entries from cache");
+ }
+ stat.close();
+ } catch (SQLException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+
public void close() throws FileNotFoundException, IOException {
- new ObjectOutputStream( new FileOutputStream(cacheFile)).writeObject(cache);
+ cleanup();
+ if (db != null) {
+ try {
+ getStatement.close();
+ putStatement.close();
+ db.close();
+ } catch (SQLException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
}
}
diff --git a/src/main/java/org/vanbest/xmltv/RTL.java b/src/main/java/org/vanbest/xmltv/RTL.java
index 69fff39..86d8495 100644
--- a/src/main/java/org/vanbest/xmltv/RTL.java
+++ b/src/main/java/org/vanbest/xmltv/RTL.java
@@ -58,7 +58,7 @@ public class RTL extends AbstractEPGSource implements EPGSource {
private static final String detail_url="http://www.rtl.nl/active/epg_data/uitzending_data/";
private static final String icon_url="http://www.rtl.nl/service/gids/components/vaste_componenten/";
private static final String xmltv_channel_suffix = ".rtl.nl";
- private static final int MAX_PROGRAMMES_PER_DAY = 200000;
+ private static final int MAX_PROGRAMMES_PER_DAY = 20;
private Connection db;
@@ -66,6 +66,7 @@ public class RTL extends AbstractEPGSource implements EPGSource {
"site_path", "wwwadres", "presentatie", "omroep", "eindtijd", "inhoud", "tt_inhoud", "alginhoud", "afl_titel", "kijkwijzer" };
Map<String,Integer> xmlKeyMap = new HashMap<String,Integer>();
+ private ProgrammeCache cache;
class RTLException extends Exception {
public RTLException(String s) {
@@ -75,6 +76,7 @@ public class RTL extends AbstractEPGSource implements EPGSource {
public RTL(Config config, boolean useDB) {
super(config);
+ cache = new ProgrammeCache(config);
try {
if (useDB) {
Properties dbProp = new Properties();
@@ -261,9 +263,7 @@ public class RTL extends AbstractEPGSource implements EPGSource {
} else if (tag.equals("omroep")) {
} else if (tag.equals("kijkwijzer")) {
} else if (tag.equals("presentatie")) {
- // A
- // A en B
- // A, B, C en D
+ // A; A en B; A, B, C en D
String[] presentatoren = e.getTextContent().split(", | en ");
for(String pres:presentatoren) {
prog.addPresenter(pres);
@@ -299,7 +299,7 @@ public class RTL extends AbstractEPGSource implements EPGSource {
List<Programme> result = new LinkedList<Programme>();
Map<String,Channel> channelMap = new HashMap<String,Channel>();
for(Channel c: channels) {
- if (c.enabled) channelMap.put(c.id, c);
+ if (c.enabled && c.source==Channel.CHANNEL_SOURCE_RTL) channelMap.put(c.id, c);
}
URL url = programmeUrl(day);
//String xmltext = fetchURL(url);
@@ -328,18 +328,30 @@ public class RTL extends AbstractEPGSource implements EPGSource {
String programme_id = p.getString(2);
String quark1 = p.getString(3);
String quark2 = p.getString(4);
- Programme prog = new Programme();
- prog.addTitle(title);
- Date start = parseTime(date, starttime);
- prog.startTime = start;
- prog.channel = channelMap.get(id);
- fetchDetail(prog, date, programme_id);
+ Programme prog = cache.get(programme_id);
+ if (prog == null) {
+ stats.cacheMisses++;
+ prog = new Programme();
+ prog.addTitle(title);
+ prog.startTime = parseTime(date, starttime);
+ prog.channel = channelMap.get(id).getXmltvChannelId();
+ if (fetchDetails) {
+ fetchDetail(prog, date, programme_id);
+ }
+ cache.put(programme_id, prog);
+ } else {
+ stats.cacheHits++;
+ }
result.add(prog);
}
}
return result;
}
+ public void close() throws FileNotFoundException, IOException {
+ super.close();
+ cache.close();
+ }
private Date parseTime(Date date, String time) {
Calendar result = Calendar.getInstance();
result.setTime(date);
@@ -381,12 +393,18 @@ public class RTL extends AbstractEPGSource implements EPGSource {
writer.writeStartElement("tv");
for(Channel c: channels) {c.serialize(writer);}
writer.flush();
- //List<Programme> programmes = rtl.getProgrammes1(channels.subList(6, 9), 0, true);
- List<Programme> programmes = rtl.getProgrammes1(channels, 0, true);
+ List<Programme> programmes = rtl.getProgrammes1(channels.subList(6, 9), 0, true);
+ //List<Programme> programmes = rtl.getProgrammes1(channels, 0, true);
for(Programme p: programmes) {p.serialize(writer);}
writer.writeEndElement();
writer.writeEndDocument();
writer.flush();
+ if (!config.quiet) {
+ EPGSource.Stats stats = rtl.getStats();
+ System.out.println("Number of programmes from cache: " + stats.cacheHits);
+ System.out.println("Number of programmes fetched: " + stats.cacheMisses);
+ System.out.println("Number of fetch errors: " + stats.fetchErrors);
+ }
rtl.close();
} catch (Exception e) {
// TODO Auto-generated catch block
diff --git a/src/main/java/org/vanbest/xmltv/TvGidsProgrammeCache.java b/src/main/java/org/vanbest/xmltv/TvGidsProgrammeCache.java
new file mode 100644
index 0000000..90afc63
--- /dev/null
+++ b/src/main/java/org/vanbest/xmltv/TvGidsProgrammeCache.java
@@ -0,0 +1,74 @@
+package org.vanbest.xmltv;
+
+/*
+ Copyright (c) 2012 Jan-Pascal van Best <janpascal@vanbest.org>
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ The full license text can be found in the LICENSE file.
+*/
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InvalidClassException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.io.FileUtils;
+
+public class TvGidsProgrammeCache {
+ private File cacheFile;
+ private Map<String,TvGidsProgrammeDetails> cache;
+
+ public TvGidsProgrammeCache(File cacheFile) {
+ this.cacheFile = cacheFile;
+ if (cacheFile.canRead()) {
+ try {
+ cache = (Map<String,TvGidsProgrammeDetails>) new ObjectInputStream( new FileInputStream( cacheFile ) ).readObject();
+ } catch (InvalidClassException e) {
+ // TODO Auto-generated catch block
+ cache = new HashMap<String,TvGidsProgrammeDetails>();
+ } catch (ClassNotFoundException e) {
+ // TODO Auto-generated catch block
+ cache = new HashMap<String,TvGidsProgrammeDetails>();
+ } catch (FileNotFoundException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ cache = new HashMap<String,TvGidsProgrammeDetails>();
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ cache = new HashMap<String,TvGidsProgrammeDetails>();
+ }
+ } else {
+ cache = new HashMap<String,TvGidsProgrammeDetails>();
+ }
+ // FileUtils.forceMkdir(root);
+ }
+
+ public TvGidsProgrammeDetails getDetails(String id) {
+ return cache.get(id);
+ }
+
+ public void add(String id, TvGidsProgrammeDetails d) {
+ cache.put(id, d);
+ }
+
+ public void close() throws FileNotFoundException, IOException {
+ new ObjectOutputStream( new FileOutputStream(cacheFile)).writeObject(cache);
+ }
+}
diff --git a/src/main/java/org/vanbest/xmltv/XmlTvWriter.java b/src/main/java/org/vanbest/xmltv/XmlTvWriter.java
index b2cb893..e8ea472 100644
--- a/src/main/java/org/vanbest/xmltv/XmlTvWriter.java
+++ b/src/main/java/org/vanbest/xmltv/XmlTvWriter.java
@@ -59,7 +59,6 @@ public class XmlTvWriter {
for(Channel c: channels) {
if (!c.enabled) continue;
c.serialize(writer);
- writeln();
}
}
--
2.39.5