/rtl.xml
/target
/testdb.*
+/tv_grab_nl_java.db.properties
public abstract class AbstractEPGSource implements EPGSource {
protected Config config;
- protected ProgrammeCache cache;
+ protected TvGidsProgrammeCache cache;
protected Stats stats = new Stats();
public static final int MAX_FETCH_TRIES=5;
public AbstractEPGSource(Config config) {
this.config = config;
- cache = new ProgrammeCache(config.cacheFile);
+ cache = new TvGidsProgrammeCache(config.cacheFile);
}
public Set<TvGidsProgramme> getProgrammes(Channel channel, int day, boolean fetchDetails)
package org.vanbest.xmltv;\r
\r
import java.util.ArrayList;\r
+import java.util.HashMap;\r
import java.util.HashSet;\r
import java.util.List;\r
+import java.util.Map;\r
import java.util.Set;\r
\r
import javax.xml.stream.XMLStreamException;\r
\r
public final static int CHANNEL_SOURCE_TVGIDS=1;\r
public final static int CHANNEL_SOURCE_RTL=2;\r
+ private final static String[] CHANNEL_SOURCE_NAMES={"tvgids.nl", "rtl.nl"};\r
+ private static Map<String,Integer> channelSourceNameMap = new HashMap<String,Integer>();\r
+\r
\r
protected Channel(int source, String id) {\r
this.id = id;\r
public String getXmltvChannelId() {\r
switch (source) {\r
case CHANNEL_SOURCE_TVGIDS:\r
- return id+".tvgids.nl";\r
case CHANNEL_SOURCE_RTL:\r
- return id;\r
default:\r
- return id;\r
+ return id+"."+getSourceName();\r
+ }\r
+ }\r
+ \r
+ public static String getChannelSourceName(int id) {\r
+ return CHANNEL_SOURCE_NAMES[id-1];\r
+ }\r
+\r
+ public String getSourceName() {\r
+ return getChannelSourceName(source);\r
+ }\r
+ \r
+ public static int getChannelSourceId(String name) {\r
+ if (channelSourceNameMap.isEmpty()) {\r
+ int i=1;\r
+ for (String s: CHANNEL_SOURCE_NAMES) {\r
+ channelSourceNameMap.put(s, i);\r
+ i++;\r
+ }\r
}\r
+ return channelSourceNameMap.get(name);\r
+ \r
}\r
\r
public void serialize(XMLStreamWriter writer) throws XMLStreamException {\r
public List<Channel> channels;
public Map<String, String> cattrans;
protected File cacheFile;
+ public String cacheDbHandle;
+ public String cacheDbUser;
+ public String cacheDbPassword;
boolean quiet = false;
public int logLevel = LOG_DEFAULT;
public static int LOG_DEFAULT = LOG_INFO;
- private final static int CURRENT_FILE_FORMAT=2;
+ private final static int CURRENT_FILE_FORMAT=3;
String project_version;
result.cattrans = getDefaultCattrans();
result.cacheFile = defaultCacheFile();
result.niceMilliseconds = 500;
+ result.cacheDbHandle = "jdbc:hsqldb:file:cachedb"; // FIXME in userdir
+ result.cacheDbUser = "SA";
+ result.cacheDbPassword = "";
return result;
}
out.println("nice-time-milliseconds: " + niceMilliseconds);
for(Channel c: channels) {
// FIXME: handle multiple channels names, icons and urls
- out.print("channel: " + c.source +
- ":" + c.id +
+ out.print("channel: " + c.getSourceName() +
+ ": " + c.id +
": " + (c.enabled?"enabled":"disabled") +
": " + escape(c.defaultName()));
if (!c.icons.isEmpty()) {
}
break;
case 2:
- c = Channel.getChannel(Integer.parseInt(parts.get(1)), parts.get(2), parts.get(4));
+ case 3:
+ int source;
+ if (fileformat==2) {
+ source = Integer.parseInt(parts.get(1));
+ } else {
+ source = Channel.getChannelSourceId(parts.get(1));
+ }
+ c = Channel.getChannel(source, parts.get(2), parts.get(4));
if (parts.size()>5) {
c.addIcon(parts.get(5));
}
Set<String> oldChannels = new HashSet<String>();
for (Channel c: config.channels) {
- oldChannels.add(c.id);
+ if (c.enabled) {
+ oldChannels.add(c.source+"::"+c.id);
+ }
}
List<Channel> channels = gids.getChannels();
boolean none = false;
boolean keep = false;
for (Channel c: channels) {
- boolean selected = oldChannels.contains(c.id);
+ boolean selected = oldChannels.contains(c.source+"::"+c.id);
System.out.print("add channel " + c.id + " (" + c.defaultName() + ") [[y]es,[n]o,[a]ll,[none],[k]eep selection (default=" + (selected?"yes":"no") + ")] ");
if (keep) {
c.enabled = selected;
package org.vanbest.xmltv;
+import java.io.Serializable;
import java.net.URL;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamWriter;
-public class Programme {
- class Title {
+public class Programme implements Serializable {
+ class Title implements Serializable {
String title;
String lang;
public Title(String title, String lang) {
this.lang = lang;
}
}
- class Actor {
+ class Actor implements Serializable {
String name;
String role;
}
- class Credits {
+ class Credits implements Serializable {
List<String> directors;
List<Actor> actors;
List<String> writers;
List<String> commentators;
List<String> guests;
}
- class Length {
+ class Length implements Serializable {
TimeUnit unit;
int count;
}
- class Icon {
+ class Icon implements Serializable {
URL url;
int width;
int height;
}
- class Episode {
+ class Episode implements Serializable {
String episode;
String system; // onscreen or xmltv_ns
}
- class Video {
+ class Video implements Serializable {
boolean present;
boolean colour;
String aspect; // eg. 16:9, 4:3
String quality; // eg. 'HDTV', '800x600'.
}
- class Audio {
+ class Audio implements Serializable {
boolean present;
String stereo; // 'mono','stereo','dolby','dolby digital','bilingual' or 'surround'.
}
- class Subtitle {
+ class Subtitle implements Serializable {
String type; // teletext | onscreen | deaf-signed
Title language;
}
public Date vpsStart;
public String showview;
public String videoplus;
- public Channel channel; // required
+ public String channel; // required xmltvid of the associated channel
public String clumpidx;
public List<Title> titles; // at least one
writer.writeStartElement("programme");
if(startTime != null) writer.writeAttribute("start", df.format(startTime));
if(endTime != null) writer.writeAttribute("stop", df.format(endTime));
- if(channel != null) writer.writeAttribute("channel", ""+channel.id);
+ if(channel != null) writer.writeAttribute("channel", ""+channel);
writeTitleList(titles,"title",writer);
writeTitleList(secondaryTitles,"sub-title", writer);
if(credits != null) {
import java.io.InvalidClassException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
+import java.sql.Connection;
+import java.sql.Date;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
import java.util.HashMap;
import java.util.Map;
+import java.util.Properties;
import org.apache.commons.io.FileUtils;
public class ProgrammeCache {
- private File cacheFile;
- private Map<String,TvGidsProgrammeDetails> cache;
+ private Connection db;
+ private Config config;
+ private PreparedStatement getStatement;
+ private PreparedStatement putStatement;
- public ProgrammeCache(File cacheFile) {
- this.cacheFile = cacheFile;
- if (cacheFile.canRead()) {
- try {
- cache = (Map<String,TvGidsProgrammeDetails>) new ObjectInputStream( new FileInputStream( cacheFile ) ).readObject();
- } catch (InvalidClassException e) {
- // TODO Auto-generated catch block
- cache = new HashMap<String,TvGidsProgrammeDetails>();
- } catch (ClassNotFoundException e) {
- // TODO Auto-generated catch block
- cache = new HashMap<String,TvGidsProgrammeDetails>();
- } catch (FileNotFoundException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- cache = new HashMap<String,TvGidsProgrammeDetails>();
- } catch (IOException e) {
- // TODO Auto-generated catch block
+ public ProgrammeCache(Config config) {
+ this.config = config;
+ try {
+ db = DriverManager.getConnection(config.cacheDbHandle, config.cacheDbUser, config.cacheDbPassword);
+ Statement stat = db.createStatement();
+ stat.execute("CREATE TABLE IF NOT EXISTS cache (id VARCHAR(64) PRIMARY KEY, date DATE, programme OTHER)");
+ stat.close();
+
+ getStatement = db.prepareStatement("SELECT programme FROM cache WHERE id=?");
+ putStatement = db.prepareStatement("INSERT INTO cache VALUES (?,?,?)");
+ } catch (SQLException e) {
+ db = null;
+ if (!config.quiet) {
+ System.out.println("Unable to open cache database, proceeding without cache");
e.printStackTrace();
- cache = new HashMap<String,TvGidsProgrammeDetails>();
}
- } else {
- cache = new HashMap<String,TvGidsProgrammeDetails>();
}
- // FileUtils.forceMkdir(root);
}
- public TvGidsProgrammeDetails getDetails(String id) {
- return cache.get(id);
+ public Programme get(String id) {
+ if (db==null) return null;
+ try {
+ getStatement.setString(1, id);
+ ResultSet r = getStatement.executeQuery();
+ if (!r.next()) return null; // not found
+ return (Programme) r.getObject("programme");
+ } catch (SQLException e) {
+ if (!config.quiet) {
+ e.printStackTrace();
+ }
+ return null;
+ }
}
- public void add(String id, TvGidsProgrammeDetails d) {
- cache.put(id, d);
+ public void put(String id, Programme prog) {
+ if (db == null) return;
+ try {
+ putStatement.setString(1, id);
+ putStatement.setDate(2, new java.sql.Date(prog.startTime.getTime()));
+ putStatement.setObject(3, prog);
+ int count = putStatement.executeUpdate();
+ if (count!=1 && !config.quiet) {
+ System.out.println("Weird, cache database update statement affected " + count + " rows");
+ }
+ } catch (SQLException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
}
-
+
+ public void cleanup() {
+ Statement stat;
+ try {
+ stat = db.createStatement();
+ int count = stat.executeUpdate("DELETE FROM cache WHERE date<CURRENT_DATE - 3 DAY");
+ if (!config.quiet) {
+ System.out.println("Purged " + count + " old entries from cache");
+ }
+ stat.close();
+ } catch (SQLException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+
public void close() throws FileNotFoundException, IOException {
- new ObjectOutputStream( new FileOutputStream(cacheFile)).writeObject(cache);
+ cleanup();
+ if (db != null) {
+ try {
+ getStatement.close();
+ putStatement.close();
+ db.close();
+ } catch (SQLException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
}
}
private static final String detail_url="http://www.rtl.nl/active/epg_data/uitzending_data/";\r
private static final String icon_url="http://www.rtl.nl/service/gids/components/vaste_componenten/";\r
private static final String xmltv_channel_suffix = ".rtl.nl";\r
- private static final int MAX_PROGRAMMES_PER_DAY = 200000;\r
+ private static final int MAX_PROGRAMMES_PER_DAY = 20;\r
\r
private Connection db;\r
\r
"site_path", "wwwadres", "presentatie", "omroep", "eindtijd", "inhoud", "tt_inhoud", "alginhoud", "afl_titel", "kijkwijzer" };\r
\r
Map<String,Integer> xmlKeyMap = new HashMap<String,Integer>();\r
+ private ProgrammeCache cache;\r
\r
class RTLException extends Exception {\r
public RTLException(String s) {\r
\r
public RTL(Config config, boolean useDB) {\r
super(config);\r
+ cache = new ProgrammeCache(config);\r
try {\r
if (useDB) {\r
Properties dbProp = new Properties();\r
} else if (tag.equals("omroep")) {\r
} else if (tag.equals("kijkwijzer")) {\r
} else if (tag.equals("presentatie")) {\r
- // A\r
- // A en B\r
- // A, B, C en D\r
+ // A; A en B; A, B, C en D\r
String[] presentatoren = e.getTextContent().split(", | en ");\r
for(String pres:presentatoren) {\r
prog.addPresenter(pres);\r
List<Programme> result = new LinkedList<Programme>();\r
Map<String,Channel> channelMap = new HashMap<String,Channel>();\r
for(Channel c: channels) {\r
- if (c.enabled) channelMap.put(c.id, c);\r
+ if (c.enabled && c.source==Channel.CHANNEL_SOURCE_RTL) channelMap.put(c.id, c);\r
}\r
URL url = programmeUrl(day);\r
//String xmltext = fetchURL(url);\r
String programme_id = p.getString(2);\r
String quark1 = p.getString(3);\r
String quark2 = p.getString(4);\r
- Programme prog = new Programme();\r
- prog.addTitle(title);\r
- Date start = parseTime(date, starttime);\r
- prog.startTime = start;\r
- prog.channel = channelMap.get(id);\r
- fetchDetail(prog, date, programme_id);\r
+ Programme prog = cache.get(programme_id);\r
+ if (prog == null) {\r
+ stats.cacheMisses++;\r
+ prog = new Programme();\r
+ prog.addTitle(title);\r
+ prog.startTime = parseTime(date, starttime);\r
+ prog.channel = channelMap.get(id).getXmltvChannelId();\r
+ if (fetchDetails) {\r
+ fetchDetail(prog, date, programme_id);\r
+ }\r
+ cache.put(programme_id, prog);\r
+ } else {\r
+ stats.cacheHits++;\r
+ }\r
result.add(prog);\r
}\r
}\r
return result;\r
}\r
\r
+ public void close() throws FileNotFoundException, IOException {\r
+ super.close();\r
+ cache.close();\r
+ }\r
private Date parseTime(Date date, String time) {\r
Calendar result = Calendar.getInstance();\r
result.setTime(date);\r
writer.writeStartElement("tv");\r
for(Channel c: channels) {c.serialize(writer);}\r
writer.flush();\r
-
//List<Programme> programmes = rtl.getProgrammes1(channels.subList(6, 9), 0, true);
\r- List<Programme> programmes = rtl.getProgrammes1(channels, 0, true);\r
+ List<Programme> programmes = rtl.getProgrammes1(channels.subList(6, 9), 0, true);\r
+
//List<Programme> programmes = rtl.getProgrammes1(channels, 0, true);
\r for(Programme p: programmes) {p.serialize(writer);}\r
writer.writeEndElement();\r
writer.writeEndDocument();\r
writer.flush();\r
+ if (!config.quiet) {\r
+ EPGSource.Stats stats = rtl.getStats();\r
+ System.out.println("Number of programmes from cache: " + stats.cacheHits);\r
+ System.out.println("Number of programmes fetched: " + stats.cacheMisses);\r
+ System.out.println("Number of fetch errors: " + stats.fetchErrors);\r
+ }\r
rtl.close();\r
} catch (Exception e) {\r
// TODO Auto-generated catch block\r
--- /dev/null
+package org.vanbest.xmltv;
+
+/*
+ Copyright (c) 2012 Jan-Pascal van Best <janpascal@vanbest.org>
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ The full license text can be found in the LICENSE file.
+*/
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InvalidClassException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.io.FileUtils;
+
+public class TvGidsProgrammeCache {
+ private File cacheFile;
+ private Map<String,TvGidsProgrammeDetails> cache;
+
+ public TvGidsProgrammeCache(File cacheFile) {
+ this.cacheFile = cacheFile;
+ if (cacheFile.canRead()) {
+ try {
+ cache = (Map<String,TvGidsProgrammeDetails>) new ObjectInputStream( new FileInputStream( cacheFile ) ).readObject();
+ } catch (InvalidClassException e) {
+ // TODO Auto-generated catch block
+ cache = new HashMap<String,TvGidsProgrammeDetails>();
+ } catch (ClassNotFoundException e) {
+ // TODO Auto-generated catch block
+ cache = new HashMap<String,TvGidsProgrammeDetails>();
+ } catch (FileNotFoundException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ cache = new HashMap<String,TvGidsProgrammeDetails>();
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ cache = new HashMap<String,TvGidsProgrammeDetails>();
+ }
+ } else {
+ cache = new HashMap<String,TvGidsProgrammeDetails>();
+ }
+ // FileUtils.forceMkdir(root);
+ }
+
+ public TvGidsProgrammeDetails getDetails(String id) {
+ return cache.get(id);
+ }
+
+ public void add(String id, TvGidsProgrammeDetails d) {
+ cache.put(id, d);
+ }
+
+ public void close() throws FileNotFoundException, IOException {
+ new ObjectOutputStream( new FileOutputStream(cacheFile)).writeObject(cache);
+ }
+}
for(Channel c: channels) {
if (!c.enabled) continue;
c.serialize(writer);
- writeln();
}
}
projects / tv_grab_nl_java / commitdiff