</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <tasks>
+ <!-- Safety -->
+ <mkdir dir="${project.build.directory}"/>
+ <tstamp>
+ <format property="last.updated" pattern="yyyy-MM-dd hh:mm:ss"/>
+ </tstamp>
+ <echo file="${basedir}/target/filter.properties" message="build.time=${last.updated}"/>
+ </tasks>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
<resources>
<filtering>true</filtering>
</resource>
</resources>
-
+ <filters>
+ <filter>${basedir}/target/filter.properties</filter>
+ </filters>
</build>
</project>
import java.io.InputStreamReader;
import java.net.URL;
import java.util.ArrayList;
+import java.util.List;
import java.util.Set;
import org.vanbest.xmltv.EPGSource.Stats;
public abstract class AbstractEPGSource implements EPGSource {
protected Config config;
- protected TvGidsProgrammeCache cache;
+ protected ProgrammeCache cache;
protected Stats stats = new Stats();
public static final int MAX_FETCH_TRIES=5;
public AbstractEPGSource(Config config) {
this.config = config;
- cache = new TvGidsProgrammeCache(config.cacheFile);
+ cache = new ProgrammeCache(config);
}
- public Set<TvGidsProgramme> getProgrammes(Channel channel, int day, boolean fetchDetails)
+ public List<Programme> getProgrammes(Channel channel, int day, boolean fetchDetails)
throws Exception {
ArrayList<Channel> list = new ArrayList<Channel>(2);
list.add(channel);
}
return buf.toString();
}
-
-
+
+ public void clearCache() {
+ ProgrammeCache cache = new ProgrammeCache(config);
+ cache.clear();
+ cache.close();
+ }
}
\ No newline at end of file
import org.apache.commons.io.FileUtils;
public class Config {
+ //constants
+ public static final int LOG_INFO = 0x0001;
+ public static final int LOG_JSON = 0x0100;
+ private final static int LOG_PROGRAMME_INFO = 0x0200;
+ private final static int CURRENT_FILE_FORMAT=4;
+ public final static int LOG_DEFAULT = LOG_INFO;
+
+ // in config file
public int niceMilliseconds;
public List<Channel> channels;
public Map<String, String> cattrans;
- protected File cacheFile;
public String cacheDbHandle;
public String cacheDbUser;
public String cacheDbPassword;
- boolean quiet = false;
- public int logLevel = LOG_DEFAULT;
- public static final int LOG_INFO = 0x0001;
- public static final int LOG_JSON = 0x0100;
- private static final int LOG_PROGRAMME_INFO = 0x0200;
+ // not stored (yet)
+ public boolean joinKijkwijzerRatings = true;
- public static int LOG_DEFAULT = LOG_INFO;
+ // command-line options
+ boolean quiet = false;
+ public int logLevel = LOG_DEFAULT;
- private final static int CURRENT_FILE_FORMAT=3;
-
String project_version;
+ String build_time;
private Config() {
Properties configProp = new Properties();
e.printStackTrace();
}
project_version=configProp.getProperty("project.version");
+ build_time=configProp.getProperty("build.time");
}
public static Config getDefaultConfig() {
Config result = new Config();
result.channels = new ArrayList<Channel>();
result.cattrans = getDefaultCattrans();
- result.cacheFile = defaultCacheFile();
result.niceMilliseconds = 500;
- result.cacheDbHandle = "jdbc:hsqldb:file:cachedb"; // FIXME in userdir
+ String cachefile = FileUtils.getFile(FileUtils.getUserDirectory(), ".xmltv", "tv_grab_nl_java.cache").getPath();
+ result.setCacheFile(cachefile);
result.cacheDbUser = "SA";
result.cacheDbPassword = "";
return result;
FileUtils.forceMkdir(configFile.getParentFile());
PrintWriter out = new PrintWriter(new OutputStreamWriter( new FileOutputStream( configFile )));
out.println("config-file-format: " + CURRENT_FILE_FORMAT);
- out.println("cache-file: " + escape(cacheFile.getPath()));
+ out.println("cache-db-handle: " + escape(cacheDbHandle));
+ out.println("cache-db-user: " + escape(cacheDbUser));
+ out.println("cache-db-password: " + escape(cacheDbPassword));
out.println("nice-time-milliseconds: " + niceMilliseconds);
for(Channel c: channels) {
// FIXME: handle multiple channels names, icons and urls
break;
case 2:
case 3:
+ case 4:
int source;
if (fileformat==2) {
source = Integer.parseInt(parts.get(1));
fileformat = CURRENT_FILE_FORMAT;
}
} else if (key.equals("cache-file")) {
- result.cacheFile = new File(parts.get(1));
+ if (fileformat<4) {
+ String cacheFile = parts.get(1);
+ result.cacheDbHandle = "jdbc:hsqldb:file:"+cacheFile;
+ result.cacheDbUser = "SA";
+ result.cacheDbPassword = "";
+ } else {
+ System.out.println("Illegal key cache-file in config file!");
+ }
+ } else if (key.equals("cache-db-handle")) {
+ result.cacheDbHandle = parts.get(1);
+ } else if (key.equals("cache-db-user")) {
+ result.cacheDbUser = parts.get(1);
+ } else if (key.equals("cache-db-password")) {
+ result.cacheDbPassword = parts.get(1);
} else if (key.equals("nice-time-milliseconds")) {
result.niceMilliseconds = Integer.parseInt(parts.get(1));
} else {
return (logLevel & LOG_PROGRAMME_INFO) != 0;
}
+ public void setCacheFile(String cacheFile) {
+ cacheDbHandle = "jdbc:hsqldb:file:"+cacheFile;
+ }
+
}
public abstract List<Channel> getChannels();
// Convenience method
- public abstract Set<TvGidsProgramme> getProgrammes(Channel channel, int day,
+ public abstract List<Programme> getProgrammes(Channel channel, int day,
boolean fetchDetails) throws Exception;
- public abstract Set<TvGidsProgramme> getProgrammes(List<Channel> channels,
+ public abstract List<Programme> getProgrammes(List<Channel> channels,
int day, boolean fetchDetails) throws Exception;
public abstract Stats getStats();
-
+
+ public void clearCache();
}
\ No newline at end of file
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import javax.xml.stream.FactoryConfigurationError;
+import javax.xml.stream.XMLEventFactory;
+import javax.xml.stream.XMLOutputFactory;
import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamWriter;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
private PrintStream outputWriter;
private int days = 5;
private int offset = 0;
+ private boolean clearCache = false;
/**
* @param args
*/
this.configFile = defaultConfigFile();
this.outputWriter = System.out;
}
-
+
+ public void showHeader() {
+ System.out.println("tv_grab_nl_java version "+config.project_version + " (built "+config.build_time+")");
+ System.out.println("Copyright (C) 2012 Jan-Pascal van Best <janpascal@vanbest.org>");
+ System.out.println("tv_grab_nl_java comes with ABSOLUTELY NO WARRANTY. It is free software, and you are welcome to redistribute it");
+ System.out.println("under certain conditions; `tv_grab_nl_java --license' for details.");
+ }
public void run() throws FactoryConfigurationError, Exception {
if (!config.quiet) {
- System.out.println("tv_grab_nl_java version "+config.project_version + ", Copyright (C) 2012 Jan-Pascal van Best <janpascal@vanbest.org>");
- System.out.println("tv_grab_nl_java comes with ABSOLUTELY NO WARRANTY. It is free software, and you are welcome to redistribute it");
- System.out.println("under certain conditions; `tv_grab_nl_java --license' for details.");
-
+ showHeader();
System.out.println("Fetching programme data for " + this.days + " starting from day " + this.offset);
int enabledCount = 0;
for(Channel c: config.channels) { if (c.enabled) enabledCount++; }
System.out.println("... from " + enabledCount + " channels");
- System.out.println("... using cache file " + config.cacheFile.getCanonicalPath());
+ System.out.println("... using cache at " + config.cacheDbHandle);
}
- XmlTvWriter writer = new XmlTvWriter(outputWriter, config);
- writer.writeChannels(config.channels);
+ EPGSource gids = new TvGids(config);
+ if (clearCache) gids.clearCache();
+
+ XMLStreamWriter writer = XMLOutputFactory.newInstance().createXMLStreamWriter(outputWriter);
+ writer.writeStartDocument();
+ writer.writeCharacters("\n");
+ writer.writeDTD("<!DOCTYPE tv SYSTEM \"xmltv.dtd\">");
+ writer.writeCharacters("\n");
+ writer.writeStartElement("tv");
+ writer.writeAttribute("generator-info-url","http://github.com/janpascal/tv_grab_nl_java");
+ writer.writeAttribute("source-info-url", "http://tvgids.nl/");
+ writer.writeAttribute("source-info-name", "TvGids.nl");
+ writer.writeAttribute("generator-info-name", "tv_grab_nl_java release "+config.project_version + ", built " + config.build_time);
- EPGSource gids = new TvGidsLegacy(config);
+ for(Channel c: config.channels) if (c.enabled) c.serialize(writer);
for (int day=offset; day<offset+days; day++) {
if (!config.quiet) System.out.print("Fetching information for day " + day);
- Set<TvGidsProgramme> programmes = new HashSet<TvGidsProgramme>();
for(Channel c: config.channels) {
if (!c.enabled) continue;
if (!config.quiet) System.out.print(".");
- Set<TvGidsProgramme> p = gids.getProgrammes(c, day, true);
- writer.writePrograms(p);
+ List<Programme> programmes = gids.getProgrammes(c, day, true);
+ for (Programme p: programmes) p.serialize(writer);
writer.flush();
}
if (!config.quiet) System.out.println();
}
+
+ writer.writeEndElement();
+ writer.writeEndDocument();
+ writer.flush();
+ writer.close();
try {
gids.close();
e.printStackTrace();
}
- writer.close();
if (!config.quiet) {
EPGSource.Stats stats = gids.getStats();
System.out.println("Number of programmes from cache: " + stats.cacheHits);
}
public void configure() throws IOException {
- EPGSource gids = new TvGidsLegacy(config);
+ showHeader();
+
+ EPGSource gids = new TvGids(config);
Set<String> oldChannels = new HashSet<String>();
for (Channel c: config.channels) {
List<Channel> channels = gids.getChannels();
BufferedReader reader = new BufferedReader(new InputStreamReader(System.in));
+
+ System.out.print("Delay between each request to the server (in milliseconds, default="+config.niceMilliseconds+"):");
+ while(true) {
+ String s = reader.readLine().toLowerCase();
+ if (s.isEmpty()) {
+ break;
+ }
+ try {
+ config.niceMilliseconds = Integer.parseInt(s);
+ break;
+ } catch (NumberFormatException e) {
+ System.out.println("\""+s+"\" is not a valid number, please try again");
+ continue;
+ }
+ }
+
+ // TODO configure cache location
+ // public String cacheDbHandle;
+ // public String cacheDbUser;
+ // public String cacheDbPassword;
+
boolean all = false;
boolean none = false;
boolean keep = false;
.hasArg()
.withDescription("Cache file location")
.create())
+ .addOption(OptionBuilder
+ .withLongOpt("clear-cache")
+ .withDescription("Clear cache, remove all cached info")
+ .create())
.addOption(OptionBuilder
.withLongOpt("help")
.withDescription("Show this help")
e.printStackTrace();
}
- if(line.hasOption("license")) {
- showLicense();
- System.exit(0);
+ if(line.hasOption("license")) {
+ showHeader();
+ showLicense();
+ System.exit(0);
}
if(line.hasOption("config-file")) {
configFile = new File(line.getOptionValue("config-file"));
config.quiet = true;
}
if (line.hasOption("description")) {
- System.out.println("tv_grab_nl_java version " + config.project_version);
+ showHeader();
+ System.out.println();
System.out.println("tv_grab_nl_java is a parser for Dutch TV listings using the tvgids.nl JSON interface");
System.exit(0);
}
config.logLevel = Integer.parseInt(line.getOptionValue("log-level"));
}
if (line.hasOption("cache")) {
- config.cacheFile = new File(line.getOptionValue("cache"));
+ config.setCacheFile(line.getOptionValue("cache"));
+ }
+ if (line.hasOption("clear-cache")) {
+ clearCache = true;
}
if (line.hasOption("days")) {
this.days = Integer.parseInt(line.getOptionValue("days"));
previouslyShown.channel = channel;
}
public boolean hasCategory(String category) {
+ if(categories==null) return false;
for(Title t: categories) {
if (t.title.toLowerCase().equals(category)) return true;
}
Rating r = new Rating();
r.system = system;
r.value = value;
+ ratings.add(r);
}
private PreparedStatement getStatement;
private PreparedStatement putStatement;
private PreparedStatement removeStatement;
+ private PreparedStatement clearStatement;
public ProgrammeCache(Config config) {
this.config = config;
getStatement = db.prepareStatement("SELECT programme FROM cache WHERE id=?");
putStatement = db.prepareStatement("INSERT INTO cache VALUES (?,?,?)");
removeStatement = db.prepareStatement("DELETE FROM cache WHERE id=?");
+ clearStatement = db.prepareStatement("DELETE FROM cache");
} catch (SQLException e) {
db = null;
if (!config.quiet) {
try {
stat = db.createStatement();
int count = stat.executeUpdate("DELETE FROM cache WHERE date<CURRENT_DATE - 3 DAY");
- if (!config.quiet) {
+ if (!config.quiet && count>0) {
System.out.println("Purged " + count + " old entries from cache");
}
stat.close();
}
}
- public void close() throws FileNotFoundException, IOException {
+ public void clear() {
+ try {
+ int count = clearStatement.executeUpdate();
+ if (!config.quiet && count>0) {
+ System.out.println("Cleared " + count + " entries from cache");
+ }
+ } catch (SQLException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+
+ public void close() {
cleanup();
if (db != null) {
try {
//prog.endTime = parseTime(date, root.)\r
}\r
\r
- public List<Programme> getProgrammes1(List<Channel> channels, int day,\r
+ @Override\r
+ public List<Programme> getProgrammes(List<Channel> channels, int day,\r
boolean fetchDetails) throws Exception {\r
List<Programme> result = new LinkedList<Programme>();\r
Map<String,Channel> channelMap = new HashMap<String,Channel>();\r
writer.writeStartElement("tv");\r
for(Channel c: channels) {c.serialize(writer);}\r
writer.flush();\r
- List<Programme> programmes = rtl.getProgrammes1(channels.subList(6, 9), 0, true);\r
- //List<Programme> programmes = rtl.getProgrammes1(channels, 0, true);\r
+ List<Programme> programmes = rtl.getProgrammes(channels.subList(6, 9), 0, true);\r
+ //List<Programme> programmes = rtl.getProgrammes(channels, 0, true);\r
for(Programme p: programmes) {p.serialize(writer);}\r
writer.writeEndElement();\r
writer.writeEndDocument();\r
}\r
}\r
\r
- @Override\r
- public Set<TvGidsProgramme> getProgrammes(List<Channel> channels, int day,\r
- boolean fetchDetails) throws Exception {\r
- // TODO Refactor EPGSource to return Programme instead of TvGidsProgramme\r
- return null;\r
- }\r
-\r
-\r
}\r
import javax.xml.stream.XMLStreamWriter;
import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.commons.lang.StringUtils;
import net.sf.ezmorph.MorpherRegistry;
import net.sf.ezmorph.ObjectMorpher;
static String detail_base_url = "http://www.tvgids.nl/json/lists/program.php";
static String html_detail_base_url = "http://www.tvgids.nl/programma/";
- static boolean initialised = false;
-
private static final int MAX_PROGRAMMES_PER_DAY = 9999;
- private ProgrammeCache cache;
-
public TvGids(Config config) {
super(config);
- cache = new ProgrammeCache(config);
- if ( ! initialised ) {
- init();
- initialised = true;
- }
- }
-
- public static void init() {
- String[] formats = {"yyyy-MM-dd HH:mm:ss"};
- MorpherRegistry registry = JSONUtils.getMorpherRegistry();
- registry.registerMorpher( new DateMorpher(formats, new Locale("nl")));
- registry.registerMorpher( new ObjectMorpher() {
- public Object morph(Object value) {
- String s = (String) value;
- return org.apache.commons.lang.StringEscapeUtils.unescapeHtml(s);
- }
- public Class morphsTo() {
- return String.class;
- }
- public boolean supports(Class clazz) {
- return clazz == String.class;
- }
- }, true);
}
public static URL programmeUrl(List<Channel> channels, int day) throws Exception {
JSONObject zender = jsonArray.getJSONObject(i);
//System.out.println( "id: " + zender.getString("id"));
//System.out.println( "name: " + zender.getString("name"));
- //TvGidsChannel c = new TvGidsChannel(zender.getInt("id"), zender.getString("name"), zender.getString("name_short"));
- //c.setIconUrl("http://tvgidsassets.nl/img/channels/53x27/" + c.id + ".png");
int id = zender.getInt("id");
String name = org.apache.commons.lang.StringEscapeUtils.unescapeHtml(zender.getString("name"));
String icon = "http://tvgidsassets.nl/img/channels/53x27/" + id + ".png";
/* (non-Javadoc)
* @see org.vanbest.xmltv.EPGSource#getProgrammes(java.util.List, int, boolean)
*/
- //@Override
- public List<Programme> getProgrammes1(List<Channel> channels, int day, boolean fetchDetails) throws Exception {
+ @Override
+ public List<Programme> getProgrammes(List<Channel> channels, int day, boolean fetchDetails) throws Exception {
List<Programme> result = new ArrayList<Programme>();
URL url = programmeUrl(channels, day);
if (result == null) {
stats.cacheMisses++;
result = new Programme();
+ // Do this here, because we can only add to these fields. Pity if they're updated
+ result.addTitle(programme.getString("titel"));
+ String genre = programme.getString("genre");
+ if (genre != null && !genre.isEmpty()) result.addCategory(config.translateCategory(genre));
+ String kijkwijzer = programme.getString("kijkwijzer");
+ if (kijkwijzer!=null && !kijkwijzer.isEmpty()) {
+ List<String> list = parseKijkwijzer(kijkwijzer);
+ if (config.joinKijkwijzerRatings) {
+ // mythtv doesn't understand multiple <rating> tags
+ result.addRating("kijkwijzer", StringUtils.join(list, ","));
+ } else {
+ for(String rating: list) {
+ result.addRating("kijkwijzer", rating);
+ }
+ }
+ // TODO add icon from HTML detail page
+ }
} else {
+ //System.out.println("From cache: " + programme.getString("titel"));
stats.cacheHits++;
}
- System.out.println(" titel:" + programme.getString("titel"));
+ //System.out.println(" titel:" + programme.getString("titel"));
//System.out.println("datum_start:" + programme.getString("datum_start"));
//System.out.println(" datum_end:" + programme.getString("datum_end"));
//System.out.println(" genre:" + programme.getString("genre"));
SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", new Locale("nl"));
result.startTime = df.parse(programme.getString("datum_start"));
result.endTime = df.parse(programme.getString("datum_end"));
- result.addTitle(programme.getString("titel"));
- String genre = programme.getString("genre");
- if (genre != null && !genre.isEmpty()) result.addCategory(config.translateCategory(genre));
- String kijkwijzer = programme.getString("kijkwijzer");
- if (kijkwijzer!=null && !kijkwijzer.isEmpty()) {
- List<String> list = parseKijkwijzer(kijkwijzer);
- for(String s: list) {
- result.addRating("kijkwijzer", s);
- // TODO add icon from HTML detail page
- }
-
- }
// TODO other fields
if (fetchDetails && !cached) {
+ // TODO also read details if those have not been cached
fillDetails(id, result);
}
if (!cached) {
replaceAll("<em>", " ").
replaceAll("</em>", " ").
trim();
+ if (value.isEmpty()) continue ;
result.addDescription(value);
} else if (key.equals("presentatie")) {
String[] parts = value.split(",");
// result.details.fixup(result, config.quiet);
}
- @Override
- public Set<TvGidsProgramme> getProgrammes(List<Channel> channels, int day,
- boolean fetchDetails) throws Exception {
- // TODO Auto-generated method stub
- // dummy, wait for superclass and interface to be generalised
- return null;
- }
-
-
/**
* @param args
*/
List<Channel> my_channels = channels.subList(0,15);
for(Channel c: my_channels) {c.serialize(writer);}
writer.flush();
- List<Programme> programmes = gids.getProgrammes1(my_channels, 2, true);
+ List<Programme> programmes = gids.getProgrammes(my_channels, 2, true);
for(Programme p: programmes) {p.serialize(writer);}
writer.writeEndElement();
writer.writeEndDocument();
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.vanbest.xmltv.EPGSource.Stats;
+
+
import net.sf.ezmorph.MorpherRegistry;
import net.sf.ezmorph.ObjectMorpher;
import net.sf.ezmorph.object.DateMorpher;
import net.sf.json.JSONObject;
import net.sf.json.util.JSONUtils;
-public class TvGidsLegacy extends AbstractEPGSource implements EPGSource {
+public class TvGidsLegacy {
static String channels_url="http://www.tvgids.nl/json/lists/channels.php";
static String programme_base_url="http://www.tvgids.nl/json/lists/programs.php";
static String detail_base_url = "http://www.tvgids.nl/json/lists/program.php";
static String html_detail_base_url = "http://www.tvgids.nl/programma/";
+ public static final int MAX_FETCH_TRIES=5;
+
static boolean initialised = false;
-
+ private Config config;
+ protected Stats stats = new Stats();
+ private TvGidsProgrammeCache cache;
+
public TvGidsLegacy(Config config) {
- super(config);
this.config = config;
+ //this.cache = new TvGidsProgrammeCache(config.cacheFile);
+ this.cache = new TvGidsProgrammeCache(new File("tv_grab_nl_java.cache"));
if ( ! initialised ) {
init();
initialised = true;
/* (non-Javadoc)
* @see org.vanbest.xmltv.EPGSource#getChannels()
*/
- @Override
public List<Channel> getChannels() {
List<Channel> result = new ArrayList<Channel>(10);
URL url = null;
/* (non-Javadoc)
* @see org.vanbest.xmltv.EPGSource#getProgrammes(java.util.List, int, boolean)
*/
- @Override
- public Set<TvGidsProgramme> getProgrammes(List<Channel> channels, int day, boolean fetchDetails) throws Exception {
+ public Set<TvGidsProgramme> getProgrammes1(List<Channel> channels, int day, boolean fetchDetails) throws Exception {
Set<TvGidsProgramme> result = new HashSet<TvGidsProgramme>();
URL url = programmeUrl(channels, day);
return p;
}
+ protected String fetchURL(URL url) throws Exception {
+ Thread.sleep(config.niceMilliseconds);
+ StringBuffer buf = new StringBuffer();
+ boolean done = false;
+ IOException finalException = null;
+ for(int count = 0; count<MAX_FETCH_TRIES && !done; count++) {
+ try {
+ BufferedReader reader = new BufferedReader( new InputStreamReader( url.openStream()));
+ String s;
+ while ((s = reader.readLine()) != null) buf.append(s);
+ done = true;
+ } catch (IOException e) {
+ if (!config.quiet) {
+ System.out.println("Error fetching from url " + url + ", count="+count);
+ }
+ finalException = e;
+ }
+ }
+ if (!done) {
+ stats.fetchErrors++;
+ throw new Exception("Error getting program data from url " + url, finalException);
+ }
+ return buf.toString();
+ }
+
private JSONObject fetchJSON(URL url) throws Exception {
String json = fetchURL(url);
if (config.logJSON()) System.out.println(json);
import javax.xml.stream.XMLEventFactory;
import javax.xml.stream.XMLStreamException;
+
public class XmlTvWriter {
private XMLStreamWriter writer;
project.version=${project.version}
project.name=${project.name}
-project.description=${project.description}
\ No newline at end of file
+project.description=${project.description}
+build.time=${build.time}
\ No newline at end of file
projects / tv_grab_nl_java / commitdiff