From: Jan-Pascal van Best Date: Mon, 29 Feb 2016 20:55:26 +0000 (+0100) Subject: Fix for new Horizon URL; better handling of 'Zender verstrekt geen informatie' progra... X-Git-Tag: v1.6.6~1 X-Git-Url: http://www.vanbest.org/gitweb/?a=commitdiff_plain;h=75d068722fc5c3624d47b5f526583e37867f9a10;p=tv_grab_nl_java Fix for new Horizon URL; better handling of 'Zender verstrekt geen informatie' programme items --- diff --git a/src/main/java/org/vanbest/xmltv/Horizon.java b/src/main/java/org/vanbest/xmltv/Horizon.java index e3bd6fe..1594fc9 100644 --- a/src/main/java/org/vanbest/xmltv/Horizon.java +++ b/src/main/java/org/vanbest/xmltv/Horizon.java @@ -35,87 +35,87 @@ import net.sf.json.JSONObject; public class Horizon extends AbstractEPGSource implements EPGSource { - static String channels_url = "https://www.horizon.tv/oesp/api/NL/nld/web/channels/"; - static String listings_url = "https://www.horizon.tv/oesp/api/NL/nld/web/listings"; + static String channels_url = "https://web-api-salt.horizon.tv/oesp/api/NL/nld/web/channels/"; + static String listings_url = "https://web-api-salt.horizon.tv/oesp/api/NL/nld/web/listings"; - private static final int MAX_DAYS_AHEAD_SUPPORTED_BY_HORIZON = 7; + private static final int MAX_DAYS_AHEAD_SUPPORTED_BY_HORIZON = 7; public final static String NAME="horizon.tv"; - static Logger logger = Logger.getLogger(Horizon.class); + static Logger logger = Logger.getLogger(Horizon.class); - public Horizon(Config config) { - super(config); - } + public Horizon(Config config) { + super(config); + } - public String getName() { - return NAME; - } + public String getName() { + return NAME; + } - public static URL programmeUrl(Channel channel, int day) - throws Exception { - StringBuilder s = new StringBuilder(listings_url); - s.append("?byStationId="); - s.append(channel.id); - s.append("&sort=startTime&range=1-100"); - Calendar startTime=Calendar.getInstance(); - startTime.set(Calendar.HOUR_OF_DAY, 0); - startTime.set(Calendar.MINUTE, 0); - startTime.set(Calendar.SECOND, 0); - startTime.set(Calendar.MILLISECOND, 0); - startTime.add(Calendar.DAY_OF_MONTH, day); - Calendar endTime = (Calendar) startTime.clone(); - endTime.add(Calendar.DAY_OF_MONTH, 1); - s.append("&byStartTime="); - s.append(startTime.getTimeInMillis()); - s.append("~"); - s.append(endTime.getTimeInMillis()); + public static URL programmeUrl(Channel channel, int day) + throws Exception { + StringBuilder s = new StringBuilder(listings_url); + s.append("?byStationId="); + s.append(channel.id); + s.append("&sort=startTime&range=1-100"); + Calendar startTime=Calendar.getInstance(); + startTime.set(Calendar.HOUR_OF_DAY, 0); + startTime.set(Calendar.MINUTE, 0); + startTime.set(Calendar.SECOND, 0); + startTime.set(Calendar.MILLISECOND, 0); + startTime.add(Calendar.DAY_OF_MONTH, day); + Calendar endTime = (Calendar) startTime.clone(); + endTime.add(Calendar.DAY_OF_MONTH, 1); + s.append("&byStartTime="); + s.append(startTime.getTimeInMillis()); + s.append("~"); + s.append(endTime.getTimeInMillis()); - return new URL(s.toString()); - } + return new URL(s.toString()); + } - /* - * (non-Javadoc) - * - * @see org.vanbest.xmltv.EPGSource#getChannels() - */ - @Override - public List getChannels() { - List result = new ArrayList(10); - URL url = null; - try { - url = new URL(channels_url); - } catch (MalformedURLException e) { - logger.error("Exception creating horizon channel list url", e); - } - JSONObject channels; - try { - channels = fetchJSON(url, "UTF8"); - } catch (Exception e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); - return result; - } - logger.debug("horizon channels json: " + channels.toString()); + /* + * (non-Javadoc) + * + * @see org.vanbest.xmltv.EPGSource#getChannels() + */ + @Override + public List getChannels() { + List result = new ArrayList(10); + URL url = null; + try { + url = new URL(channels_url); + } catch (MalformedURLException e) { + logger.error("Exception creating horizon channel list url", e); + } + JSONObject channels; + try { + channels = fetchJSON(url, "UTF8"); + } catch (Exception e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + return result; + } + logger.debug("horizon channels json: " + channels.toString()); - int numChannels = Integer.parseInt(channels.getString("totalResults")); - JSONArray jsonArray = channels.getJSONArray("channels"); - for (int i = 0; i < jsonArray.size(); i++) { - JSONObject zender = jsonArray.getJSONObject(i); - - // System.out.println( "id: " + zender.getString("id")); - // System.out.println( "name: " + zender.getString("name")); - JSONArray stationSchedules=zender.getJSONArray("stationSchedules"); - assert(1 == stationSchedules.size()); - - JSONObject firstSchedule = stationSchedules.getJSONObject(0); - JSONObject station = firstSchedule.getJSONObject("station"); - logger.debug("firstschedule: " + firstSchedule.toString()); - long horizonId = station.getLong("id"); - String name = station.getString("title"); - // Use the largest available station logo as icon - JSONArray images = station.getJSONArray("images"); - String icon = ""; + int numChannels = Integer.parseInt(channels.getString("totalResults")); + JSONArray jsonArray = channels.getJSONArray("channels"); + for (int i = 0; i < jsonArray.size(); i++) { + JSONObject zender = jsonArray.getJSONObject(i); + + // System.out.println( "id: " + zender.getString("id")); + // System.out.println( "name: " + zender.getString("name")); + JSONArray stationSchedules=zender.getJSONArray("stationSchedules"); + assert(1 == stationSchedules.size()); + + JSONObject firstSchedule = stationSchedules.getJSONObject(0); + JSONObject station = firstSchedule.getJSONObject("station"); + logger.debug("firstschedule: " + firstSchedule.toString()); + long horizonId = station.getLong("id"); + String name = station.getString("title"); + // Use the largest available station logo as icon + JSONArray images = station.getJSONArray("images"); + String icon = ""; int maxSize = 0; for( int j=0; j getProgrammes(List channels, int day) - throws Exception { - List result = new ArrayList(); - if (day > MAX_DAYS_AHEAD_SUPPORTED_BY_HORIZON) { - return result; // empty list - } - for (Channel c : channels) { - URL url = programmeUrl(c, day); - logger.debug("Programme url:" + url); - JSONObject jsonObject = fetchJSON(url, "UTF-8"); - logger.debug(jsonObject.toString()); - JSONArray listings = jsonObject.getJSONArray("listings"); - for (int i = 0; i < listings.size(); i++) { - JSONObject listing = listings.getJSONObject(i); - Programme p = programmeFromJSON(listing, - config.fetchDetails); - p.channel = c.getXmltvChannelId(); - result.add(p); - } - } + /* + * (non-Javadoc) + * + * @see org.vanbest.xmltv.EPGSource#getProgrammes(java.util.List, int, + * boolean) + */ + @Override + public List getProgrammes(List channels, int day) + throws Exception { + List result = new ArrayList(); + if (day > MAX_DAYS_AHEAD_SUPPORTED_BY_HORIZON) { + return result; // empty list + } + for (Channel c : channels) { + URL url = programmeUrl(c, day); + logger.debug("Programme url:" + url); + JSONObject jsonObject = fetchJSON(url, "UTF-8"); + logger.debug(jsonObject.toString()); + JSONArray listings = jsonObject.getJSONArray("listings"); + for (int i = 0; i < listings.size(); i++) { + JSONObject listing = listings.getJSONObject(i); + Programme p = programmeFromJSON(listing, + config.fetchDetails); + if (p != null) { + p.channel = c.getXmltvChannelId(); + result.add(p); + } + } + } - return result; - } + return result; + } - /* - * {"id":"crid:~~2F~~2Feventis.nl~~2F00000000-0000-1000-0004-00000189B7F0-imi:001017B90000FCD2", - * "countryCode":"NL", - * "languageCode":"nld", - * "deviceCode":"web", - * "locationId":"15332128", - * "startTime":1362399000000, - * "endTime":1362399300000, - * "stationId":"28070126", - * "imi":"imi:001017B90000FCD2", - * "program":{"id":"crid:~~2F~~2Feventis.nl~~2F00000000-0000-1000-0004-00000189B7F0", - * "mediaGroupId":"crid:~~2F~~2Feventis.nl~~2F00000000-0000-1000-0008-000000007784", - * "title":"NOS Sportjournaal (Ned1) - 13:10", - * "secondaryTitle":"NOS Sportjournaal", - * "description":"Aandacht voor het actuele sportnieuws.", - * "shortDescription":"Aandacht voor het actuele sportnieuws.", - * "longDescription":"Aandacht voor het actuele sportnieuws.", - * "countryCode":"NL", - * "languageCode":"nld", - * "deviceCode":"web", - * "medium":"TV", - * "categories":[{"id":"13946291","title":"sports","scheme":"urn:tva:metadata:cs:UPCEventGenreCS:2009"}, - * {"id":"13946352","title":"sports/sports","scheme":"urn:tva:metadata:cs:UPCEventGenreCS:2009"}], - * "mediaType":"Episode", - * "isAdult":false, - * "seriesEpisodeNumber":"50108216", - * "cast":[], - * "directors":[], - * "videos":[], - * "images":[{"assetType":"tva-boxcover","width":180,"height":260,"url":"https://www.horizon.tv/static-images/926/511/36654624.p.jpg"}, - * {"assetType":"boxart-xlarge","width":210,"height":303,"url":"https://www.horizon.tv/static-images/926/511/36654624.p_210x303_34273348255.jpg"}, - * {"assetType":"boxart-large","width":180,"height":260,"url":"https://www.horizon.tv/static-images/926/511/36654624.p_180x260_34273348256.jpg"}, - * {"assetType":"boxart-medium","width":110,"height":159,"url":"https://www.horizon.tv/static-images/926/511/36654624.p_110x159_34273348257.jpg"}, - * {"assetType":"boxart-small","width":75,"height":108,"url":"https://www.horizon.tv/static-images/926/511/36654624.p_75x108_34273348258.jpg"}]}} - */ - private Programme programmeFromJSON(JSONObject json, - boolean fetchDetails) throws Exception { - String id = json.getString("id"); - Programme result = cache.get(getName(), id); - boolean cached = (result != null); - boolean doNotCache = false; - if (result == null) { - stats.cacheMisses++; - result = new Programme(); - result.startTime = new Date(json.getLong("startTime")); - result.endTime = new Date(json.getLong("endTime")); - JSONObject prog = json.getJSONObject("program"); - // Sometimes the JSON doesnt contains a prog item - if (prog==null || prog.isNullObject()) return result; - String title = null; - if (prog.has("title")){ - title = prog.getString("title"); - } - String secondary = null; - if (prog.has("secondaryTitle")) { + /* + * {"id":"crid:~~2F~~2Feventis.nl~~2F00000000-0000-1000-0004-00000189B7F0-imi:001017B90000FCD2", + * "countryCode":"NL", + * "languageCode":"nld", + * "deviceCode":"web", + * "locationId":"15332128", + * "startTime":1362399000000, + * "endTime":1362399300000, + * "stationId":"28070126", + * "imi":"imi:001017B90000FCD2", + * "program":{"id":"crid:~~2F~~2Feventis.nl~~2F00000000-0000-1000-0004-00000189B7F0", + * "mediaGroupId":"crid:~~2F~~2Feventis.nl~~2F00000000-0000-1000-0008-000000007784", + * "title":"NOS Sportjournaal (Ned1) - 13:10", + * "secondaryTitle":"NOS Sportjournaal", + * "description":"Aandacht voor het actuele sportnieuws.", + * "shortDescription":"Aandacht voor het actuele sportnieuws.", + * "longDescription":"Aandacht voor het actuele sportnieuws.", + * "countryCode":"NL", + * "languageCode":"nld", + * "deviceCode":"web", + * "medium":"TV", + * "categories":[{"id":"13946291","title":"sports","scheme":"urn:tva:metadata:cs:UPCEventGenreCS:2009"}, + * {"id":"13946352","title":"sports/sports","scheme":"urn:tva:metadata:cs:UPCEventGenreCS:2009"}], + * "mediaType":"Episode", + * "isAdult":false, + * "seriesEpisodeNumber":"50108216", + * "cast":[], + * "directors":[], + * "videos":[], + * "images":[{"assetType":"tva-boxcover","width":180,"height":260,"url":"https://www.horizon.tv/static-images/926/511/36654624.p.jpg"}, + * {"assetType":"boxart-xlarge","width":210,"height":303,"url":"https://www.horizon.tv/static-images/926/511/36654624.p_210x303_34273348255.jpg"}, + * {"assetType":"boxart-large","width":180,"height":260,"url":"https://www.horizon.tv/static-images/926/511/36654624.p_180x260_34273348256.jpg"}, + * {"assetType":"boxart-medium","width":110,"height":159,"url":"https://www.horizon.tv/static-images/926/511/36654624.p_110x159_34273348257.jpg"}, + * {"assetType":"boxart-small","width":75,"height":108,"url":"https://www.horizon.tv/static-images/926/511/36654624.p_75x108_34273348258.jpg"}]}} + */ + private Programme programmeFromJSON(JSONObject json, + boolean fetchDetails) throws Exception { + String id = json.getString("id"); + Programme result = cache.get(getName(), id); + boolean cached = (result != null); + boolean doNotCache = false; + if (result == null) { + stats.cacheMisses++; + result = new Programme(); + result.startTime = new Date(json.getLong("startTime")); + result.endTime = new Date(json.getLong("endTime")); + JSONObject prog = json.getJSONObject("program"); + // Sometimes the JSON doesnt contains a prog item + if (prog==null || prog.isNullObject()) return null; + String title = null; + if (prog.has("title")){ + title = prog.getString("title"); + if (title.contains("Zender verstrekt geen informatie")) { + // To be filled in later + return null; + } + } + String secondary = null; + if (prog.has("secondaryTitle")) { secondary = prog.getString("secondaryTitle"); if (secondary.contains("Zal snel bekend")) secondary = null; - } - if (title != null && secondary!=null && title.contains(secondary)) { - title=secondary; - secondary=null; - } + } + if (title != null && secondary!=null && title.contains(secondary)) { + title=secondary; + secondary=null; + } if (title != null && !title.isEmpty()) { result.addTitle(title); if (secondary!=null && !secondary.isEmpty()) { @@ -241,12 +247,12 @@ public class Horizon extends AbstractEPGSource implements EPGSource { } String description = null; if (prog.has("longDescription")) description = prog.getString("longDescription"); - if (description==null || description.isEmpty()) { - if (prog.has("description")) description = prog.getString("description"); - } - if (description==null || description.isEmpty()) { + if (description==null || description.isEmpty()) { + if (prog.has("description")) description = prog.getString("description"); + } + if (description==null || description.isEmpty()) { if (prog.has("shortDescription")) description = prog.getString("shortDescription"); - } + } if (description!= null && !description.isEmpty()) { result.addDescription(description); } else { @@ -254,101 +260,101 @@ public class Horizon extends AbstractEPGSource implements EPGSource { } if (prog.has("cast")) { - JSONArray cast = prog.getJSONArray("cast"); - for( int j=0; j channels = horizon.getChannels(); - System.out.println("Channels: " + channels); - XMLStreamWriter writer = XMLOutputFactory.newInstance() - .createXMLStreamWriter(new FileWriter("horizon.xml")); - writer.writeStartDocument(); - writer.writeCharacters("\n"); - writer.writeDTD(""); - writer.writeCharacters("\n"); - writer.writeStartElement("tv"); - // List my_channels = channels; - List my_channels = channels.subList(0, 5); - for (Channel c : my_channels) { - c.serialize(writer, true); - } - writer.flush(); - for(int day=0; day<5; day++) { - List programmes = horizon.getProgrammes(my_channels, day); - for (Programme p : programmes) { - p.serialize(writer); - } - } - writer.writeEndElement(); - writer.writeEndDocument(); - writer.flush(); - if (!config.quiet) { - EPGSource.Stats stats = horizon.getStats(); - System.out.println("Number of programmes from cache: " - + stats.cacheHits); - System.out.println("Number of programmes fetched: " - + stats.cacheMisses); - System.out.println("Number of fetch errors: " - + stats.fetchErrors); - } - horizon.close(); - } catch (Exception e) { - logger.error("Error in horizon testing", e); - } - } + if (!cached && !doNotCache) { + cache.put(getName(), id, result); + } + logger.debug(result); + return result; + } + /** + * @param args + */ + public static void main(String[] args) { + Logger.getRootLogger().setLevel(Level.TRACE); + Config config = Config.getDefaultConfig(); + Horizon horizon = new Horizon(config); + horizon.clearCache(); + try { + List channels = horizon.getChannels(); + System.out.println("Channels: " + channels); + XMLStreamWriter writer = XMLOutputFactory.newInstance() + .createXMLStreamWriter(new FileWriter("horizon.xml")); + writer.writeStartDocument(); + writer.writeCharacters("\n"); + writer.writeDTD(""); + writer.writeCharacters("\n"); + writer.writeStartElement("tv"); + // List my_channels = channels; + List my_channels = channels.subList(0, 5); + for (Channel c : my_channels) { + c.serialize(writer, true); + } + writer.flush(); + for(int day=0; day<5; day++) { + List programmes = horizon.getProgrammes(my_channels, day); + for (Programme p : programmes) { + p.serialize(writer); + } + } + writer.writeEndElement(); + writer.writeEndDocument(); + writer.flush(); + if (!config.quiet) { + EPGSource.Stats stats = horizon.getStats(); + System.out.println("Number of programmes from cache: " + + stats.cacheHits); + System.out.println("Number of programmes fetched: " + + stats.cacheMisses); + System.out.println("Number of fetch errors: " + + stats.fetchErrors); + } + horizon.close(); + } catch (Exception e) { + logger.error("Error in horizon testing", e); + } + } }