mServer.crawler.sender.arte.MediathekArte_de Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of MServer Show documentation
Show all versions of MServer Show documentation
The crawler for mediathekview/MediathekView
/*
* MediathekView
* Copyright (C) 2008 W. Xaver
* W.Xaver[at]googlemail.com
* http://zdfmediathk.sourceforge.net/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see Sendetermine: ";
protected String TIME_2 = "um";
public MediathekArte_de(FilmeSuchen ssearch, int startPrio) {
super(ssearch, SENDERNAME,/* threads */ 2, /* urlWarten */ 200, startPrio);
}
public MediathekArte_de(FilmeSuchen ssearch, int startPrio, String name) {
super(ssearch, name,/* threads */ 2, /* urlWarten */ 200, startPrio);
}
//===================================
// public
//===================================
@Override
public void addToList() {
meldungStart();
if (Config.getStop()) {
meldungThreadUndFertig();
} else {
if (CrawlerTool.loadLongMax()) {
addCategories();
meldungAddMax(listeThemen.size());
for (int t = 0; t < getMaxThreadLaufen(); ++t) {
Thread th = new CategoryLoader();
th.setName(getSendername() + t);
th.start();
}
} else {
addTage();
meldungAddMax(listeThemen.size());
for (int t = 0; t < getMaxThreadLaufen(); ++t) {
Thread th = new ThemaLaden();
th.setName(getSendername() + t);
th.start();
}
}
}
}
private void addCategories() {
for (String subCategory : SUBCATEGORIES) {
String subCategoryUrl = String.format(URL_SUBCATEGORY, LANG_CODE.toLowerCase(), subCategory, 1);
listeThemen.add(new String[]{subCategory, subCategoryUrl});
}
}
private void addTage() {
// http://www.arte.tv/guide/de/plus7/videos?day=-2&page=1&isLoading=true&sort=newest&country=DE
for (int i = 0; i <= 14; ++i) {
String u = String.format(ARTE_API_TAG_URL_PATTERN, LANG_CODE.toUpperCase(), LocalDate.now().minusDays(i).format(ARTE_API_DATEFORMATTER));
listeThemen.add(new String[]{u});
}
}
class ThemaLaden extends Thread {
@Override
public void run() {
try {
meldungAddThread();
String link[];
while (!Config.getStop() && (link = listeThemen.getListeThemen()) != null) {
meldungProgress(link[0] /* url */);
addFilmeForTag(link[0]);
}
} catch (Exception ex) {
Log.errorLog(894330854, ex, "");
}
meldungThreadUndFertig();
}
private void addFilmeForTag(String aUrl) {
Gson gson = new GsonBuilder().registerTypeAdapter(ListeFilme.class, new ArteDatenFilmDeserializer(LANG_CODE, getSendername())).create();
ListeFilme loadedFilme = ArteHttpClient.executeRequest(LOG, gson, aUrl, ListeFilme.class);
if (loadedFilme != null) {
loadedFilme.forEach((film) -> {
addFilm(film);
});
}
}
}
/**
* Lädt die Filme für jede Kategorie
*/
class CategoryLoader extends Thread {
@Override
public void run() {
try {
meldungAddThread();
String link[];
while (!Config.getStop() && (link = listeThemen.getListeThemen()) != null) {
meldungProgress(link[0] + "/" + link[1] /* url */);
loadSubCategory(link[0], link[1]);
}
} catch (Exception ex) {
Log.errorLog(894330854, ex, "");
}
meldungThreadUndFertig();
}
private void loadSubCategory(String aCategory, String aUrl) {
Gson gson = new GsonBuilder().registerTypeAdapter(ArteCategoryFilmsDTO.class, new ArteCategoryFilmListDeserializer()).create();
// erste Seite laden
int i = 2;
ArteCategoryFilmsDTO dto = loadSubCategoryPage(gson, aUrl);
if (dto != null) {
ArteCategoryFilmsDTO nextDto = dto;
while (nextDto != null && nextDto.hasNextPage()) {
// weitere Seiten laden und zu programId-liste des ersten DTO hinzufügen
String url = String.format(URL_SUBCATEGORY, LANG_CODE.toLowerCase(), aCategory, i);
nextDto = loadSubCategoryPage(gson, url);
if (nextDto != null) {
nextDto.getProgramIds().forEach(programId -> dto.addProgramId(programId));
}
i++;
}
// alle programIds verarbeiten
ListeFilme loadedFilme = loadPrograms(dto);
loadedFilme.forEach((film) -> {
addFilm(film);
});
}
}
private ListeFilme loadPrograms(ArteCategoryFilmsDTO dto) {
ListeFilme listeFilme = new ListeFilme();
Collection futureFilme = new ArrayList<>();
dto.getProgramIds().forEach(programId -> {
try {
futureFilme.add(new ArteProgramIdToDatenFilmCallable(programId, LANG_CODE, getSendername()).call());
} catch (Exception exception) {
LOG.error("Es ist ein Fehler beim lesen der Arte Filme aufgetreten.", exception);
}
});
CopyOnWriteArrayList finishedFilme = new CopyOnWriteArrayList<>();
futureFilme.parallelStream().forEach(finishedFilm -> {
try {
if (finishedFilm != null) {
finishedFilme.add(finishedFilm);
}
} catch (Exception exception) {
LOG.error("Es ist ein Fehler beim lesen der Arte Filme aufgetreten.", exception);
}
});
listeFilme.addAll(finishedFilme);
return listeFilme;
}
private ArteCategoryFilmsDTO loadSubCategoryPage(Gson gson, String aUrl) {
return ArteHttpClient.executeRequest(LOG, gson, aUrl, ArteCategoryFilmsDTO.class);
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy