From 42736a904924d0e8fa601877c12b78cbfb06ca3a Mon Sep 17 00:00:00 2001 From: pidoubleyou <22942659+pidoubleyou@users.noreply.github.com> Date: Sat, 13 Jun 2026 14:56:53 +0200 Subject: [PATCH 1/5] add compilation subpage handling --- .../crawler/sender/ard/ArdConstants.java | 2 + .../crawler/sender/ard/ArdTopicInfoDto.java | 72 ++++++++++++------- .../json/ArdTopicCompilationDeserializer.java | 24 ++++++- .../ard/json/ArdTopicPageDeserializer.java | 6 +- .../ard/tasks/ArdTopicCompilationTask.java | 56 ++++++++++++++- .../sender/ard/tasks/ArdTopicPageTask.java | 2 +- 6 files changed, 131 insertions(+), 31 deletions(-) diff --git a/src/main/java/mServer/crawler/sender/ard/ArdConstants.java b/src/main/java/mServer/crawler/sender/ard/ArdConstants.java index 5a1fb063d..50813508f 100644 --- a/src/main/java/mServer/crawler/sender/ard/ArdConstants.java +++ b/src/main/java/mServer/crawler/sender/ard/ArdConstants.java @@ -11,11 +11,13 @@ public class ArdConstants { public static final String TOPICS_URL = API_URL + "/page-gateway/pages/%s/editorial/experiment-a-z?embedded=false"; public static final String TOPICS_COMPILATION_URL = API_URL + "/page-gateway/widgets/%s/editorials/%s?pageNumber=0&pageSize=%s"; + public static final String TOPIC_COMPILATION_PAGE_URL = API_URL + "/page-gateway/widgets/%s/compilation/%s?pageNumber=%d&pageSize=%d"; public static final String TOPIC_URL = API_URL + "/page-gateway/widgets/ard/asset/%s?pageSize=%d"; public static final String DAY_PAGE_URL = "https://programm-api.ard.de/program/api/program?day=%s&channelIds=%s&mode=channel"; public static final int TOPICS_COMPILATION_PAGE_SIZE = 100; public static final int TOPIC_PAGE_SIZE = 50; + public static final int MAX_COMPILATION_SUB_PAGES = 4; public static final String DEFAULT_CLIENT = "ard"; diff --git a/src/main/java/mServer/crawler/sender/ard/ArdTopicInfoDto.java b/src/main/java/mServer/crawler/sender/ard/ArdTopicInfoDto.java index 459da273f..0b50fc0d9 100644 --- a/src/main/java/mServer/crawler/sender/ard/ArdTopicInfoDto.java +++ b/src/main/java/mServer/crawler/sender/ard/ArdTopicInfoDto.java @@ -6,53 +6,77 @@ public class ArdTopicInfoDto { private final Set filmInfos; private String id; - private int subPageNumber; + private int pageNumber; + private int pageSize; + private int totalElements; private int maxSubPageNumber; + public ArdTopicInfoDto(final Set filmInfos) { this.filmInfos = filmInfos; - subPageNumber = 0; - maxSubPageNumber = 0; + setPageNumber(0); + setPageSize(0); + setTotalElements(0); + setMaxSubPageNumber(0); } public Set getFilmInfos() { return filmInfos; } - public String getId() { - return id; + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof final ArdTopicInfoDto that)) { + return false; + } + return getId() == that.getId() + && getPageNumber() == that.getPageNumber() + && getPageSize() == that.getPageSize() + && getTotalElements() == that.getTotalElements() + && Objects.equals(filmInfos, that.filmInfos); } - public void setId(String id) { - this.id = id; + @Override + public int hashCode() { + return Objects.hash(filmInfos, getId(), getPageNumber(), getPageSize(), getTotalElements()); } - public int getSubPageNumber() { - return subPageNumber; + public String getId() { return id; } + + public int getPageNumber() { + return pageNumber; } - public void setSubPageNumber(final int subPageNumber) { - this.subPageNumber = subPageNumber; + public void setPageNumber(int pageNumber) { + this.pageNumber = pageNumber; } - public int getMaxSubPageNumber() { - return maxSubPageNumber; + public int getPageSize() { + return pageSize; } - public void setMaxSubPageNumber(final int maxSubPageNumber) { - this.maxSubPageNumber = maxSubPageNumber; + public void setPageSize(int pageSize) { + this.pageSize = pageSize; } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ArdTopicInfoDto that = (ArdTopicInfoDto) o; - return Objects.equals(id, that.id) && subPageNumber == that.subPageNumber && maxSubPageNumber == that.maxSubPageNumber && Objects.equals(filmInfos, that.filmInfos); + public int getTotalElements() { + return totalElements; } - @Override - public int hashCode() { - return Objects.hash(id, filmInfos, subPageNumber, maxSubPageNumber); + public void setTotalElements(int totalElements) { + this.totalElements = totalElements; + } + + public void setId(String id) { this.id = id; } + + public int getMaxSubPageNumber() { + return maxSubPageNumber; + } + + public void setMaxSubPageNumber(final int maxSubPageNumber) { + this.maxSubPageNumber = maxSubPageNumber; } } diff --git a/src/main/java/mServer/crawler/sender/ard/json/ArdTopicCompilationDeserializer.java b/src/main/java/mServer/crawler/sender/ard/json/ArdTopicCompilationDeserializer.java index 33375a647..938ad3526 100644 --- a/src/main/java/mServer/crawler/sender/ard/json/ArdTopicCompilationDeserializer.java +++ b/src/main/java/mServer/crawler/sender/ard/json/ArdTopicCompilationDeserializer.java @@ -33,19 +33,39 @@ public ArdTopicInfoDto deserialize( results.addAll(parseTeasers(teasers)); } }); + } else if (showPageObject.getAsJsonObject().has(ELEMENT_TEASERS)) { + final JsonArray teasers = showPageObject.getAsJsonObject().get(ELEMENT_TEASERS).getAsJsonArray(); + results.addAll(parseTeasers(teasers)); } - final JsonElement paginationElement = showPageObject.get(ELEMENT_PAGINATION); + final JsonElement paginationElement = findPaginationElement(showPageObject); final int pageNumber = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_PAGE_NUMBER); final int totalElements = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_TOTAL_ELEMENTS); + ardTopicInfoDto.setTotalElements(totalElements); final int pageSize = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_PAGE_SIZE); + ardTopicInfoDto.setPageSize(pageSize); ardTopicInfoDto.setMaxSubPageNumber(pageSize == 0 ? 0 : (totalElements + pageSize - 1) / pageSize); - ardTopicInfoDto.setSubPageNumber(pageNumber); + ardTopicInfoDto.setPageNumber(pageNumber); return ardTopicInfoDto; } + private JsonElement findPaginationElement(JsonObject showPageObject) { + if (showPageObject.has(ELEMENT_WIDGETS)) { + final JsonElement widgetElement = showPageObject.get(ELEMENT_WIDGETS); + if (widgetElement.isJsonArray()) { + final JsonArray widgetArray = widgetElement.getAsJsonArray(); + if (!widgetArray.isEmpty()) { + return widgetArray.get(0).getAsJsonObject().get(ELEMENT_PAGINATION); + } + } + } else if (showPageObject.has(ELEMENT_PAGINATION)) { + return showPageObject.get(ELEMENT_PAGINATION); + } + return null; + } + private int getChildElementAsIntOrNullIfNotExist( final JsonElement parentElement, final String childElementName) { if (parentElement == null || parentElement.isJsonNull()) { diff --git a/src/main/java/mServer/crawler/sender/ard/json/ArdTopicPageDeserializer.java b/src/main/java/mServer/crawler/sender/ard/json/ArdTopicPageDeserializer.java index bf14da3b0..e7ebf6a09 100644 --- a/src/main/java/mServer/crawler/sender/ard/json/ArdTopicPageDeserializer.java +++ b/src/main/java/mServer/crawler/sender/ard/json/ArdTopicPageDeserializer.java @@ -35,12 +35,12 @@ public ArdTopicInfoDto deserialize( id.ifPresent(ardTopicInfoDto::setId); final JsonElement paginationElement = showPageObject.get(ELEMENT_PAGINATION); - ardTopicInfoDto.setSubPageNumber( + ardTopicInfoDto.setPageNumber( getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_PAGE_NUMBER)); final int totalElements = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_TOTAL_ELEMENTS); + ardTopicInfoDto.setTotalElements(totalElements); final int pageSize = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_PAGE_SIZE); - ardTopicInfoDto.setMaxSubPageNumber(pageSize == 0 ? 0 : - (totalElements + pageSize - 1) / pageSize); + ardTopicInfoDto.setPageSize(pageSize); return ardTopicInfoDto; } diff --git a/src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicCompilationTask.java b/src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicCompilationTask.java index 0d4894115..c6ec483d2 100644 --- a/src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicCompilationTask.java +++ b/src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicCompilationTask.java @@ -3,19 +3,28 @@ import com.google.gson.reflect.TypeToken; import jakarta.ws.rs.client.WebTarget; import mServer.crawler.sender.MediathekReader; +import mServer.crawler.sender.ard.ArdConstants; import mServer.crawler.sender.ard.ArdFilmInfoDto; import mServer.crawler.sender.ard.ArdTopicInfoDto; import mServer.crawler.sender.ard.json.ArdTopicCompilationDeserializer; import mServer.crawler.sender.base.AbstractRecursivConverterTask; import mServer.crawler.sender.base.CrawlerUrlDTO; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.lang.reflect.Type; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.regex.Matcher; +import java.util.regex.Pattern; public class ArdTopicCompilationTask extends ArdTaskBase { + private static final Logger LOG = LogManager.getLogger(ArdTopicCompilationTask.class); + private static final Pattern UUID_PATTERN = Pattern.compile("/compilation/([^/?]+)([/?])"); + private static final Pattern WIDGETS_SENDER_PATTERN = Pattern.compile("/pages/([^/]+)/"); // captures the segment after /widgets/ (e.g. "ard") private static final Type ARDTOPICINFODTO_TYPE_TOKEN = - new TypeToken() {}.getType(); + new TypeToken() { + }.getType(); public ArdTopicCompilationTask( final MediathekReader aCrawler, final ConcurrentLinkedQueue aUrlToCrawlDtos) { @@ -31,7 +40,52 @@ protected void processRestTarget(final CrawlerUrlDTO aDTO, final WebTarget aTarg && topicInfo.getFilmInfos() != null && !topicInfo.getFilmInfos().isEmpty()) { taskResults.addAll(topicInfo.getFilmInfos()); + + final ConcurrentLinkedQueue subpages = createSubPageUrls(aDTO, topicInfo); + if (!subpages.isEmpty()) { + taskResults.addAll(createNewOwnInstance(subpages).fork().join()); + } + } + } + + private ConcurrentLinkedQueue createSubPageUrls(CrawlerUrlDTO urlDto, ArdTopicInfoDto topicInfo) { + final ConcurrentLinkedQueue subpages = new ConcurrentLinkedQueue<>(); + if (topicInfo.getTotalElements() < topicInfo.getPageSize() || topicInfo.getPageNumber() > 0) { + return subpages; + } + // + final String sender = extractSender(urlDto.getUrl()); + final String id = extractCompilation(urlDto.getUrl()); + + int index = 0; + while ((topicInfo.getPageSize() + (index * topicInfo.getPageSize())) < topicInfo.getTotalElements()) { + subpages.add(new CrawlerUrlDTO(String.format(ArdConstants.TOPIC_COMPILATION_PAGE_URL, sender, id, index + 1, topicInfo.getPageSize()))); + index++; + if (index >= ArdConstants.MAX_COMPILATION_SUB_PAGES) { + LOG.debug("ignore more subpage due to limit of {} pages but found {}", ArdConstants.MAX_COMPILATION_SUB_PAGES, Integer.valueOf(topicInfo.getTotalElements() / topicInfo.getPageSize())); + break; + } + } + return subpages; + } + + private String extractCompilation(String url) { + final Matcher m = UUID_PATTERN.matcher(url); + if (m.find()) { + return m.group(1); + } + + LOG.warn("no compilation pattern found in url {}", url); + return ""; + } + + private String extractSender(String url) { + final Matcher m = WIDGETS_SENDER_PATTERN.matcher(url); + if (m.find()) { + return m.group(1); } + LOG.warn("no sender pattern found in url {}", url); + return ""; } @Override diff --git a/src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicPageTask.java b/src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicPageTask.java index 62aa8e416..f15ece061 100644 --- a/src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicPageTask.java +++ b/src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicPageTask.java @@ -68,7 +68,7 @@ private ConcurrentLinkedQueue createSubPageUrls( final WebTarget aTarget, final ArdTopicInfoDto topicInfo) { final ConcurrentLinkedQueue subpages = new ConcurrentLinkedQueue<>(); - final int actualSubPageNumber = topicInfo.getSubPageNumber(); + final int actualSubPageNumber = topicInfo.getPageNumber(); final int maximumAllowedSubpages = getMaximumSubpages(topicInfo.getId()); if (actualSubPageNumber != 0) { LOG.debug("Sub page {} is already the maximum allowed sub page.", actualSubPageNumber); From a9c8652ae1322fe902e9a1f8779f75b4f4db4ba2 Mon Sep 17 00:00:00 2001 From: pidoubleyou <22942659+pidoubleyou@users.noreply.github.com> Date: Sun, 21 Jun 2026 21:48:20 +0200 Subject: [PATCH 2/5] fix --- .../crawler/sender/ard/json/ArdTopicPageDeserializer.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/mServer/crawler/sender/ard/json/ArdTopicPageDeserializer.java b/src/main/java/mServer/crawler/sender/ard/json/ArdTopicPageDeserializer.java index e7ebf6a09..41c0b7e44 100644 --- a/src/main/java/mServer/crawler/sender/ard/json/ArdTopicPageDeserializer.java +++ b/src/main/java/mServer/crawler/sender/ard/json/ArdTopicPageDeserializer.java @@ -41,6 +41,8 @@ public ArdTopicInfoDto deserialize( ardTopicInfoDto.setTotalElements(totalElements); final int pageSize = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_PAGE_SIZE); ardTopicInfoDto.setPageSize(pageSize); + ardTopicInfoDto.setMaxSubPageNumber(pageSize == 0 ? 0 : + (totalElements + pageSize - 1) / pageSize); return ardTopicInfoDto; } From 7c549472a266bf3456fbc1b7a5f42aa20b2b35f4 Mon Sep 17 00:00:00 2001 From: pidoubleyou <22942659+pidoubleyou@users.noreply.github.com> Date: Sun, 21 Jun 2026 21:48:41 +0200 Subject: [PATCH 3/5] deep search for all retro topics --- .../crawler/sender/ard/tasks/ArdTopicPageTask.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicPageTask.java b/src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicPageTask.java index f15ece061..ddf1a2da7 100644 --- a/src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicPageTask.java +++ b/src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicPageTask.java @@ -34,6 +34,16 @@ public class ArdTopicPageTask extends ArdTaskBase private static final Set TOPICS_LOAD_ALL_PAGES = new HashSet<>(); static { + // Retro + TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL3JhZGlvYnJlbWVuLmRlL2IwYTJlZWFlLWI2NjAtNDI5Yi05ZTE3LTM5YzlkZDhmNTc4Ng"); + TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL2JyLmRlL2Jyb2FkY2FzdFNlcmllcy8yOGEwMzU4Yi00N2ViLTQ0MDktOGFmZi02ZjVkMDE5NDA2NDc"); + TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL3dkci5kZS93ZHJyZXRybw"); + TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL25kci5kZS80NTg4"); + TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL3JiYi1vbmxpbmUuZGUvYmVybGluZXItYWJlbmRzY2hhdQ"); + TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL2hyLW9ubGluZS8zODIyMDExOA"); + TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL3N3ci5kZS8yNDEwMzE1Ng"); + TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL2hyLW9ubGluZS8zODIyMDA5Ng"); + TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL3NyLW9ubGluZS5kZS9SRVRSTy1IRA"); // example: TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL2Rhc2Vyc3RlLmRlL3Nwb3J0c2NoYXU"); } From 64fe566d4d154aecc97684bcf0a7b26a69e26a47 Mon Sep 17 00:00:00 2001 From: pidoubleyou <22942659+pidoubleyou@users.noreply.github.com> Date: Sun, 21 Jun 2026 22:07:11 +0200 Subject: [PATCH 4/5] deep search for Nordstory --- .../java/mServer/crawler/sender/ard/tasks/ArdTopicPageTask.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicPageTask.java b/src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicPageTask.java index ddf1a2da7..18efe8cd9 100644 --- a/src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicPageTask.java +++ b/src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicPageTask.java @@ -44,6 +44,8 @@ public class ArdTopicPageTask extends ArdTaskBase TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL3N3ci5kZS8yNDEwMzE1Ng"); TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL2hyLW9ubGluZS8zODIyMDA5Ng"); TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL3NyLW9ubGluZS5kZS9SRVRSTy1IRA"); + // NDR Nordstory + TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL25kci5kZS8xNDk1"); // example: TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL2Rhc2Vyc3RlLmRlL3Nwb3J0c2NoYXU"); } From 4c6f2161b8b73a217241bed44376b5a339c9a1cc Mon Sep 17 00:00:00 2001 From: pidoubleyou <22942659+pidoubleyou@users.noreply.github.com> Date: Sun, 21 Jun 2026 22:17:15 +0200 Subject: [PATCH 5/5] deep search for Eisenbahn-Romantik --- .../java/mServer/crawler/sender/ard/tasks/ArdTopicPageTask.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicPageTask.java b/src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicPageTask.java index 18efe8cd9..bb788d741 100644 --- a/src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicPageTask.java +++ b/src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicPageTask.java @@ -46,6 +46,8 @@ public class ArdTopicPageTask extends ArdTaskBase TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL3NyLW9ubGluZS5kZS9SRVRSTy1IRA"); // NDR Nordstory TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL25kci5kZS8xNDk1"); + // SWR Eisenbahn-Romantik + TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL3N3ci5kZS8yMjUwOTg2"); // example: TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL2Rhc2Vyc3RlLmRlL3Nwb3J0c2NoYXU"); }