From e548df8f3e835afdc9fbb363985b7fde4fb67ffe Mon Sep 17 00:00:00 2001 From: pidoubleyou <22942659+pidoubleyou@users.noreply.github.com> Date: Mon, 10 Mar 2025 22:34:19 +0100 Subject: [PATCH 1/5] tagesschau24 als eigener Sender --- src/main/java/mServer/crawler/sender/ard/ArdCrawler.java | 1 + .../mServer/crawler/sender/ard/json/ArdFilmDeserializer.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/mServer/crawler/sender/ard/ArdCrawler.java b/src/main/java/mServer/crawler/sender/ard/ArdCrawler.java index a2c2d9683..6ca856cc5 100644 --- a/src/main/java/mServer/crawler/sender/ard/ArdCrawler.java +++ b/src/main/java/mServer/crawler/sender/ard/ArdCrawler.java @@ -57,6 +57,7 @@ protected synchronized void meldungThreadUndFertig() { mlibFilmeSuchen.meldenFertig("ONE"); mlibFilmeSuchen.meldenFertig("ARD-alpha"); mlibFilmeSuchen.meldenFertig("Funk.net"); + mlibFilmeSuchen.meldenFertig("tagesschau24"); mlibFilmeSuchen.meldenFertig(Const.SR); mlibFilmeSuchen.meldenFertig(Const.PHOENIX); } diff --git a/src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java b/src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java index c3877c385..7f94f4bf6 100644 --- a/src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java +++ b/src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java @@ -98,7 +98,7 @@ public class ArdFilmDeserializer implements JsonDeserializer> { ADDITIONAL_SENDER.put("hr", Const.HR); ADDITIONAL_SENDER.put("br", Const.BR); ADDITIONAL_SENDER.put("radio_bremen", "rbtv"); - ADDITIONAL_SENDER.put("tagesschau24", Const.ARD); + ADDITIONAL_SENDER.put("tagesschau24", "tagesschau24"); ADDITIONAL_SENDER.put("das_erste", Const.ARD); ADDITIONAL_SENDER.put("one", Const.ARD); // ONE ADDITIONAL_SENDER.put("ard-alpha", Const.ARD); // ARD-alpha From 28eb691a9c4b594454e50789a7a13c0b5a2dd44d Mon Sep 17 00:00:00 2001 From: pidoubleyou <22942659+pidoubleyou@users.noreply.github.com> Date: Sun, 27 Apr 2025 21:22:02 +0200 Subject: [PATCH 2/5] tagesschau24 als eigener Sender --- .../java/mServer/crawler/AddToFilmlist.java | 10 +++++++ .../crawler/sender/ard/ArdCrawler.java | 8 ++--- .../sender/ard/json/ArdFilmDeserializer.java | 8 ++--- .../mServer/crawler/AddToFilmlistTest.java | 30 +++++++++++++++++++ 4 files changed, 48 insertions(+), 8 deletions(-) diff --git a/src/main/java/mServer/crawler/AddToFilmlist.java b/src/main/java/mServer/crawler/AddToFilmlist.java index aae368cde..f1cde752a 100644 --- a/src/main/java/mServer/crawler/AddToFilmlist.java +++ b/src/main/java/mServer/crawler/AddToFilmlist.java @@ -138,6 +138,7 @@ private void performInitialCleanup() { updateAudioDescriptionSrf(listeEinsortieren); updateTitle(listeEinsortieren); updateArdWebsite(listeEinsortieren); + updateSenderTagesschau24(listeEinsortieren); updateFunkMissingHost(listeEinsortieren); removeSrfUrlParameter(listeEinsortieren); } @@ -196,6 +197,15 @@ private void updateArdWebsite(ListeFilme listeEinsortieren) { list.forEach(film -> film.arr[DatenFilm.FILM_WEBSEITE] = film.arr[DatenFilm.FILM_WEBSEITE].replace("/ard/player/", "/video/").trim()); } + private void updateSenderTagesschau24(ListeFilme listeEinsortieren) { + final List whiteListTagesschau24 = Arrays.stream(new String[] { "Ulrich Timm im Gespräch", "tagesschau in Einfacher Sprache", "tagesschau in 100 Sekunden", "tagesschau24" }).toList(); + final List list = listeEinsortieren.parallelStream() + .filter(film -> film.arr[DatenFilm.FILM_SENDER].equals(Const.ARD) && whiteListTagesschau24.contains(film.arr[DatenFilm.FILM_THEMA])) + .collect(Collectors.toList()); + Log.sysLog("ARD: set sender tagesschau24 für " + list.size() + " Einträge."); + + list.forEach(film -> film.arr[DatenFilm.FILM_SENDER] = Const.TAGESSCHAU24); + } private void updateAudioDescriptionOrf(ListeFilme listeEinsortieren) { final List list = listeEinsortieren.parallelStream() diff --git a/src/main/java/mServer/crawler/sender/ard/ArdCrawler.java b/src/main/java/mServer/crawler/sender/ard/ArdCrawler.java index 6ca856cc5..a4554da2c 100644 --- a/src/main/java/mServer/crawler/sender/ard/ArdCrawler.java +++ b/src/main/java/mServer/crawler/sender/ard/ArdCrawler.java @@ -53,11 +53,11 @@ protected synchronized void meldungThreadUndFertig() { mlibFilmeSuchen.meldenFertig(Const.WDR); mlibFilmeSuchen.meldenFertig(Const.HR); mlibFilmeSuchen.meldenFertig(Const.BR); - mlibFilmeSuchen.meldenFertig("rbtv"); - mlibFilmeSuchen.meldenFertig("ONE"); - mlibFilmeSuchen.meldenFertig("ARD-alpha"); + mlibFilmeSuchen.meldenFertig(Const.RBTV); + mlibFilmeSuchen.meldenFertig(Const.ONE); + mlibFilmeSuchen.meldenFertig(Const.ARD_ALPHA); mlibFilmeSuchen.meldenFertig("Funk.net"); - mlibFilmeSuchen.meldenFertig("tagesschau24"); + mlibFilmeSuchen.meldenFertig(Const.TAGESSCHAU24); mlibFilmeSuchen.meldenFertig(Const.SR); mlibFilmeSuchen.meldenFertig(Const.PHOENIX); } diff --git a/src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java b/src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java index 7f94f4bf6..1115e3b17 100644 --- a/src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java +++ b/src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java @@ -97,11 +97,11 @@ public class ArdFilmDeserializer implements JsonDeserializer> { ADDITIONAL_SENDER.put("wdr", Const.WDR); ADDITIONAL_SENDER.put("hr", Const.HR); ADDITIONAL_SENDER.put("br", Const.BR); - ADDITIONAL_SENDER.put("radio_bremen", "rbtv"); - ADDITIONAL_SENDER.put("tagesschau24", "tagesschau24"); + ADDITIONAL_SENDER.put("radio_bremen", Const.RBTV); + ADDITIONAL_SENDER.put("tagesschau24", Const.TAGESSCHAU24); ADDITIONAL_SENDER.put("das_erste", Const.ARD); - ADDITIONAL_SENDER.put("one", Const.ARD); // ONE - ADDITIONAL_SENDER.put("ard-alpha", Const.ARD); // ARD-alpha + ADDITIONAL_SENDER.put("one", Const.ONE); // ONE + ADDITIONAL_SENDER.put("ard-alpha", Const.ARD_ALPHA); // ARD-alpha ADDITIONAL_SENDER.put("funk", "Funk.net"); // Funk.net ADDITIONAL_SENDER.put("sr", Const.SR); ADDITIONAL_SENDER.put("phoenix", Const.PHOENIX); diff --git a/src/test/developTest/java/mServer/crawler/AddToFilmlistTest.java b/src/test/developTest/java/mServer/crawler/AddToFilmlistTest.java index 0f43ed5a8..7f8998331 100644 --- a/src/test/developTest/java/mServer/crawler/AddToFilmlistTest.java +++ b/src/test/developTest/java/mServer/crawler/AddToFilmlistTest.java @@ -477,6 +477,36 @@ public void testArdEntriesOfZdfArteKikaRemoved() { assertFalse(list.contains(testFilmZdf)); } + @Test + public void testArdTagesschau24EntriesNotAdded() { + final DatenFilm testFilmArd24 = createTestFilm(Const.ARD, "tagesschau24", "film title", FILM_NAME_ONLINE); + final DatenFilm testFilmArdOk = createTestFilm(Const.ARD, "tagesschau", "film title", FILM_NAME_ONLINE2); + listToAdd.add(testFilmArd24); + listToAdd.add(testFilmArdOk); + list.add(createTestFilm(Const.TAGESSCHAU24, "tagesschau24", "film title", FILM_NAME_ONLINE)); + + AddToFilmlist target =new AddToFilmlist(list, listToAdd); + target.addOldList(); + + assertEquals(list.size(),4); + assertFalse(list.contains(testFilmArd24)); + assertTrue(list.contains(testFilmArdOk)); + assertEquals(Const.ARD, testFilmArdOk.arr[DatenFilm.FILM_SENDER]); + } + + @Test + public void testArdTagesschau24UpdateSender() { + final DatenFilm testFilmArd24 = createTestFilm(Const.ARD, "tagesschau24", "film title", FILM_NAME_ONLINE); + listToAdd.add(testFilmArd24); + + AddToFilmlist target =new AddToFilmlist(list, listToAdd); + target.addOldList(); + + assertEquals(list.size(),3); + assertTrue(list.contains(testFilmArd24)); + assertEquals(Const.TAGESSCHAU24, testFilmArd24.arr[DatenFilm.FILM_SENDER]); + } + private static DatenFilm createTestFilm(String sender, String topic, String title, String filmUrl) { DatenFilm film = new DatenFilm(sender, topic, "url", title, baseUrl + filmUrl, "", "", "", 12, From 7fcb64d36074ae0e2e7859f58f7d4a2dc1cdc995 Mon Sep 17 00:00:00 2001 From: pidoubleyou <22942659+pidoubleyou@users.noreply.github.com> Date: Fri, 16 May 2025 20:17:59 +0200 Subject: [PATCH 3/5] add todo --- src/main/java/mServer/crawler/AddToFilmlist.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/mServer/crawler/AddToFilmlist.java b/src/main/java/mServer/crawler/AddToFilmlist.java index f1cde752a..5f48a6f8a 100644 --- a/src/main/java/mServer/crawler/AddToFilmlist.java +++ b/src/main/java/mServer/crawler/AddToFilmlist.java @@ -138,6 +138,7 @@ private void performInitialCleanup() { updateAudioDescriptionSrf(listeEinsortieren); updateTitle(listeEinsortieren); updateArdWebsite(listeEinsortieren); + // todo geht so was auch für ONE und ardalpha? updateSenderTagesschau24(listeEinsortieren); updateFunkMissingHost(listeEinsortieren); removeSrfUrlParameter(listeEinsortieren); From 762641e2a3b71ba5ab68655c99ab5ea99183ecaf Mon Sep 17 00:00:00 2001 From: pidoubleyou <22942659+pidoubleyou@users.noreply.github.com> Date: Fri, 16 May 2025 21:26:41 +0200 Subject: [PATCH 4/5] add test for one entries --- .../java/mServer/crawler/AddToFilmlistTest.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/test/developTest/java/mServer/crawler/AddToFilmlistTest.java b/src/test/developTest/java/mServer/crawler/AddToFilmlistTest.java index 7f8998331..f70254b49 100644 --- a/src/test/developTest/java/mServer/crawler/AddToFilmlistTest.java +++ b/src/test/developTest/java/mServer/crawler/AddToFilmlistTest.java @@ -477,6 +477,23 @@ public void testArdEntriesOfZdfArteKikaRemoved() { assertFalse(list.contains(testFilmZdf)); } + @Test + public void testArdEntriesNotAddedIfOneEntryExists() { + final DatenFilm testFilmOne = createTestFilm(Const.ARD, "one topic", "one title", FILM_NAME_ONLINE); + final DatenFilm testFilmArdOk = createTestFilm(Const.ARD, "ardtopic", "ard title", FILM_NAME_ONLINE2); + listToAdd.add(testFilmOne); + listToAdd.add(testFilmArdOk); + list.add(createTestFilm(Const.ONE, "one topic", "one title", FILM_NAME_ONLINE)); + + AddToFilmlist target =new AddToFilmlist(list, listToAdd); + target.addOldList(); + + assertEquals(list.size(),4); + assertFalse(list.contains(testFilmOne)); + assertTrue(list.contains(testFilmArdOk)); + assertEquals(Const.ARD, testFilmArdOk.arr[DatenFilm.FILM_SENDER]); + } + @Test public void testArdTagesschau24EntriesNotAdded() { final DatenFilm testFilmArd24 = createTestFilm(Const.ARD, "tagesschau24", "film title", FILM_NAME_ONLINE); From c3a28f046534dcff223622c87cb6b98b2b217173 Mon Sep 17 00:00:00 2001 From: pidoubleyou <22942659+pidoubleyou@users.noreply.github.com> Date: Fri, 16 May 2025 21:39:55 +0200 Subject: [PATCH 5/5] remove todo --- src/main/java/mServer/crawler/AddToFilmlist.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/mServer/crawler/AddToFilmlist.java b/src/main/java/mServer/crawler/AddToFilmlist.java index 5f48a6f8a..f1cde752a 100644 --- a/src/main/java/mServer/crawler/AddToFilmlist.java +++ b/src/main/java/mServer/crawler/AddToFilmlist.java @@ -138,7 +138,6 @@ private void performInitialCleanup() { updateAudioDescriptionSrf(listeEinsortieren); updateTitle(listeEinsortieren); updateArdWebsite(listeEinsortieren); - // todo geht so was auch für ONE und ardalpha? updateSenderTagesschau24(listeEinsortieren); updateFunkMissingHost(listeEinsortieren); removeSrfUrlParameter(listeEinsortieren);