Mirrors/kleinanzeigen-bot
1
0
Fork 0
mirror of https://github.com/Second-Hand-Friends/kleinanzeigen-bot.git synced 2026-03-12 10:31:50 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix: continue own-ad extraction when links are incomplete (#854)

Browse Source
This commit is contained in:
Jens
2026-03-02 06:05:21 +01:00
committed by GitHub
parent ed6137c8ae
commit c4a2d1c4f5
3 changed files with 77 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
src/kleinanzeigen_bot/extract.py
View File

@@ -164,13 +164,33 @@ class AdExtractor(WebScrapingMixin):
list_items = await self.web_find_all(By.CLASS_NAME, "cardbox", parent = ad_list_container)
LOG.info("Found %s ad items on page %s.", len(list_items), page_num)
page_refs:list[str] = [str((await self.web_find(By.CSS_SELECTOR, "div h3 a.text-onSurface", parent = li)).attrs["href"]) for li in list_items]
page_refs:list[str] = []
for index, li in enumerate(list_items, start = 1):
try:
link_elem = await self.web_find(By.CSS_SELECTOR, "div h3 a.text-onSurface", parent = li)
href = link_elem.attrs.get("href")
if href:
page_refs.append(str(href))
else:
LOG.warning(
"Skipping ad item %s/%s on page %s: ad reference link has no href attribute.",
index,
len(list_items),
page_num,
)
except TimeoutError:
LOG.warning(
"Skipping ad item %s/%s on page %s: no ad reference link found (likely unpublished or draft item).",
index,
len(list_items),
page_num,
)
refs.extend(page_refs)
LOG.info("Successfully extracted %s refs from page %s.", len(page_refs), page_num)
return False # Continue to next page
except TimeoutError:
LOG.warning("Could not find ad list container or items on page %s.", page_num)
LOG.warning("Could not find ad list container or ad items on page %s.", page_num)
return True # Stop pagination (ads disappeared)
except Exception as e:
# Continue despite error for resilience against transient web scraping issues

4
src/kleinanzeigen_bot/resources/translations.de.yaml
View File

@@ -299,9 +299,11 @@ kleinanzeigen_bot/extract.py:
"No ad URLs were extracted.": "Es wurden keine Anzeigen-URLs extrahiert."
extract_page_refs:
"Could not find ad list container or items on page %s.": "Anzeigenlistencontainer oder Elemente auf Seite %s nicht gefunden."
"Could not find ad list container or ad items on page %s.": "Anzeigenlistencontainer oder Anzeigenelemente auf Seite %s nicht gefunden."
"Error extracting refs on page %s: %s": "Fehler beim Extrahieren der Referenzen auf Seite %s: %s"
"Found %s ad items on page %s.": "%s Anzeigen-Elemente auf Seite %s gefunden."
"Skipping ad item %s/%s on page %s: ad reference link has no href attribute.": "Überspringe Anzeigenelement %s/%s auf Seite %s: Anzeigenlink hat kein href-Attribut."
"Skipping ad item %s/%s on page %s: no ad reference link found (likely unpublished or draft item).": "Überspringe Anzeigenelement %s/%s auf Seite %s: kein Anzeigenlink gefunden (wahrscheinlich unveröffentlicht oder Entwurf)."
"Successfully extracted %s refs from page %s.": "%s Referenzen von Seite %s erfolgreich extrahiert."
navigate_to_ad_page: