# Robots.txt for crawler

User-agent: facebookexternalhit
Allow: /
Disallow: /Sso
Disallow: /Benutzer
Disallow: /User
Disallow: /Anzeigen
Disallow: /Dateien
Disallow: /Nachrichten/Suche
Disallow: /Motif/*.jpg

User-agent: Facebot
Allow: /
Disallow: /Sso
Disallow: /Benutzer
Disallow: /User
Disallow: /Anzeigen
Disallow: /Dateien
Disallow: /Nachrichten/Suche
Disallow: /Motif/*.jpg

User-agent: *

# Disallow Crawler
Disallow: /Sso
Disallow: /Benutzer
Disallow: /User
Disallow: /Anzeigen
Disallow: /Dateien
Disallow: /Nachrichten/Suche
# possibly do not allow motif pictures to prevent file generation in advance
#Disallow: /Motif/*.jpg

# disallow some KI bots
User-agent: ChatGPT-User
Disallow: /
User-agent: GPTBot
Disallow: /
User-agent: CCBot
Disallow: /

# Max crawler time per page in sec
Crawl-Delay: 2

#Sitemap
Sitemap: https://www.tageblatt.lu/Sitemap_Index.xml.gz

# Legal notice: tageblatt.lu expressly reserves the right to use its content for commercial text and data mining (� 44b UrhG). 
# The use of robots or other automated means to access tageblatt.lu or collect or mine data without the express permission of tageblatt.lu is strictly prohibited. 
# If you would like to apply for permission to crawl tageblatt.lu, collect or use data, please contact syndication@tageblatt.lu