User-agent: *
Allow: 		/ 
Disallow: /errorpages/
Disallow: /articoli/ajax/
Disallow: /articoli/includes/
Disallow: /includes/
Disallow: */typo3temp/
Disallow: /*?refresh_ce
Disallow: /*?p= 

# sitemap
Sitemap:  https://agrigento.gds.it/sitemap.xml
Sitemap:  https://agrigento.gds.it/sitemap-news.xml

# misc
User-agent: GPTBot 
Disallow: 		/ 
User-agent: CCBot 
Disallow: 		/ 
User-agent: Google-Extended 
Disallow: 		/ 
User-agent: anthropic-ai 
Disallow: 		/ 
User-agent: Omgilibot 
Disallow: 		/ 
User-agent: cohere-ai 
Disallow: 		/ 
User-agent: FacebookBot 
Disallow: 		/ 
User-agent: Yandex 
Disallow: 		/ 
User-agent: PetalBot 
Disallow: 		/ 
User-agent: PerplexityBot 
Disallow: 		/ 
User-agent: searchpreview 
Disallow: 		/ 
User-agent: BacklinkCrawler 
Disallow: 		/ 
User-agent: Kangaroo Bot 
Disallow: 		/ 
User-agent: Quora-Bot 
Disallow: 		/ 
User-agent: Arquivo-web-crawler 
Disallow: 		/ 
User-agent: JikeSpider 
Disallow: 		/ 
User-agent: Primalbot 
Disallow: 		/ 
User-agent: Amazonbot 
Disallow: 		/ 
User-agent: ImagesiftBot 
Disallow: 		/ 
User-agent: peer39_crawler/1.0 
Disallow: 		/ 
User-agent: Zealbot 
Disallow: 		/ 
User-agent: GumGum Bot 
Disallow: 		/ 
User-agent: Openbot 
Disallow: 		/ 
User-agent: WeSEE:Search 
Disallow: 		/ 
User-agent: Flamingo_SearchEngine 
Disallow: 		/ 
User-agent: Nicecrawler 
Disallow: 		/ 
User-agent: verticalsearch 
Disallow: 		/ 
User-agent: Exabot 
Disallow: 		/ 
User-agent: netEstate NE Crawler 
Disallow: 		/ 
User-agent: UbiCrawler 
Disallow: 		/ 
User-agent: Download Ninja 
Disallow: 		/ 
User-agent: nabot 
Disallow: 		/ 
User-agent: trovitBot 
Disallow: 		/ 
User-agent: discoverybot 
Disallow: 		/ 
User-agent: MJ12bot 
Disallow: 		/ 
User-agent: spbot 
Disallow: 		/ 
User-agent: DataForSeoBot 
Disallow: 		/ 
User-agent: Livelapbot 
Disallow: 		/ 
User-agent: SiteBot/0.1 
Disallow: 		/ 
User-agent: ConveraCrawler 
Disallow: 		/ 
User-agent: SEOENGBot 
Disallow: 		/ 
User-agent: BLEXBot 
Disallow: 		/ 
User-agent: LexxeBot/1.0 
Disallow: 		/ 
User-agent: Scrapy 
Disallow: 		/ 
User-agent: AwarioSmartBot 
Disallow: 		/ 
User-agent: k2spider 
Disallow: 		/ 
User-agent: QuerySeekerSpider 
Disallow: 		/ 
User-agent: ArchiveBot 
Disallow: 		/ 
User-agent: Jetbot 
Disallow: 		/ 
User-agent: Pixray-Seeker 
Disallow: 		/ 
User-agent: AhrefsBot 
Disallow: 		/ 
User-agent: ia_archiver-web.archive.org 
Disallow: 		/ 
User-agent: peer39_crawler 
Disallow: 		/ 
User-agent: YouBot 
Disallow: 		/ 
User-agent: Gigabot 
Disallow: 		/ 
User-agent: nutch 
Disallow: 		/ 
User-agent: web-archive-net.com.bot 
Disallow: 		/ 
User-agent: Fetch 
Disallow: 		/ 
User-agent: NextGenSearchBot 
Disallow: 		/ 
User-agent: URL_Spider_Pro 
Disallow: 		/ 
User-agent: europarchive.org 
Disallow: 		/ 
User-agent: NerdByNature.Bot 
Disallow: 		/ 
User-agent: TurnitinBot 
Disallow: 		/ 
User-agent: dotbot 
Disallow: 		/ 
User-agent: MSIECrawler 
Disallow: 		/ 
User-agent: trendictionbot 
Disallow: 		/ 
User-agent: discobot 
Disallow: 		/ 
User-agent: meta-externalfetcher 
Disallow: 		/ 
User-agent: Sosospider 
Disallow: 		/ 
User-agent: CrystalSemanticsBot 
Disallow: 		/ 
User-agent: LinkextractorPro 
Disallow: 		/ 
User-agent: SiteBot 
Disallow: 		/ 
User-agent: Cliqzbot 
Disallow: 		/ 
User-agent: SemrushBot 
Disallow: 		/ 
User-agent: BIXOCRAWLER 
Disallow: 		/ 
User-agent: kbcrawl 
Disallow: 		/ 
User-agent: rogerbot 
Disallow: 		/ 
User-agent: AwarioRssBot 
Disallow: 		/ 
User-agent: Jyxobot 
Disallow: 		/ 
User-agent: psbot 
Disallow: 		/ 
User-agent: archive.org_bot 
Disallow: 		/ 
User-agent: IstellaBot 
Disallow: 		/ 
User-agent: PiplBot 
Disallow: 		/ 
User-agent: ia_archiver 
Disallow: 		/ 
User-agent: PanguBot 
Disallow: 		/ 
User-agent: Wotbot 
Disallow: 		/ 
User-agent: FriendlyCrawler 
Disallow: 		/ 
User-agent: NPBot 
Disallow: 		/ 
User-agent: WBSearchBot 
Disallow: 		/ 
User-agent: ExtractorPro 
Disallow: 		/ 
User-agent: nNetSeer crawler 
Disallow: 		/ 
User-agent: uMBot-LN 
Disallow: 		/ 
User-agent: DuckAssistBot 
Disallow: 		/ 
User-agent: NaverBot 
Disallow: 		/ 
User-agent: True_Robot 
Disallow: 		/ 
User-agent: dloader(NaverRobot) 
Disallow: 		/ 
User-agent: Moreoverbot 
Disallow: 		/ 
User-agent: Timpibot 
Disallow: 		/ 
User-agent: Diffbot 
Disallow: 		/ 
User-agent: magpie-crawler 
Disallow: 		/ 
User-agent: Slurp 
Disallow: 		/ 
User-agent: crawler4j 
Disallow: 		/ 
User-agent: LinkArchiver 
Disallow: 		/ 
User-agent: SEOkicks-Robot 
Disallow: 		/ 
User-agent: Bytespider 
Disallow: 		/ 



User-agent: AliyunSecBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: AudigentAdBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: EchoboxBot
Disallow: /

User-agent: Google-CloudVertexBot
Disallow: /

User-agent: Jetslide
Disallow: /

User-agent: Meta-ExternalAgent
User-agent: meta-externalagent
Disallow: /

User-agent: MyCentralAIScraperBot
Disallow: /

User-agent: NewsNow
Disallow: /

User-agent: news-please
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: Poseidon Research Crawler
Disallow: /

User-agent: quillbot.com
Disallow: /

User-agent: SeekrBot
Disallow: /

User-agent: SeznamHomepageCrawler
Disallow: /

User-agent: TaraGroup Intelligent Bot
Disallow: /

User-agent: ViennaTinyBot
Disallow: /





User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /