User-agent: *
Allow: /
Disallow: /tag/

# WordPress system
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /?s=
Disallow: /search/

# Sitemaps (Yoast SEO)
Sitemap: https://itech-news.ru/sitemap_index.xml
Sitemap: https://itech-news.ru/google-news.xml/

# AI crawlers
User-agent: GPTBot
Allow: /
Disallow: /tag/

User-agent: Google-Extended
Allow: /
Disallow: /tag/

User-agent: anthropic-ai
Allow: /
Disallow: /tag/

User-agent: PerplexityBot
Allow: /
Disallow: /tag/

User-agent: OAI-SearchBot
Allow: /
Disallow: /tag/

User-agent: ChatGPT-User
Allow: /
Disallow: /tag/

User-agent: ClaudeBot
Allow: /
Disallow: /tag/
# 2026-09-12: обходил сотни страниц глубокой пагинации
# (/category/products/page/454/ и т.п.) — это создавало промахи кеша.
# Статьи и рубрики остаются открытыми.
Disallow: /*/page/
Disallow: /page/

User-agent: Applebot-Extended
Allow: /
Disallow: /tag/

User-agent: DuckAssistBot
Allow: /
Disallow: /tag/

User-agent: YouBot
Allow: /
Disallow: /tag/

User-agent: MistralAI-User
Allow: /
Disallow: /tag/

# https://itech-news.ru/llms.txt