User-agent: *

User-agent: *

Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-login.php
Disallow: /wp-content/plugins/
Disallow: /?s=*  # поисковые запросы
Disallow: /readme.html
Disallow: /xmlrpc.php  # Защита от взлома
Disallow: /wp-json/    # Блокировка REST API (если не используется)

Allow: /wp-admin/admin-ajax.php  # важно для работы сайта!
Allow: /wp-content/uploads/      # разрешаем изображения
Allow: /wp-content/themes/*/css/ # разрешаем CSS
Allow: /wp-content/themes/*/js/  # разрешаем JS

Host: spbprompt.ru # Для Yandex
Disallow: /*?* # Блокировка параметров URL

Sitemap: https://spbprompt.ru/sitemap_index.xml
Sitemap: https://spbprompt.ru/post-sitemap.xml
Sitemap: https://spbprompt.ru/page-sitemap.xml
Sitemap: https://spbprompt.ru/sitemap.xml

# ======== Скрытые от поиска страницы (новые дописывыать) ======== #

Disallow: /admin
Disallow: /videos
Disallow: /episode3/admin
Disallow: /episode4/admin
Disallow: /docs2018/admin

Disallow: /docs2018
Disallow: /docs2019
Disallow: /docs2020
Disallow: /docs2021

Disallow: /episode1
Disallow: /episode2
Disallow: /episode3
Disallow: /episode4
Disallow: /episode5
Disallow: /episode6
Disallow: /episode7
Disallow: /episode8
Disallow: /episode07

Disallow: /translation333
Disallow: /translation2k

Disallow: /old_14_07_2017
Disallow: /old_translation2016
Disallow: /old_backup
Disallow: /backups/
Disallow: /retrospective
Disallow: /spik_do2020
Disallow: /spik2021
Disallow: /spik2022
Disallow: /spik2023

Disallow: /program9sezon

Disallow: /online
Disallow: /videos
Disallow: /video2k
Disallow: /videos2k

Disallow: /login_translation
Disallow: /login

Disallow: /becomeaspeaker
Disallow: /thankyou
Disallow: /thankpreregister
Disallow: /thankyouTEST

Disallow: /quote

Disallow: /soglas
Disallow: /privacy
Disallow: /personal
Disallow: /oferta
Disallow: /credit
Disallow: /covid-19
Disallow: /posleconf


Disallow: /test
Disallow: /test_result1
Disallow: /test_result2
Disallow: /test_result3
Disallow: /test_source
Disallow: /randompriz

Disallow: /Amoder
Disallow: /mastermind



# ======== Основные AI-краулеры ======== #

# OpenAI (ChatGPT, GPT-4)
User-agent: GPTBot
Allow: /
Disallow: /wp-admin/
Disallow: /private/
Crawl-delay: 5
# Источник: https://platform.openai.com/docs/gptbot

# Anthropic (Claude)
User-agent: ClaudeBot
User-agent: Claude-Web
Allow: /
Disallow: /wp-admin/
Disallow: /cgi-bin/
# Источник: https://www.anthropic.com/crawler

# Google AI (Bard, Vertex AI)
User-agent: Google-Extended
Allow: /
Disallow: /wp-admin/
Disallow: /search/
# Управление через GSC: https://developers.google.com/search/apis/indexing-api/v3/prereqs#opt-in

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Disallow: /wp-admin/
Disallow: /comments/
# Источник: https://docs.perplexity.ai/docs/crawler

# Common Crawl (основа для многих AI-моделей)
User-agent: CCBot
Allow: /
Disallow: /wp-admin/
Disallow: /?*
Crawl-delay: 10
# Источник: https://commoncrawl.org/big-picture/frequently-asked-questions/

# ======== Специализированные AI-краулеры ======== #

# You.com
User-agent: YouBot
Allow: /
Disallow: /wp-admin/

# Cohere AI
User-agent: cohere-ai
Allow: /
Disallow: /wp-admin/

# Neeva AI (теперь часть Snowflake)
User-agent: Neevabot
Allow: /
Disallow: /wp-admin/

# Seekr
User-agent: Seekrbot
Allow: /
Disallow: /wp-admin/

# Diffbot
User-agent: Diffbot
Allow: /
Disallow: /wp-admin/
Crawl-delay: 3

# ======== Поисковые системы с AI-функциями ======== #

# Bing AI (Copilot)
User-agent: bingbot
Allow: /
Disallow: /wp-admin/
# Источник: https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0

# Yandex AI (Алиса)
User-agent: YandexBot
Allow: /
Disallow: /wp-admin/
Host: spbprompt.ru
# Источник: https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html

# Baidu AI (文心一言)
User-agent: Baiduspider
Allow: /
Disallow: /wp-admin/
Crawl-delay: 5

# ======== Другие AI-краулеры ======== #

User-agent: anthropic-ai
Allow: /
Disallow: /wp-admin/

User-agent: llm-crawler
Allow: /
Disallow: /wp-admin/

User-agent: ChttrBot
Allow: /
Disallow: /wp-admin/

User-agent: OmniExplorer_Bot
Allow: /
Disallow: /wp-admin/