# ==============================================================
# robots.txt — Beerka Research (beerka.co.id)
# Strategy: ALLOW all AI crawlers + search engines
# Goal: Answer Engine Optimization (AEO) & LLM brand visibility
# Last updated: June 2026
# ==============================================================


# --------------------------------------------------------------
# 1. SEARCH ENGINE CRAWLERS — Full Access
# --------------------------------------------------------------

User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /


# --------------------------------------------------------------
# 2. AI TRAINING & SEARCH CRAWLERS — Full Access
# Strategy: Izinkan semua AI crawler agar Beerka bisa dikutip
# sebagai sumber otoritatif di ChatGPT, Claude, Perplexity, dll.
# --------------------------------------------------------------

# OpenAI — ChatGPT training & search
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic — Claude training & search
User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: anthropic-ai
Allow: /

# Google — Gemini & AI Overviews training
User-agent: Google-Extended
Allow: /

# Meta — LLaMA training
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

# Perplexity — AI search engine
User-agent: PerplexityBot
Allow: /

# Common Crawl — Dataset untuk banyak open-source LLM
User-agent: CCBot
Allow: /

# Cohere — AI search & retrieval
User-agent: cohere-ai
Allow: /

# You.com AI search
User-agent: YouBot
Allow: /

# Bytedance / Grok
User-agent: Bytespider
Allow: /

# Apple — Applebot for AI features
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Amazon — Alexa & AI
User-agent: ia_archiver
Allow: /

# Diffbot — AI content extraction
User-agent: Diffbot
Allow: /

# Webz.io
User-agent: Webzio-Extended
Allow: /


# --------------------------------------------------------------
# 3. GLOBAL FALLBACK — Allow semua crawler lain
# --------------------------------------------------------------

User-agent: *
Allow: /

# Jangan blokir halaman apapun dari indexing


# --------------------------------------------------------------
# 4. BLOKIR PATH TEKNIS (bukan konten)
# Hanya blokir backend & file teknis WordPress
# --------------------------------------------------------------

User-agent: *
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-json/
Disallow: /xmlrpc.php
Disallow: /feed/
Disallow: /trackback/
Disallow: /?s=
Disallow: /search/
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /readme.html
Disallow: /license.txt
Disallow: /dashboard-bpd-diy/


# --------------------------------------------------------------
# 5. SITEMAPS — Eksplisit untuk semua crawler
# --------------------------------------------------------------

Sitemap: https://beerka.co.id/sitemap.xml
Sitemap: https://beerka.co.id/sitemap_index.xml
Sitemap: https://beerka.co.id/page-sitemap.xml
Sitemap: https://beerka.co.id/post-sitemap.xml


# --------------------------------------------------------------
# 6. LLMs.txt — Referensi untuk AI yang support standar ini
# --------------------------------------------------------------

# LLMs.txt: https://beerka.co.id/llms.txt