User-agent: *

# Allow crawling of public pages
Allow: /

# Block specific paths that should not be indexed
Disallow: /examples
Disallow: /ai-reddit-agent
Disallow: /freelance-job-feed

# Block private and auth areas
Disallow: /dashboard*
Disallow: /settings*
Disallow: /auth*
Disallow: /admin*
Disallow: /create-plan*
Disallow: /plan/*
Disallow: /all-projects
Disallow: /ai-agent-lab*
Disallow: /api*
Disallow: /upwork
Disallow: /_nuxt/
Disallow: /.nuxt/

# Block specific file types
Allow: /sitemap.xml
Disallow: *.json$
Disallow: *.xml$
Disallow: *.ts$
Disallow: *.js$
Disallow: *.vue$

# Crawl delay to be respectful
Crawl-delay: 1

# Sitemap location
Sitemap: https://devta.so/sitemap.xml

# AI Crawlers - Explicitly allow AI tools to access public content

User-agent: GPTBot
Allow: /
Crawl-delay: 1

User-agent: ChatGPT-User
Allow: /
Crawl-delay: 1

User-agent: CCBot
Allow: /
Crawl-delay: 1

User-agent: anthropic-ai
Allow: /
Crawl-delay: 1

User-agent: Claude-Web
Allow: /
Crawl-delay: 1

User-agent: Google-Extended
Allow: /
Crawl-delay: 1

User-agent: Gemini
Allow: /
Crawl-delay: 1

User-agent: Googlebot-Extended
Allow: /
Crawl-delay: 1

User-agent: PerplexityBot
Allow: /
Crawl-delay: 1

User-agent: YouBot
Allow: /
Crawl-delay: 1

User-agent: cohere-ai
Allow: /
Crawl-delay: 1

User-agent: meta-externalagent
Allow: /
Crawl-delay: 1

User-agent: Applebot-Extended
Allow: /
Crawl-delay: 1