# Qasim Zahid Portfolio - Robots.txt
# This file helps search engines crawl and index the website efficiently

# Allow all search engines to crawl the site
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /private/
Disallow: /temp/
Disallow: /*.pdf$
Disallow: /*.zip$

# Specify crawl delay for bots (in seconds)
Crawl-delay: 1

# Specific rules for Google
User-agent: Googlebot
Allow: /
Crawl-delay: 0

# Specific rules for Bing
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Prevent bots from crawling private directories
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /?s=
Disallow: /search/
Disallow: /author/
Disallow: /comment/

# Sitemap location - CRITICAL for SEO
Sitemap: https://qasimzahid.com/sitemap.xml

# Cache control
# Allows Google to cache pages for faster indexing
User-agent: Googlebot-Image
Allow: /

# Allow specific mobile crawlers
User-agent: MobileBot
Allow: /

# Block bad bots and scrapers
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Blekkobot
Disallow: /

# Default for any other bots not specified
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /private/