# Robots.txt for lucaberton.com
# Host: https://lucaberton.com
#
# NOTE: Cloudflare's "AI Audit / Content Signals" feature prepends a
# Cloudflare-managed block (with the non-standard "Content-Signal:" field)
# to this file at the edge. That block causes Lighthouse's robots-txt
# audit to report the file as invalid. To restore a perfect score,
# disable AI Audit in the Cloudflare dashboard:
#   Cloudflare -> AI Audit -> Content Signals -> off

User-agent: *
Disallow: /blog/search
Disallow: /blog/search/
Disallow: /blog/search?
Disallow: /blog/search/*
Disallow: /*?q=
# /slides/ intentionally NOT disallowed: PDFs carry X-Robots-Tag "noindex, follow"
# (see _headers), so crawlers can fetch them, honor noindex, and follow internal
# links without wasting link equity on a robots-blocked resource.
Disallow: /cdn-cgi/
Disallow: /private/
Disallow: /*?ref=
Disallow: /*?utm_
Disallow: /*?fbclid=
Allow: /

# AI crawlers — explicit allow (Cloudflare may add its own disallow block above)
User-agent: Google-Extended
Allow: /

User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

Sitemap: https://lucaberton.com/sitemap-index.xml
Sitemap: https://lucaberton.com/video-sitemap.xml