#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# Per-language wildcards (e.g. /*/user/login) cover all 14 dinilu
# domains' secondary-language URL prefixes (/en/, /fr/, /de/, /nl/, etc.).
# Google supports * wildcards in Disallow values.
#
# For commerce endpoints (cart / checkout / order / quote) the rules
# use '/path$' (exact match) and '/path/' (sub-paths) instead of bare
# '/path', so static pages like /about-us/order-process don't get blocked
# by accident. Googlebot supports both '*' and '$' robots.txt operators.
#
# NOTE: The legacy Drupal-era directory blocks (/includes/, /misc/,
# /modules/, /profiles/, /scripts/, /themes/) are intentionally NOT
# listed below. Google needs to fetch CSS/JS/images from those paths
# to render and rank pages; blocking them caused GSC "Page resources"
# fetch failures. Real defense for raw CMS files is in .htaccess
# (deny .module/.install/.info + Options -Indexes), not robots.txt.
#
# For syntax checking, see:
# http://www.frobee.com/robots-txt-check

User-agent: *
# Files
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /MAINTAINERS.txt
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /xmlrpc.php
# Auth + admin paths (default-language at /, secondary languages at /<lang>/)
Disallow: /user/login
Disallow: /*/user/login
Disallow: /user/password
Disallow: /*/user/password
Disallow: /user/register
Disallow: /*/user/register
Disallow: /user/logout
Disallow: /*/user/logout
Disallow: /logout
Disallow: /*/logout
Disallow: /admin
Disallow: /*/admin
Disallow: /node/add
Disallow: /*/node/add
Disallow: /comment/reply
Disallow: /*/comment/reply
Disallow: /filter/tips
Disallow: /*/filter/tips
Disallow: /search
Disallow: /*/search
# Commerce endpoints — use $ + / so /about-us/order-process etc. aren't matched
Disallow: /cart$
Disallow: /cart/
Disallow: /*/cart$
Disallow: /*/cart/
Disallow: /checkout$
Disallow: /checkout/
Disallow: /*/checkout$
Disallow: /*/checkout/
Disallow: /order$
Disallow: /order/
Disallow: /*/order$
Disallow: /*/order/
Disallow: /quote$
Disallow: /quote/
Disallow: /*/quote$
Disallow: /*/quote/
# The /*/... rules further down are LANGUAGE-PREFIX rules (/en/quote,
# /de/order). But "*" in robots.txt matches any sequence INCLUDING "/", so
# "/*/quote$" also matches "/api/quote" — it was blocking the one endpoint
# the comment below invites AI agents to call. These Allow rules override it:
# Google picks the LONGEST matching pattern, so each must stay longer than the
# Disallow it beats (/api/quote = 10 chars vs /*/quote$ = 9). Do not shorten
# them to a bare "Allow: /api/" — 5 chars loses, and the block comes back.
# NB: robots.txt has no "?" wildcard, only "*" and "$", so "/??/quote" would
# be two literal question marks and would silently un-block /en/quote instead.
Allow: /api/
Allow: /api/quote
Allow: /api/products
Allow: /api/product/
Allow: /api/cart
Allow: /api/order
Allow: /api/search

# API endpoints: crawlable on purpose. They serve JSON, not pages, and are
# kept out of search results by an X-Robots-Tag: noindex header instead of
# a Disallow here. A Disallow blocked AI assistants from reading the quote
# API on a user's behalf - the very thing the LLMS-Sitemap line below
# invites them to do. Do not re-add Disallow: /api/.
# AI agents and LLMs: see our product catalogue and quote API description at:
# https://dinilu.ca/llms.txt  (summary)
# https://dinilu.ca/llms-full.txt  (full catalogue + API docs)
LLMS-Sitemap: https://dinilu.ca/llms-full.txt
# https://dinilu.ca/mcp  (Model Context Protocol endpoint, JSON-RPC 2.0 over POST)
Sitemap: https://dinilu.ca/fr/sitemap.xml
Sitemap: https://dinilu.ca/sitemap.xml