# llms.txt for https://www.colinst.com
# Purpose: Declare permissions and preferences for Large Language Models (LLMs)
# Scope: AI training, dataset construction, long-term caching, and automated reuse
# Notes: This file complements robots.txt. When in doubt, this file is authoritative for AI/LLM usage.

version: 1.0
site: https://www.colinst.com
owner: ColinST (Colinst)
contact: mailto:hello@colinst.com
last_updated: 2025-09-11
policy_default: disallow_training, disallow_dataset, allow_ephemeral_access

###############################################################################
# Global policy
###############################################################################
[global]
# Human-usable browsing by end users (short-lived retrieval to render a page)
access: allow_ephemeral
# Use of content to train, fine-tune, or improve models
training: disallow
# Use of content to build or enrich datasets, embeddings corpora, or indexes
dataset: disallow
# Long-term caching or storage of page text, media, or metadata
cache: disallow
# Quoting: short excerpts (<= 200 chars) with source attribution are allowed
quoting: allow_short_with_attribution
# Derivatives: generating summaries/paraphrases based on live fetch (no storage)
derivatives: allow_ephemeral_only
# Attribution requirements for any allowed use
attribution: required
# Rate limits for automated agents that respect this file
rate_limit:
  requests_per_minute: 10
  burst: 20

###############################################################################
# Canonical resources & sitemaps
###############################################################################
[resources]
sitemaps:
  - https://www.colinst.com/sitemap.xml
  - https://www.colinst.com/sitemap.txt
preferred_canonical: https://www.colinst.com/

###############################################################################
# Path-specific rules (override global when listed)
###############################################################################
# Common sensitive or non-content paths are explicitly blocked for any AI use.
[paths]
disallow:
  - /admin/
  - /wp-admin/
  - /wp-json/
  - /cart
  - /checkout
  - /account
  - /login
  - /logout
  - /api/
  - /cdn-cgi/
  - /search
  - /privacy
  - /terms
  - /legal
  - /_next/
  - /static/
  - /assets/

# Example: If you have a public docs area you WANT LLMs to summarize in-session
# but still not train on, you can allow ephemeral with no training:
# allow_ephemeral_only:
#   - /docs/
#   - /knowledge-base/

###############################################################################
# Bot-/vendor-specific directives (override global for each agent)
###############################################################################
# OpenAI
[agent "GPTBot"]
training: disallow
dataset: disallow
cache: disallow
access: allow_ephemeral

# OpenAI Search crawler (if applicable)
[agent "OAI-SearchBot"]
training: disallow
dataset: disallow
cache: disallow
access: allow_ephemeral

# Google AI (Gemini) training control
[agent "Google-Extended"]
training: disallow
dataset: disallow
cache: disallow

# Anthropic
[agent "ClaudeBot"]
training: disallow
dataset: disallow
cache: disallow
access: allow_ephemeral

[agent "Claude-Web"]
training: disallow
dataset: disallow
cache: disallow
access: allow_ephemeral

# Perplexity
[agent "PerplexityBot"]
training: disallow
dataset: disallow
cache: disallow
access: disallow   # block autonomous fetching if desired

# Common Crawl (feeds many AI datasets)
[agent "CCBot"]
training: disallow
dataset: disallow
cache: disallow
access: disallow

# Apple
[agent "Applebot-Extended"]
training: disallow
dataset: disallow
cache: disallow

# Meta (Facebook/Instagram) AI crawling
[agent "Meta-External"]
training: disallow
dataset: disallow
cache: disallow

# ByteDance
[agent "Bytespider"]
training: disallow
dataset: disallow
cache: disallow
access: disallow

# Amazon
[agent "Amazonbot"]
training: disallow
dataset: disallow
cache: disallow

# CoCounselors / misc research bots (add as discovered)
[agent "*"]
training: disallow
dataset: disallow
cache: disallow
access: allow_ephemeral
respect_robots_txt: true

###############################################################################
# Attribution & citation guidance
###############################################################################
[attribution]
required_fields:
  - source_url
  - site_name: "ColinST"
  - title
  - retrieved_at
excerpt_limit_chars: 200
linkback: https://www.colinst.com

###############################################################################
# Licensing statement
###############################################################################
[licensing]
copyright: "© ColinST. All rights reserved."
permitted_use:
  - "End-user ephemeral browsing by humans or assistants acting on behalf of a user session."
prohibited_use:
  - "Training, fine-tuning, benchmarking, red-teaming dataset creation."
  - "Any form of long-term storage, vectorization for reuse, or redistribution."
  - "Commercial resale or bundling of site content."

###############################################################################
# Enforcement & contact
###############################################################################
[enforcement]
request_compliance: true
violation_reporting: mailto:hello@colinst.com