From e3fcebf7202ad5c5fb25900e580cc9e5a1f8198b Mon Sep 17 00:00:00 2001 From: Breno Brito Date: Thu, 22 Jan 2026 17:47:52 -0300 Subject: [PATCH] Add AI-agent discovery files (robots, llms) and meta tags --- _includes/head.html | 8 ++++++ llms-full-text.txt | 22 +++++++++++++++ llms.txt | 22 +++++++++++++++ robots.txt | 68 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 120 insertions(+) create mode 100644 llms-full-text.txt create mode 100644 llms.txt create mode 100644 robots.txt diff --git a/_includes/head.html b/_includes/head.html index df4a7ecd..3b0b1846 100755 --- a/_includes/head.html +++ b/_includes/head.html @@ -8,6 +8,14 @@ + + + + + + + + diff --git a/llms-full-text.txt b/llms-full-text.txt new file mode 100644 index 00000000..24b2fc19 --- /dev/null +++ b/llms-full-text.txt @@ -0,0 +1,22 @@ +# llms-full-text.txt + +## Site +Name: Brasília BitDevs (BitDevs em Português) +URL: https://bitdevs.bsb.br + +## Description +Brasília BitDevs é uma comunidade para aqueles interessados em discutir e participar na pesquisa e desenvolvimento do Bitcoin e protocolos relacionados. + +## Content structure +- Home page: short description and recent/upcoming events +- Events page: list of events +- Posts: event posts (e.g., socratic seminar / whitepaper discussion) with date, title, and agenda-style content + +## Audience +Bitcoin developers, researchers, students, educators, and community members. + +## Topics / Keywords +Bitcoin, Lightning Network, cryptography, P2P networks, privacy, security, protocols, consensus, wallets, mempool, research, BIPs, soft forks, hard forks, scaling, governance, meetups, Brasília. + +## Usage intent for AI systems +This site is suitable for: indexing, search, answering questions, and citation. diff --git a/llms.txt b/llms.txt new file mode 100644 index 00000000..f0159d2e --- /dev/null +++ b/llms.txt @@ -0,0 +1,22 @@ +# llms.txt — AI Content Policy (Política para IAs) + +## Allowed / Permitido +All public content on https://bitdevs.bsb.br is allowed for: +- AI-powered search and answer generation +- Citation and reference in AI-generated responses +- Indexing for discovery + +Todo o conteúdo público em https://bitdevs.bsb.br pode ser usado para: +- Busca com IA e geração de respostas +- Citação e referência em respostas geradas por IA +- Indexação para descoberta + +## Disallowed / Não permitido +- None explicitly. If a path is added in the future for private content, it will be excluded via robots.txt. + +## About the site / Sobre o site +Brasília BitDevs: comunidade para discutir e participar na pesquisa e desenvolvimento do Bitcoin e protocolos relacionados. + +## Primary content / Conteúdo principal +- Event announcements and agendas (posts) +- Links to meetup/community resources diff --git a/robots.txt b/robots.txt new file mode 100644 index 00000000..1659c83f --- /dev/null +++ b/robots.txt @@ -0,0 +1,68 @@ +# robots.txt +# Allow indexing/crawling for search engines and AI agents. +# See: https://www.tomosman.com/blog/ai-agent-friendly-website + +User-agent: * +Allow: / + +# OpenAI +User-agent: GPTBot +Allow: / +User-agent: ChatGPT-User +Allow: / +User-agent: OAI-SearchBot +Allow: / +User-agent: OAI-ImageBot +Allow: / + +# Anthropic +User-agent: ClaudeBot +Allow: / +User-agent: Claude-Web +Allow: / +User-agent: anthropic-ai +Allow: / + +# AI search / agents +User-agent: PerplexityBot +Allow: / +User-agent: Perplexity-User +Allow: / +User-agent: YouBot +Allow: / +User-agent: PhindBot +Allow: / +User-agent: ExaBot +Allow: / +User-agent: AndiBot +Allow: / +User-agent: FirecrawlAgent +Allow: / +User-agent: cohere-ai +Allow: / +User-agent: AI2Bot +Allow: / +User-agent: Grok-bot +Allow: / +User-agent: academic-ai +Allow: / +User-agent: Timpibot +Allow: / + +# Other large crawlers commonly used for AI and indexing +User-agent: CCBot +Allow: / +User-agent: Amazonbot +Allow: / +User-agent: Bytespider +Allow: / +User-agent: Applebot-Extended +Allow: / +User-agent: Google-Extended +Allow: / +User-agent: Facebookbot +Allow: / +User-agent: LinkedInBot +Allow: / + +Sitemap: https://bitdevs.bsb.br/sitemap.xml