#!/usr/bin/env bash
# Fetch the full Cloudflare Radar Bot Directory via the official API
# Docs: https://developers.cloudflare.com/api/resources/radar/subresources/bots/methods/list/
#
# Requires: curl, jq
# Set your token: export CF_API_TOKEN="your_token_here"
# Get a free token at: https://dash.cloudflare.com/profile/api-tokens
#   (use the "Cloudflare Radar: Read" template, or grant radar:read)

set -euo pipefail

API_BASE="https://api.cloudflare.com/client/v4/radar/bots"
TOKEN="${CF_API_TOKEN:?Set CF_API_TOKEN env var}"
OUT_DIR="cf_bot_directory"
PAGE_SIZE=100   # max supported by the API

mkdir -p "$OUT_DIR"

echo "==> Fetching bot list (paginated)..."

page=0
total_fetched=0
all_slugs=()

while true; do
    response=$(curl -fsSL \
        -H "Authorization: Bearer $TOKEN" \
        -H "Accept: application/json" \
        "${API_BASE}?limit=${PAGE_SIZE}&offset=$((page * PAGE_SIZE))&format=json")

    # Check API success
    success=$(echo "$response" | jq -r '.success')
    if [[ "$success" != "true" ]]; then
        echo "API error: $(echo "$response" | jq -c '.errors')" >&2
        exit 1
    fi

    # Extract slugs from this page
    page_slugs=$(echo "$response" | jq -r '.result.bots[].slug // .result[].slug')
    count=$(echo "$page_slugs" | grep -c . || true)

    if [[ $count -eq 0 ]]; then
        break
    fi

    mapfile -t page_arr <<< "$page_slugs"
    all_slugs+=("${page_arr[@]}")
    total_fetched=$((total_fetched + count))

    echo "    Page $((page+1)): fetched $count bots (total so far: $total_fetched)"

    # Save raw page for reference
    echo "$response" | jq '.' > "${OUT_DIR}/page_$(printf '%03d' $page).json"

    # If fewer than PAGE_SIZE returned, we're done
    if [[ $count -lt $PAGE_SIZE ]]; then
        break
    fi

    page=$((page + 1))
    sleep 0.2   # be polite
done

echo "==> Total bots found: ${#all_slugs[@]}"

# Optionally fetch per-bot detail records
echo "==> Fetching per-bot details..."
details_dir="${OUT_DIR}/details"
mkdir -p "$details_dir"

for slug in "${all_slugs[@]}"; do
    out_file="${details_dir}/${slug}.json"
    if [[ -f "$out_file" ]]; then
        echo "    [skip] $slug (already fetched)"
        continue
    fi

    detail=$(curl -fsSL \
        -H "Authorization: Bearer $TOKEN" \
        -H "Accept: application/json" \
        "${API_BASE}/${slug}?format=json")

    echo "$detail" | jq '.' > "$out_file"
    echo "    Fetched: $slug"
    sleep 0.15
done

# Merge all details into a single JSON array
echo "==> Merging into ${OUT_DIR}/bots_all.json ..."
jq -s '[.[].result.bot]' "${details_dir}"/*.json > "${OUT_DIR}/bots_all.json"

total=$(jq 'length' "${OUT_DIR}/bots_all.json")
echo "==> Done. $total bots saved to ${OUT_DIR}/bots_all.json"

# Print a summary table
echo ""
echo "Category breakdown:"
jq -r '.[].category' "${OUT_DIR}/bots_all.json" | sort | uniq -c | sort -rn