{
  "title": "TIME AI crawler response check",
  "checkedAt": "2026-08-19T13:22:54Z",
  "method": {
    "request": "GET",
    "headers": {
      "Accept": "*/*",
      "Accept-Encoding": "identity"
    },
    "notes": [
      "Each request used the current full User-Agent string published by the operator, except Anthropic, which publishes the ClaudeBot token.",
      "Redirects were followed.",
      "Response bodies are not republished. SHA-256 hashes provide a receipt without copying the article.",
      "Results are a dated observation. A shortened User-Agent can produce a different response."
    ]
  },
  "targets": [
    {
      "url": "https://time.com/article/2026/08/04/morning-light-better-sleep/",
      "responses": [
        {
          "crawler": "Googlebot",
          "userAgent": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
          "status": 200,
          "contentType": "text/html; charset=utf-8",
          "bytes": 304804,
          "bodySha256": "0650117f39c9d0a39aefb8d0ecb785e0931534ee158e5176780a0d6970651145"
        },
        {
          "crawler": "ClaudeBot",
          "userAgent": "ClaudeBot",
          "status": 200,
          "contentType": "text/markdown; charset=utf-8",
          "bytes": 13409,
          "responseHeaders": {
            "x-mobian-format": "md",
            "x-mobian-tokens": "3323",
            "cache-control": "no-store",
            "vary": "Accept"
          },
          "bodySha256": "87a981dc8d4049c2931fafaaa82488a27b1ff30beea4f07fbe4591cb3aacaf58"
        },
        {
          "crawler": "ChatGPT-User",
          "userAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot",
          "status": 406,
          "bytes": 0,
          "bodySha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
        },
        {
          "crawler": "PerplexityBot",
          "userAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)",
          "status": 200,
          "contentType": "text/markdown; charset=utf-8",
          "bytes": 13409,
          "responseHeaders": {
            "x-mobian-format": "md",
            "x-mobian-tokens": "3323",
            "cache-control": "no-store",
            "vary": "Accept"
          },
          "bodySha256": "87a981dc8d4049c2931fafaaa82488a27b1ff30beea4f07fbe4591cb3aacaf58"
        },
        {
          "crawler": "OAI-SearchBot",
          "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36; compatible; OAI-SearchBot/1.4; +https://openai.com/searchbot",
          "status": 406,
          "bytes": 0,
          "bodySha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
        },
        {
          "crawler": "GPTBot",
          "userAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.4; +https://openai.com/gptbot",
          "status": 406,
          "bytes": 0,
          "bodySha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
        }
      ]
    },
    {
      "url": "https://time.com/collections/best-inventions-2025/",
      "responses": [
        {
          "crawler": "ClaudeBot",
          "userAgent": "ClaudeBot",
          "status": 200,
          "contentType": "text/markdown; charset=utf-8",
          "bytes": 33188,
          "responseHeaders": {
            "x-mobian-format": "md",
            "x-mobian-tokens": "8297",
            "cache-control": "no-store",
            "vary": "Accept"
          },
          "bodySha256": "1c21041d956b1a8fef2e1a1bf80ce620942e75a8c31dc5c5177dbc94880d9b3a"
        },
        {
          "crawler": "ChatGPT-User",
          "userAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot",
          "status": 406,
          "bytes": 0,
          "bodySha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
        },
        {
          "crawler": "PerplexityBot",
          "userAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)",
          "status": 200,
          "contentType": "text/markdown; charset=utf-8",
          "bytes": 33188,
          "responseHeaders": {
            "x-mobian-format": "md",
            "x-mobian-tokens": "8297",
            "cache-control": "no-store",
            "vary": "Accept"
          },
          "bodySha256": "c15521beed609d15613bba395090746e106bd0b94aea0bd78021f677483a0acf"
        }
      ]
    }
  ]
}