Free AI crawler checker, no signup

Free AI Crawler Checker Can ChatGPT, Claude and Perplexity read your page? Test the facts your customers ask about, from prices and stock to services and opening hours.

Most AI crawlers do not run JavaScript. If a price, stock label, service list or opening hours are added by a script, they never see them. Tell the tool what kind of business you run and what type of page you are checking, and it fetches the raw HTML the way a crawler does, tests the facts that page should carry, and tells you what to fix first. A shop's product page is tested for price, stock and specs; a law firm's service page for what the service covers, fees and contact details.

Fetches as GPTBot, ClaudeBot, PerplexityBot and more10 industries, 22 page typesProducts, categories, services, locations, menus and articles

Check what AI crawlers can read

Takes about 10 seconds. No script on your page is run.

1. What kind of business is the page for?

The checks change with your answer. A shop, a clinic and a law firm need different facts in their HTML.

2. What type of page is it?

3. Which page should we check?

A quick Cloudflare check confirms you are a person before a live scan runs. It keeps bots from using this tool to crawl other sites.

'; function setMsg(t, c) { var m = $('#ppxMsg'); m.textContent = t || ''; m.className = 'msg' + (c ? ' ' + c : ''); } function normUrl(v) { v = (v || '').trim(); if (!v) return ''; return /^https?:\/\//i.test(v) ? v : 'https://' + v; } function cleanPaste(t) { var a = t.search(/ 0) t = t.slice(a); var b = t.toLowerCase().lastIndexOf(''); if (b > 0) t = t.slice(0, b + 7); return t; } /* ---------- industry + mode ---------- */ function btn(attr, v, on, label) { return ''; } function curPage() { var ps = IND[state.industry].pages; return ps.filter(function (p) { return p.id === state.page; })[0] || ps[0]; } function drawInd() { $('#ppxInd').innerHTML = Object.keys(IND).map(function (k) { return btn('data-k', k, k === state.industry, IND[k].label); }).join(''); var p = curPage(); state.page = p.id; $('#ppxPage').innerHTML = IND[state.industry].pages.map(function (x) { return btn('data-p', x.id, x.id === p.id, x.label); }).join(''); $('#ppxPageBlurb').textContent = p.blurb; $('#ppxUrl').placeholder = p.placeholder; $('#ppxHint').textContent = p.hint + ' A bare domain such as example.com works.'; } $('#ppxInd').addEventListener('click', function (e) { var b = e.target.closest('button[data-k]'); if (!b) return; state.industry = b.getAttribute('data-k'); state.page = IND[state.industry].pages[0].id; drawInd(); }); $('#ppxPage').addEventListener('click', function (e) { var b = e.target.closest('button[data-p]'); if (!b) return; state.page = b.getAttribute('data-p'); drawInd(); }); function applyMode() { var paste = state.mode === 'paste'; $('#ppxPasteBox').hidden = !paste; $('#ppxGate').hidden = paste; [].forEach.call($('#ppxMode').children, function (b) { b.classList.toggle('on', b.getAttribute('data-v') === state.mode); }); if (!paste) { loadTurnstile(); if (!WORKER_URL) setMsg('Live scanning is not connected on this copy of the page. Use “Paste raw HTML” to test a page now.', ''); else setMsg('', ''); } else setMsg('Pasted HTML is analysed in your browser. Nothing is sent anywhere.', ''); } $('#ppxMode').addEventListener('click', function (e) { var b = e.target.closest('button[data-v]'); if (!b) return; state.mode = b.getAttribute('data-v'); applyMode(); }); /* ---------- Cloudflare Turnstile: verify the visitor is human before a live scan ---------- */ function loadTurnstile() { var box = $('#ppxTs'); if (!TURNSTILE_SITEKEY) { box.innerHTML = 'Human verification is not configured on this copy of the page (add the Turnstile site key).'; return; } if (state.tsId !== null || state.tsLoading) return; state.tsLoading = true; var go = function () { state.tsLoading = false; if (!window.turnstile) { box.innerHTML = 'The Cloudflare check could not load. Refresh the page or use “Paste raw HTML”.'; return; } state.tsId = window.turnstile.render('#ppxTs', { sitekey: TURNSTILE_SITEKEY, theme: 'light', appearance: 'always', callback: function (t) { state.token = t; if ($('#ppxMsg').classList.contains('err')) setMsg('', ''); }, 'expired-callback': function () { state.token = ''; }, 'error-callback': function () { state.token = ''; setMsg('The Cloudflare check failed. Reload the page and try again.', 'err'); } }); }; if (window.turnstile) { go(); return; } var s = document.createElement('script'); s.src = 'https://challenges.cloudflare.com/turnstile/v0/api.js?render=explicit'; s.async = true; s.onload = go; s.onerror = function () { state.tsLoading = false; box.innerHTML = 'The Cloudflare check could not load. Refresh the page or use “Paste raw HTML”.'; }; document.head.appendChild(s); } function resetTurnstile() { state.token = ''; try { if (window.turnstile && state.tsId !== null) window.turnstile.reset(state.tsId); } catch (e) { } } /* ---------- report ---------- */ var PILL = { pass: 'Pass', warn: 'Warn', fail: 'Fail', info: 'Skipped' }; function gstats(list) { var sc = list.filter(function (c) { return c.status !== 'info'; }); var pass = sc.filter(function (c) { return c.status === 'pass'; }).length; var pts = sc.reduce(function (a, c) { return a + (c.status === 'pass' ? 1 : c.status === 'warn' ? 0.5 : 0); }, 0); return { pass: pass, total: sc.length, pct: sc.length ? Math.round(100 * pts / sc.length) : null }; } function vcls(s) { return s >= 80 ? 'pass' : s >= 55 ? 'cond' : 'fail'; } function chk(list) { return '
    ' + list.map(function (c) { return '
  • ' + PILL[c.status] + '
    ' + esc(c.label) + '' + esc(c.evidence) + '
    ' + (c.source && c.status !== 'info' ? '' + esc(c.source) + '' : '') + '
  • '; }).join('') + '
'; } var lastReport = null; function render(r, info) { lastReport = { r: r, info: info }; var all = r.groups.concat([r.industryGroup]); var bars = all.map(function (g) { var s = gstats(g.checks); return s.pct == null ? '
' + esc(g.label) + '–
' : '
' + esc(g.label) + '' + s.pass + '/' + s.total + '
'; }).join(''); var ind = gstats(r.industryGroup.checks); var fixes = r.steps.length ? r.steps.map(function (s, i) { return '
' + (i + 1) + '
' + esc(s.label) + ' ' + (s.status === 'fail' ? 'Fix first' : 'Improve') + '

' + esc(s.fix) + '

' + esc(s.evidence) + '
'; }).join('') : '

Nothing to fix. Re-run this check after every template or plugin change.

'; var blocked = r.groups[0].checks.some(function (c) { return c.status === 'fail'; }); var ents = r.entities.groups.map(function (g) { return '
' + esc(g.group) + '
    ' + g.items.map(function (i) { return '
  • ' + esc(i) + '
  • '; }).join('') + '
'; }).join(''); if (r.entities.widgets.length) ents += '
Needs JavaScript to show content
    ' + r.entities.widgets.map(function (i) { return '
  • ' + esc(i) + '
  • '; }).join('') + '
'; $('#ppxRes').innerHTML = '
' + (info.example ? 'Example result · made-up sample product page
' : '') + '
AI retrievability · ' + esc(r.verdict) + '

' + esc(r.headline) + '

' + esc(info.line) + '

' + '
' + r.scores.overall + '/100Overall score
' + (ind.total ? ind.pass + '/' + ind.total : '–') + '' + esc(r.page.label) + ' facts found
' + r.steps.length + 'Fixes to make
' + r.wordCount + 'Words a crawler reads
' + '
' + bars + '
' + '
' + '' + '' + '' + '' + '
' + '

Your next steps

Ordered by how much each one limits what AI crawlers can read.

' + fixes + '

' + (blocked ? 'A crawler is being blocked before it reads anything. Check your firewall or bot-protection settings, then run the check again. ' : '') + 'After fixing, test again as ClaudeBot and PerplexityBot, not only GPTBot.

' + '

' + esc(r.industry.label) + ' · ' + esc(r.page.label) + ' facts in raw HTML

' + esc(r.industry.blurb) + '

' + chk(r.industryGroup.checks) + '
' + '

What the crawler reads

The start of the text left once scripts and styles are removed (' + r.wordCount + ' words in total).

' + esc(r.preview || 'No readable text found.') + '
' + '

Technical checks

Access, content, metadata, schema and links, as a crawler sees them.

' + r.groups.map(function (g) { return '

' + esc(g.label) + '

' + chk(g.checks); }).join('') + '
' + '

Entities found in the HTML

Names, contact details, prices and schema types a crawler can lift from this page.

' + (ents || '

No entities found.

') + '
'; } $('#ppxRes').addEventListener('click', function (e) { var t = e.target.closest('#ppxTabs button'); if (t) { var n = t.getAttribute('data-t'); [].forEach.call($('#ppxTabs').children, function (b) { b.classList.toggle('on', b === t); }); [].forEach.call(document.querySelectorAll('#ppxRes .ppx-tab'), function (p) { p.classList.toggle('on', p.getAttribute('data-p') === n); }); return; } var c = e.target.closest('#ppxCopy'); if (c && lastReport) copyReport(lastReport.r, lastReport.info, c); }); function reportText(r, info) { var L = ['AI Crawler Checker', info.line, '', 'Verdict: ' + r.verdict + ' (' + r.scores.overall + '/100)', r.headline, '', 'Next steps:']; r.steps.forEach(function (s, i) { L.push((i + 1) + '. ' + s.label + ': ' + s.fix + (s.evidence ? ' [' + s.evidence + ']' : '')); }); L.push('', r.industry.label + ' · ' + r.page.label + ' facts:'); r.industryGroup.checks.forEach(function (c) { L.push('- [' + c.status.toUpperCase() + '] ' + c.label + ': ' + c.evidence); }); return L.join('\n'); } function copyReport(r, info, btn) { var t = reportText(r, info); var done = function (m) { btn.textContent = m; setTimeout(function () { btn.textContent = 'Copy report'; }, 2200); }; var fb = function () { var ta = document.createElement('textarea'); ta.value = t; ta.setAttribute('readonly', ''); ta.style.cssText = 'position:fixed;left:0;top:0;width:1px;height:1px;opacity:0'; document.body.appendChild(ta); ta.select(); var ok = false; try { ok = document.execCommand('copy'); } catch (x) { } document.body.removeChild(ta); done(ok ? 'Copied' : 'Press Ctrl+C on the report'); }; if (navigator.clipboard && navigator.clipboard.writeText) navigator.clipboard.writeText(t).then(function () { done('Copied'); }, fb); else fb(); } function run(html, extra, line) { var r = AICheck.analyze(html, { industry: state.industry, page: state.page, url: extra.url, fetch: extra.fetch }); render(r, { line: line }); var el = $('#ppxRes'); if (el && el.scrollIntoView) el.scrollIntoView({ behavior: matchMedia('(prefers-reduced-motion:reduce)').matches ? 'auto' : 'smooth', block: 'start' }); } async function live(url, bot, token) { var ctl = new AbortController(); var t = setTimeout(function () { ctl.abort(); }, 35000); try { var res = await fetch(WORKER_URL.replace(/\/$/, '') + '/?url=' + encodeURIComponent(url) + '&bot=' + encodeURIComponent(bot) + '&token=' + encodeURIComponent(token), { signal: ctl.signal }); var j = await res.json(); if (!j.ok) throw new Error(j.error || 'The page could not be fetched.'); return j; } finally { clearTimeout(t); } } $('#ppxForm').addEventListener('submit', async function (e) { e.preventDefault(); var go = $('#ppxGo'), bot = $('#ppxBot').value; try { if (state.mode === 'paste') { var raw = cleanPaste($('#ppxRaw').value || ''); if (raw.length < 100) { setMsg('Paste the full raw HTML first. Run the curl command shown under the box and paste its output.', 'err'); return; } var u = normUrl($('#ppxUrl').value); setMsg('', ''); run(raw, { url: u || null, fetch: null }, 'Pasted HTML · ' + Math.round(raw.length / 1024) + ' KB' + (u ? ' · ' + u : '') + ' · live checks (HTTP, robots.txt, headers) skipped'); return; } var url = normUrl($('#ppxUrl').value); if (!url) { setMsg('Enter a domain or page URL.', 'err'); $('#ppxUrl').focus(); return; } if (!WORKER_URL) { setMsg('Live scanning is not connected on this copy of the page. Use “Paste raw HTML” to test a page now.', 'err'); return; } if (!TURNSTILE_SITEKEY) { setMsg('Human verification is not configured, so live scans are switched off.', 'err'); return; } if (!state.token) { setMsg('Complete the Cloudflare check above first, then press Analyse page.', 'err'); return; } go.disabled = true; go.textContent = 'Fetching as ' + bot + '…'; setMsg('Requesting the page as ' + bot + '. This takes a few seconds.', ''); var tok = state.token; resetTurnstile(); var j = await live(url, bot, tok); var line = j.finalUrl + ' · fetched as ' + j.bot + ' · HTTP ' + j.status + ' · ' + Math.round(j.bytes / 1024) + ' KB' + (j.truncated ? ' (cut at 2 MB)' : '') + (j.redirects && j.redirects.length ? ' · ' + j.redirects.length + ' redirect(s)' : ''); setMsg('', ''); run(j.html, { url: j.finalUrl, fetch: { status: j.status, ttfbMs: j.ttfbMs, bot: j.bot, bytes: j.bytes, baseline: j.baseline, headers: j.headers, robots: j.robots, finalUrl: j.finalUrl } }, line); } catch (err) { resetTurnstile(); setMsg((err && err.name === 'AbortError') ? 'The check timed out. Try again or paste the raw HTML.' : (err && err.message) || 'Something went wrong. Try again.', 'err'); } finally { go.disabled = false; go.textContent = 'Analyse page'; } }); drawInd(); applyMode(); /* opening state: a worked example so the layout of a real report is visible */ (function () { var r = AICheck.analyze(SAMPLE, { industry: 'ecommerce', page: 'product', url: 'https://shop.example/products/aurora-trail-backpack', fetch: null }); render(r, { example: true, line: 'shop.example/products/aurora-trail-backpack · enter your own page above to run a real check' }); })(); })();
What you get

What this AI crawler checker tells you

Raw HTML visibility
Whether the facts a customer asks about are present in the first HTML response from your server, before any JavaScript runs. For a shop that is price, stock and specs; for a service business it is what you offer, where, for how much and how to book.

Googlebot renders pages, so a shop can look perfect in Google and still be blank to ChatGPT, Claude and Perplexity. The tool requests your page with a crawler's user agent, reads the response as plain markup, and reports what is there and what is missing. The checks change with the industry and the page type you pick: a product page is tested for price, stock and specs, a category page for crawlable product links and prices, a service page for what the service includes and costs, a location page for address and hours, and a restaurant menu for dishes and prices.

Names and prices

Product titles and prices, plan prices, menu prices, room rates, consultation fees or "from" prices for a service, whichever your page should carry, in the visible HTML and in schema.

Stock and availability

"In stock" or "Only 3 left" on a product page; opening hours, service area and booking links on a service or location page.

Specs and service details

Spec tables for products, feature lists for software, treatment and practitioner details for clinics, and a full description of what a service includes.

Listings and categories

On category, collection, search results and blog hub pages: whether every item is a crawlable link with its name and price, or an empty grid filled by JavaScript.

Crawler access and firewalls

The HTTP status each bot receives, Cloudflare or other firewall (WAF) challenges, silent empty responses, how fast the HTML arrives, and what robots.txt allows for OAI-SearchBot, ChatGPT-User, ClaudeBot, PerplexityBot and GoogleOther.

A clean <head>

Whether the title, meta description, canonical and H1 are in the raw response, and whether a stray tag closes the <head> early so the metadata after it is ignored.

One set of facts everywhere

Whether the price and stock in the visible HTML match the JSON-LD. Keep your Google Merchant Center feed the same as both.

Agentic commerce fields

On product pages: priceValidUntil, shippingDetails.deliveryTime and merchantReturnDays in schema, plus sizing guides and warranty terms as text.

Structured data

Valid JSON-LD for the type that fits your industry, duplicate or empty schema blocks, and values that disagree with the page.

Script-only widgets

Review, booking and stock widgets that arrive by script and show a crawler an empty box.

Entities

The names, contact details, prices and schema types a crawler can lift from the page, so you can see what an AI answer could quote.

A ranked fix list

Failures that hide price, stock or contact details come first, then schema and markup clean-up.

The evidence

Why AI crawlers need your facts in raw HTML

When someone asks ChatGPT for a hiking backpack under £100, or a chauffeur in Berkshire who covers Heathrow, the answer is built from text a crawler could read. A price, stock label, service area or phone number that only appears after JavaScript runs is not in that text, and a competitor with the same facts in plain HTML gets cited instead. The research agrees.

searchVIU, November 202569% of the AI crawlers in its dataset cannot execute JavaScript. GPTBot, ClaudeBot, PerplexityBot and Meta-ExternalAgent read static HTML only. Read the analysis
Vercel and MERJ, December 2024None of the major AI crawlers rendered JavaScript. GPTBot made 569 million fetches and ClaudeBot 370 million in the month studied; Applebot was the exception because it renders like Googlebot. Read the study
Google Search CentralGoogle renders JavaScript, yet still recommends server-side or pre-rendering, because not all bots can run it. Read the guidance
Google Search CentralProduct structured data can surface price, availability, ratings and shipping in richer results, including Google Images and Lens. Read the documentation
OpenAIOAI-SearchBot, GPTBot and ChatGPT-User do different jobs and can each be allowed or blocked separately in robots.txt. Read the crawler overview
CrawlerOperatorJobRuns JavaScript?
GPTBotOpenAIModel trainingNo (Vercel and MERJ)
OAI-SearchBotOpenAISurfaces sites in ChatGPT searchNo (Vercel and MERJ)
ChatGPT-UserOpenAIFetches a page when a person asksNo (Vercel and MERJ)
ClaudeBotAnthropicModel trainingNo (Vercel and MERJ)
Claude-SearchBotAnthropicSearch indexingTreat as no
PerplexityBotPerplexitySearch indexingTreat as no
CCBotCommon CrawlOpen web archive used in training setsTreat as no
GooglebotGoogleSearch and AI OverviewsYes
ApplebotAppleSearch and SiriYes (Vercel and MERJ)

Official crawler pages: OpenAI, Anthropic and Perplexity.

By industry

What each industry and page type must show in raw HTML

IndustryPage typesFacts checked in raw HTMLSchema expected
E-commerceProduct, category or collectionProduct page: title, price, currency, stock level, specs, SKU or GTIN, brand, reviews, shipping, price and stock matching the schema, priceValidUntil, deliveryTime, merchantReturnDays, sizing and warranty. Category page: quick answer, product links, prices, intro copy, paginationProduct, Offer, CollectionPage, ItemList
SaaS / softwarePricing, feature or productPlans, prices, features, integrations, trial or demo link, security claimsSoftwareApplication, Offer
Local servicesHomepage, service, location or areaName, address, phone, opening hours, services, service area, price guidance, reviewsLocalBusiness, Service
Restaurant / cafeMenu, homepage or locationMenu items with prices, cuisine, hours, address, reservation link, dietary informationRestaurant, Menu
Healthcare / clinicTreatment, practitioner profile, clinic homepageTreatments, practitioners, credentials, location, hours, booking, medical disclaimerMedicalClinic, Physician
Real estateProperty listing, search results or areaListing price, bedrooms, bathrooms, size, address, sale or rental status, agent contactRealEstateListing
Hotels / travelHotel homepage, room, tour or packageRoom types, rates, amenities, check-in times, policies, booking linkHotel, LodgingBusiness
Publisher / blogArticle, category or topic hubHeadline, author, dates, full article body, outbound citationsArticle, BlogPosting, NewsArticle
Professional servicesService page, homepageServices, named clients, credentials, pricing model, contact details, processProfessionalService, Service
Other businessHomepage, service or offer pageWhat you do, who it is for, price, reviews, contact details, FAQOrganization, Service
How it works

How the checker reads your page

  1. Verify you are human

    Cloudflare Turnstile runs a quick check on live scans, so the tool cannot be used by bots to crawl other people's sites.

  2. Fetch as a crawler

    A Cloudflare Worker requests your URL with the user agent you choose, follows redirects, and also reads robots.txt and the response headers.

  3. Read the raw HTML

    The response is parsed as plain markup. No script runs, so the text left is what a non-rendering crawler gets.

  4. Test the facts for your page type

    The tool picks the tests for your industry and page type. Each fact is looked for in visible text and in JSON-LD, then labelled as found in the page, found only in schema, or missing.

  5. Rank the fixes

    You get a score, a verdict and a short list of what to change first.

One limit to know. The fetch comes from Cloudflare's network, not from the crawlers' own published IP ranges. A site that verifies bots by IP can still block the real crawler even when this test passes.

If your page fails

How to get facts into the HTML

Every fix comes down to one rule: the HTML response from your server must already contain the fact.

Fix the firewall first

Check that Cloudflare, Vercel or your host's bot rules do not answer AI user agents with a 403, a challenge page or an empty response. A blocked crawler reads nothing.

Clean up the <head>

Move body tags such as div, img and iframe out of the <head> and close every tag, so the title, description and canonical stay where crawlers expect them.

Render on the server

Use server-side rendering, static generation or a prerender service for pages built with React, Vue or Angular. Keep client-side rendering for extras such as chat widgets.

Print facts in the template

Output price, stock, specs, address and hours from the theme or template, not from an API call made in the browser. Avoid loading specs in tabs by script.

Mirror facts in JSON-LD

Add Product, LocalBusiness or the type for your industry to the HTML response. Keep price, stock and specs identical across the page, the schema and your Merchant Center feed.

Lead with a quick answer

Open category and service pages with a 2 to 3 sentence answer that AI engines can lift as a passage.

Replace script-only widgets

Print a few reviews, rates or stock lines as text beside widgets that load by script.

WordPress and Elementor

Look for duplicate schema from two plugins, heavy inline CSS, and links missing a leading slash. Keep one schema plugin.

Shopify and WooCommerce

Themes usually render price and stock on the server. Review apps, bundle apps and custom variant pickers are the usual causes of empty HTML.

FAQ

AI crawler HTML questions, answered

Can ChatGPT, Claude and Perplexity read JavaScript?

Mostly not. searchVIU found that 69% of the AI crawlers in its dataset cannot execute JavaScript, and in the Vercel and MERJ crawl study none of the major AI crawlers rendered it. GPTBot and ClaudeBot fetched JavaScript files in some cases but did not run them. Googlebot and Applebot do render pages. If your prices, stock, services or opening hours only appear after a script runs, most AI crawlers will not see them.

How does the AI crawler checker work?

Pick your industry and the type of page, enter its URL and choose a crawler. The tool fetches the raw HTML with that crawler's user agent, does not run any scripts, and checks that the facts that page should carry appear in the source. You get a score, a verdict and a ranked list of fixes.

Is it only for online shops?

No. It covers e-commerce, SaaS, local services, restaurants, clinics, real estate, hotels and travel, publishers, professional services and other businesses. Each industry has its own page types, such as product and category pages for shops, service and location pages for local firms, and menu pages for restaurants.

What does it check on a service page?

Whether the page explains what the service includes, gives price guidance, names the area covered, shows a phone number and booking link, carries reviews and an FAQ as text, and has Service schema. Those are the facts AI answers quote when someone asks who to hire.

What does it check on a product or category page?

On a product page: title, price, currency, stock level, specifications, SKU, brand, reviews, delivery and Product schema. On a category or collection page: whether every product is a crawlable link with its price, plus intro copy, pagination and ItemList schema.

Why is my page marked as a JavaScript shell?

The HTML the server sends is almost empty and the content is built in the browser. Fix it with server-side rendering, static generation or pre-rendering, so names, prices and details are in the first response.

Why do I have to complete a Cloudflare check?

Live scans fetch other websites, so Cloudflare Turnstile confirms a person is running the scan before the tool requests the page. It stops bots from using the tool to crawl sites. Pasted HTML is analysed in your browser and needs no check.

Does passing this check mean AI search will cite my page?

No. It shows that crawlers can read your facts. Citation also depends on content quality, authority and whether the crawler is allowed in robots.txt. Passing is the necessary first step.

Keep going

More free tools and guides

Want your key pages readable by ChatGPT, Claude and Perplexity?

Get your free growth plan More free tools