# Public retrieval/search crawler policy reviewed 2026-10-05.
# These are the current official identities for public search or user-directed
# retrieval. They may fetch the public site and its linked job pages.
User-agent: OAI-SearchBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Googlebot
User-agent: bingbot
Allow: /

# Keep known search/filter query variants out of crawling for these named
# clients too; a named group does not inherit the wildcard group below.
Disallow: /*?page=
Disallow: /*?q=
Disallow: /*?employmentType=
Disallow: /*?experienceLevel=
Disallow: /*?salaryRange=
Disallow: /*?remote=
Disallow: /*?withSalary=
Disallow: /*?postedWithin=
Disallow: /*?location=
# RSC client-navigation payloads: pages are server-rendered, so crawlers need only the HTML.
Disallow: /*?_rsc=
Disallow: /*&_rsc=
Disallow: /api/
Allow: /api/og
Allow: /api/company-avatar
# Public JSON job list (noindex via X-Robots-Tag). The $ rule matches the bare
# path, the ? rule any query. /api/jobs/<id>/view stays blocked.
Allow: /api/jobs$
Allow: /api/jobs?

# Company pages 2+ (/company/<slug>?page=N) are the only internal links to jobs
# beyond a company's first 100, so they must stay crawlable. The longer Allow
# beats Disallow: /*?page=. Filtered variants such as ?q=eng&page=2 do not match
# it (they start with ?q=) and stay blocked by the rules above. Anything after
# the page value (?page=2&q=foo, utm tags) is blocked by the longer Disallow.
Allow: /company/*?page=
Disallow: /company/*?page=*&

# AI visibility policy, 2026-10-05: the owner wants maximum AI discoverability.
# Training crawlers (GPTBot, ClaudeBot, Google-Extended, Applebot-Extended, and
# similar) stay allowed through the wildcard group below. Common Crawl (CCBot)
# is allowed too, because it feeds many open models.

# Block known aggressive scrapers and non-beneficial crawlers. AhrefsBot is
# allowed: the site uses Ahrefs to measure rankings, and a blocked crawler
# left it reporting a months-old snapshot.
User-agent: cohere-ai
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: ZoominfoBot
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: BaiduSpider
Disallow: /

# Allow all other bots (search engines, social previews, etc.).
User-agent: *
Allow: /

# Same rules as the named-bot group above. Do not add per-directory Allow
# lines here: a longer Allow (e.g. /cities/) beats these shorter Disallows and
# silently re-opens every query variant under that directory.
Disallow: /*?page=
Disallow: /*?q=
Disallow: /*?employmentType=
Disallow: /*?experienceLevel=
Disallow: /*?salaryRange=
Disallow: /*?remote=
Disallow: /*?withSalary=
Disallow: /*?postedWithin=
Disallow: /*?location=
# RSC client-navigation payloads: pages are server-rendered, so crawlers need only the HTML.
Disallow: /*?_rsc=
Disallow: /*&_rsc=
Disallow: /api/
Allow: /api/og
Allow: /api/company-avatar
# Public JSON job list (noindex via X-Robots-Tag). The $ rule matches the bare
# path, the ? rule any query. /api/jobs/<id>/view stays blocked.
Allow: /api/jobs$
Allow: /api/jobs?
Allow: /company/*?page=
Disallow: /company/*?page=*&

# Sitemaps
Sitemap: https://www.nextgenenergyjobs.com/sitemap.xml
# Individual sitemaps
Sitemap: https://www.nextgenenergyjobs.com/sitemap/static.xml
Sitemap: https://www.nextgenenergyjobs.com/sitemap/companies.xml
Sitemap: https://www.nextgenenergyjobs.com/sitemap/job_openings_1.xml
Sitemap: https://www.nextgenenergyjobs.com/sitemap/job_openings_2.xml
Sitemap: https://www.nextgenenergyjobs.com/sitemap/job_openings_3.xml
Sitemap: https://www.nextgenenergyjobs.com/sitemap/job_openings_4.xml
Sitemap: https://www.nextgenenergyjobs.com/sitemap/job_openings_5.xml
Sitemap: https://www.nextgenenergyjobs.com/sitemap/job_openings_6.xml
Sitemap: https://www.nextgenenergyjobs.com/sitemap/job_openings_7.xml
Sitemap: https://www.nextgenenergyjobs.com/sitemap/job_openings_8.xml
Sitemap: https://www.nextgenenergyjobs.com/sitemap/job_openings_9.xml
Sitemap: https://www.nextgenenergyjobs.com/sitemap/job_openings_10.xml
Sitemap: https://www.nextgenenergyjobs.com/sitemap/job_openings_11.xml
Sitemap: https://www.nextgenenergyjobs.com/sitemap/job_openings_12.xml
Sitemap: https://www.nextgenenergyjobs.com/sitemap/job_openings_13.xml
Sitemap: https://www.nextgenenergyjobs.com/sitemap/job_openings_14.xml
Sitemap: https://www.nextgenenergyjobs.com/sitemap/job_openings_15.xml
Sitemap: https://www.nextgenenergyjobs.com/sitemap/job_openings_16.xml
Sitemap: https://www.nextgenenergyjobs.com/sitemap/countries.xml
Sitemap: https://www.nextgenenergyjobs.com/sitemap/job_titles.xml
Sitemap: https://www.nextgenenergyjobs.com/sitemap/career-advice.xml
Sitemap: https://www.nextgenenergyjobs.com/sitemap/states.xml
Sitemap: https://www.nextgenenergyjobs.com/sitemap/cities.xml
Sitemap: https://www.nextgenenergyjobs.com/sitemap/industries.xml
Sitemap: https://www.nextgenenergyjobs.com/sitemap/skills.xml