#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used: http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html
User-agent: *
# CSS, JS, Images
Allow: /core/*.css$
Allow: /core/*.css?
Allow: /core/*.js$
Allow: /core/*.js?
Allow: /core/*.gif
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.png
Allow: /core/*.svg
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /profiles/*.svg
# Directories
Disallow: /core/
Disallow: /profiles/
# Files
Disallow: /README.md
Disallow: /composer/Metapackage/README.txt
Disallow: /composer/Plugin/ProjectMessage/README.md
Disallow: /composer/Plugin/Scaffold/README.md
Disallow: /composer/Plugin/VendorHardening/README.txt
Disallow: /composer/Template/README.txt
Disallow: /modules/README.txt
Disallow: /sites/README.txt
Disallow: /themes/README.txt
Disallow: /web.config
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register
Disallow: /user/password
Disallow: /user/login
Disallow: /user/logout
Disallow: /media/oembed
Disallow: /*/media/oembed
# Paths (no clean URLs)
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password
Disallow: /index.php/user/register
Disallow: /index.php/user/login
Disallow: /index.php/user/logout
Disallow: /index.php/media/oembed
Disallow: /index.php/*/media/oembed
User-Agent: serpstatbot
Crawl-Delay: 20
User-agent: rogerbot
Crawl-Delay: 20
User-agent: Linguee Bot
Crawl-Delay: 20
User-agent: ZoominfoBot
Crawl-Delay: 20
User-agent: ia_archiver
Crawl-Delay: 20
User-agent: linkdexbot
User-agent: linkdexbot/2.1
Crawl-Delay: 20
User-agent: Gigabot
Crawl-Delay: 20
User-agent: ccbot
Crawl-Delay: 20
User-agent: meta-externalagent
Crawl-Delay: 20
User-agent: dotbot
Crawl-delay: 20
User-agent: Bingbot
Crawl-delay: 10
User-agent: SemrushBot
Crawl-delay: 30
User-agent: Amazonbot
User-agent: Amazonbot/0.1
Crawl-delay: 30
User-agent: Scrapy
User-agent: Scrapy/2.6.2
Crawl-delay: 30
User-agent: Monsidobot
User-agent: Monsidobot/2.2
Crawl-delay: 30
User-agent: SiteAuditBot
Crawl-delay: 30
User-agent: archive.org_bot
Crawl-delay: 20
User-agent: BLEXBot
Crawl-delay: 30
User-agent: VelenPublicWebCrawler
Crawl-delay: 30
User-agent: DataForSeoBot
Crawl-delay: 30
User-agent: DuckDuckBot
Crawl-delay: 30
User-agent: Sogou blog
User-agent: Sogou inst spider
User-agent: Sogou News Spider
User-agent: Sogou Orion spider
User-agent: Sogou spider2
User-agent: Sogou web spider
Disallow: /
User-agent: Buck
Disallow: /
User-agent: coccoc
User-agent: coccocbot-image
User-agent: coccocbot-web
Disallow: /
User-agent: SEOkicks
Disallow: /
User-agent: IonCrawl
Disallow: /
User-agent: SeznamBot
Disallow: /
User-agent: AwarioRssBot
User-agent: AwarioSmartBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: MegaIndex.ru
User-agent: MegaIndex.ru/2.0
Disallow: /
User-agent: SeekportBot
Disallow: /
User-agent: megaindex.com
Disallow: /
User-agent: Yeti
Disallow: /
User-agent: Haosou 360 spider
Disallow: /
User-agent: Dotmic DotBot
Disallow: /
User-agent: Baiduspider
User-agent: Baiduspider-image
User-agent: Baiduspider-news
User-agent: Baiduspider-video
Disallow: /
User-agent: YandexBot
User-agent: Yandex
User-agent: YandexMobileBot
User-agent: YandexBot/3.0
Disallow: /
User-agent: ecoresearchCrawler
Disallow: /
User-agent: ecoresearch
Disallow: /
User-agent: Barkrowler
Disallow: /
User-agent: Zeus 32297 Webster Pro V2.9 Win32
Disallow: /
User-agent: GPTBot
Disallow: /
User-agent: ChatGPT-User
Disallow: /
User-agent: Google-Extended
Disallow: /
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
Disallow: /
User-agent: Omgilibot
Disallow: /
User-agent: Omgili
Disallow: /
User-agent: FacebookBot
Disallow: /
{
"status": 200,
"statusText": "",
"headers": {
"accept-ranges": "bytes",
"age": "5622",
"cache-control": "max-age=10800, public",
"connection": "keep-alive",
"content-encoding": "gzip",
"content-language": "en",
"content-length": "15213",
"content-type": "text/html; charset=UTF-8",
"date": "Sat, 08 Feb 2025 10:39:17 GMT",
"etag": "W/\"1738940549\"",
"expires": "Sun, 19 Nov 1978 05:00:00 GMT",
"last-modified": "Fri, 07 Feb 2025 15:02:29 GMT",
"permissions-policy": "autoplay=*, camera=*, fullscreen=*",
"referrer-policy": "strict-origin-when-cross-origin",
"server": "nginx",
"strict-transport-security": "max-age=31622400",
"vary": "Accept-Encoding, Cookie, Cookie, Cookie",
"via": "1.1 varnish, 1.1 varnish",
"x-cache": "HIT, HIT",
"x-cache-hits": "4, 0",
"x-content-type-options": "nosniff",
"x-drupal-cache": "MISS",
"x-drupal-dynamic-cache": "MISS",
"x-frame-options": "SAMEORIGIN",
"x-generator": "Drupal 10 (https://www.drupal.org)",
"x-pantheon-styx-hostname": "styx-fe1-b-6f847bcb88-4bzm9",
"x-served-by": "cache-chi-klot8100035-CHI, cache-iad-kiad7000051-IAD",
"x-styx-req-id": "8d04ad2c-e564-11ef-8414-823b89b9e272",
"x-timer": "S1739011158.671687,VS0,VE3"
}
}
{
"latency": 203,
"ipAddresses": [
"23.185.0.253",
"2620:12a:8000::253",
"2620:12a:8001::253"
],
"dns": {
"NS": [
"cody.ns.cloudflare.com.",
"dolly.ns.cloudflare.com."
],
"A": [
"23.185.0.253"
],
"AAAA": [
"2620:12a:8000::253",
"2620:12a:8001::253"
]
}
}
[
{
"url": "https://nsbufl.com",
"technologies": [
{
"slug": "mariadb",
"name": "MariaDB",
"versions": [],
"trafficRank": 0,
"confirmedAt": 1723605152,
"icon": "mariadb.svg",
"categories": [
{
"id": 34,
"slug": "databases",
"name": "Databases"
}
]
},
{
"slug": "hsts",
"name": "HSTS",
"versions": [],
"trafficRank": 0,
"confirmedAt": 1723605152,
"icon": "default.svg",
"categories": [
{
"id": 16,
"slug": "security",
"name": "Security"
}
]
},
{
"slug": "fastly",
"name": "Fastly",
"versions": [],
"trafficRank": 0,
"confirmedAt": 1723605152,
"icon": "Fastly.svg",
"categories": [
{
"id": 31,
"slug": "cdn",
"name": "CDN"
}
]
},
{
"slug": "nginx",
"name": "Nginx",
"versions": [],
"trafficRank": 0,
"confirmedAt": 1723605152,
"icon": "Nginx.svg",
"categories": [
{
"id": 22,
"slug": "web-servers",
"name": "Web servers"
},
{
"id": 64,
"slug": "reverse-proxies",
"name": "Reverse proxies"
}
]
},
{
"slug": "google-tag-manager",
"name": "Google Tag Manager",
"versions": [],
"trafficRank": 0,
"confirmedAt": 1723605152,
"icon": "Google Tag Manager.svg",
"categories": [
{
"id": 42,
"slug": "tag-managers",
"name": "Tag managers"
}
]
},
{
"slug": "drupal",
"name": "Drupal",
"cpe": "cpe:2.3:a:drupal:drupal:10:*:*:*:*:*:*:*",
"versions": [
"10"
],
"trafficRank": 0,
"confirmedAt": 1723605152,
"icon": "Drupal.svg",
"categories": [
{
"id": 1,
"slug": "cms",
"name": "CMS"
}
]
},
{
"slug": "pantheon",
"name": "Pantheon",
"versions": [],
"trafficRank": 0,
"confirmedAt": 1723605152,
"icon": "Pantheon.svg",
"categories": [
{
"id": 62,
"slug": "paas",
"name": "PaaS"
}
]
},
{
"slug": "uswds",
"name": "USWDS",
"versions": [],
"trafficRank": 0,
"confirmedAt": 1723605152,
"icon": "USWDS.svg",
"categories": [
{
"id": 66,
"slug": "ui-frameworks",
"name": "UI frameworks"
}
]
},
{
"slug": "godaddy",
"name": "GoDaddy",
"versions": [],
"trafficRank": 0,
"confirmedAt": 1723605152,
"icon": "GoDaddy.svg",
"categories": [
{
"id": 88,
"slug": "hosting",
"name": "Hosting"
}
]
},
{
"slug": "recaptcha",
"name": "reCAPTCHA",
"versions": [],
"trafficRank": 0,
"confirmedAt": 1723605152,
"icon": "reCAPTCHA.svg",
"categories": [
{
"id": 16,
"slug": "security",
"name": "Security"
}
]
},
{
"slug": "jquery",
"name": "jQuery",
"cpe": "cpe:2.3:a:jquery:jquery:3.7.1:*:*:*:*:*:*:*",
"versions": [
"3.7.1"
],
"trafficRank": 0,
"confirmedAt": 1723605152,
"icon": "jQuery.svg",
"categories": [
{
"id": 59,
"slug": "javascript-libraries",
"name": "JavaScript libraries"
}
]
},
{
"slug": "php",
"name": "PHP",
"versions": [],
"trafficRank": 0,
"confirmedAt": 1723605152,
"icon": "PHP.svg",
"categories": [
{
"id": 27,
"slug": "programming-languages",
"name": "Programming languages"
}
]
},
{
"slug": "microsoft-365",
"name": "Microsoft 365",
"versions": [],
"trafficRank": 0,
"confirmedAt": 1723605152,
"icon": "Microsoft 365.svg",
"categories": [
{
"id": 30,
"slug": "webmail",
"name": "Webmail"
},
{
"id": 75,
"slug": "email",
"name": "Email"
}
]
},
{
"slug": "google-analytics",
"name": "Google Analytics",
"versions": [],
"trafficRank": 0,
"confirmedAt": 1723605152,
"icon": "Google Analytics.svg",
"categories": [
{
"id": 10,
"slug": "analytics",
"name": "Analytics"
}
]
},
{
"slug": "varnish",
"name": "Varnish",
"versions": [],
"trafficRank": 0,
"confirmedAt": 1723605152,
"icon": "Varnish.svg",
"categories": [
{
"id": 23,
"slug": "caching",
"name": "Caching"
}
]
}
]
}
]
{
"error": false,
"result": {
"success": true,
"ogSiteName": "New Smyrna Beach Utilities",
"ogType": "website",
"ogUrl": "https://www.nsbufl.com/",
"ogTitle": "New Smyrna Beach Utilities",
"ogDescription": "Official website of New Smyrna Beach Utilities",
"ogImage": [
{
"url": "https://www.nsbufl.com/sites/default/files/NSBU_header3.png",
"alt": "New Smyrna Beach Utilities",
"type": "png"
}
],
"ogLocale": "en",
"favicon": "/sites/default/files/NSBU_fav.png",
"charset": "utf-8",
"requestUrl": "https://nsbufl.com"
},
"response": {}
}
{
"data": {
"total": 17,
"personal_emails": 14,
"generic_emails": 3,
"department": {
"executive": 2,
"it": 0,
"finance": 1,
"management": 2,
"sales": 0,
"legal": 0,
"support": 1,
"hr": 1,
"marketing": 0,
"communication": 1,
"education": 0,
"design": 0,
"health": 0,
"operations": 0
},
"seniority": {
"junior": 0,
"senior": 2,
"executive": 3
}
},
"meta": {
"params": {
"domain": "nsbufl.com",
"company": null,
"type": null
}
}
}
{
"KnowBe4": "Security awareness system.",
"Loggly": "Cloud-based solution that tries to makes sense of log data coming from applications, platforms, and systems. Owned by Solarwinds.",
"Global Site Tag": "Google's primary tag for Google Measurement/Conversion Tracking, Adwords and DoubleClick.",
"Google Analytics": "Google Analytics offers a host of compelling features and benefits for everyone from senior executives and advertising and marketing professionals to site owners and content developers.",
"Google Analytics 4": "Google Analytics 4 formerly known as App + Web is a new version of Google Analytics that was released in October 2020.",
"Fastly": "Real-time Analytics and CDN platform. Analyze your web and server traffic patterns in real-time.",
"WebEx": "Online meeting scheduling widget.",
"Google Tag Manager": "Tag management that lets you add and update website tags without changes to underlying website code.",
"US Privacy User Signal Mechanism": "The US Privacy API (USP API) is a lightweight API used to communicate signals represented in the US Privacy String.",
"CrUX Dataset": "CrUX is a data collection system that gathers information about how real users interact with websites. This website is included in the user experiences data gathered from Google Chrome and thus considered sufficiently popular on the Internet.",
"CrUX Top 5m": "Relative measure of site popularity within the CrUX dataset, measured by the total number of navigations on the origin. This site is in the top 5 million.",
"CrUX Top 1m": "Relative measure of site popularity within the CrUX dataset, measured by the total number of navigations on the origin. This site is in the top 1 million.",
"WebEx Panel": "WebEx system.",
"Cloudflare Radar": "The website appears on the Cloudflare Radar Top 1m sites list",
"English - Inferred": "Based on the title and description text the website content is potentially English.",
"Mobile Optimized": "Microsoft invented the MobileOptimized META tag to control the layout width for mobile markup rendered in Internet Explorer Mobile.",
"Viewport Meta": "This page uses the viewport meta tag which means the content may be optimized for mobile content.",
"IPhone / Mobile Compatible": "The website contains code that allows the page to support IPhone / Mobile Content.",
"Muni-Link": "Muni-Link offers cloud-based utility billing software with online solutions for municipalities.",
"Drupal": "An engine suitable to setup or build a content driven or community driven website. Modular design allows flexibility in design.",
"Drupal 10": "Version 10 of the popular CMS Drupal.",
"Intersection Observer": "API that can be used to understand the visibility and position of DOM elements relative to a containing element or to the top-level viewport.",
"jQuery": "JQuery is a fast, concise, JavaScript Library that simplifies how you traverse HTML documents, handle events, perform animations, and add Ajax interactions to your web pages. jQuery is designed to change the way that you write JavaScript.",
"jQuery 3.7.1": "jQuery version 3.7.1",
"USWDS": "U.S. Web Design System is an open source UI components and visual style guide for U.S. government websites.",
"Facebook": "The website mentions facebook.com in some form.",
"LinkedIn": "The website mentions linkedin.com in some form.",
"Instagram": "The website mentions Instagram in some form.",
"YouTube Link": "This website mentions YouTube.com on it in some form or another.",
"Accessibility Statement": "A statement that outlines how a website has been designed to be accessible to people with disabilities.",
"DoubleClick.Net": "DoubleClick enables agencies, marketers and publishers to work together successfully and profit from their digital marketing investments. Owned by Google and now referred to as DoubleClick Digital Marketing or Google Enterprise Advertising.",
"AmazonBot Disallow": "AmazonBot is a crawler that web content publishers can refer to for information - website has disallow rules.",
"CodeTwo": "Email signature solution.",
"SPF": "The Sender Policy Framework is an open standard specifying a technical method to prevent sender address forgery.",
"Microsoft Exchange Online": "A rich hosted Exchange environment for every user without having to manage a server.",
"Office 365 Mail": "Email sent from this domain has records showing Office 365 usage.",
"SSL by Default": "The website redirects traffic to an HTTPS/SSL version by default.",
"LetsEncrypt": "Let’s Encrypt is a free open Certificate Authority.",
"HSTS": "Forces browsers to only communicate with the site using HTTPS.",
"Pantheon": "Drupal and WordPress hosting company.",
"Fastly Hosted": "Hosted at Fastly",
"Fastly Load Balancer": "Hosted on Fastly's Load Balancing system.",
"U.S. Server Location": "The web server is located in the United States.",
"GoDaddy DNS": "DNS services provided by GoDaddy.",
"Cloudflare DNS": "DNS services provided by Cloudflare.",
"Varnish": "Varnish is a web accelerator / reverse proxy caching server.",
"nginx": "nginx [engine x] is a HTTP server and mail proxy server written by Igor Sysoev.",
"IPv6": "The website has an IPv6 record.",
"Baidu Bot Disallow": "The website has disallow rules for Baidu Baiduspider bot.",
"Alexa Bot Disallow": "The website has disallow rules for Alexa's ia_archiver bot.",
"Majestic Bot Disallow": "The website has disallow rules for Majestic Bot.",
"Yandex Bot Disallow": "The website has disallow rules for Yandex YandexBot.",
"Microsoft BingBot Disallow": "The website has disallow rules for Microsoft Bing Bot.",
"Semrush Bot Disallow": "The website has disallow rules for Semrush Bot.",
"GPTBot Disallow": "The website has disallow rules for OpenAI GPTBot.",
"Sogou Bot Disallow": "The website has disallow rules for Sogou Bot.",
"Common Crawl Bot Disallow": "Block a site from being crawled by Common Crawl Crawler.",
"Anthropic Claude Bot Disallow": "Block Anthropic Claude bot.",
"Facebook Bot Disallow": "Blocks Facebook crawling.",
"Google Bard Disallow": "The website blocks Bard AI and Vertex AI scraping requests.",
"Webz Omgili Disallow": "Webz.io disallow block for big data indexing.",
"DuckDuckGo Bot Disallow": "The website has disallow rules for DuckDuckGo bot."
}
{
"whois.verisign-grs.com": {
"Domain Status": [
"clientDeleteProhibited https://icann.org/epp#clientDeleteProhibited",
"clientRenewProhibited https://icann.org/epp#clientRenewProhibited",
"clientTransferProhibited https://icann.org/epp#clientTransferProhibited",
"clientUpdateProhibited https://icann.org/epp#clientUpdateProhibited"
],
"Name Server": [
"CODY.NS.CLOUDFLARE.COM",
"DOLLY.NS.CLOUDFLARE.COM"
],
"Domain Name": "NSBUFL.COM",
"Registry Domain ID": "2671447676_DOMAIN_COM-VRSN",
"Registrar WHOIS Server": "whois.godaddy.com",
"Registrar URL": "http://www.godaddy.com",
"Updated Date": "2024-11-13T13:18:58Z",
"Created Date": "2022-01-28T14:35:25Z",
"Expiry Date": "2032-01-28T14:35:25Z",
"Registrar": "GoDaddy.com, LLC",
"Registrar IANA ID": "146",
"Registrar Abuse Contact Email": "[email protected]",
"Registrar Abuse Contact Phone": "480-624-2505",
"DNSSEC": "unsigned",
"URL of the ICANN Whois Inaccuracy Complaint Form": "https://www.icann.org/wicf/",
">>> Last update of whois database": "2025-02-08T10:39:06Z <<<",
"text": [
"For more information on Whois status codes, please visit https://icann.org/epp",
"",
"NOTICE: The expiration date displayed in this record is the date the",
"registrar's sponsorship of the domain name registration in the registry is",
"currently set to expire. This date does not necessarily reflect the expiration",
"date of the domain name registrant's agreement with the sponsoring",
"registrar. Users may consult the sponsoring registrar's Whois database to",
"view the registrar's reported date of expiration for this registration.",
"",
"TERMS OF USE: You are not authorized to access or query our Whois",
"database through the use of electronic processes that are high-volume and",
"automated except as reasonably necessary to register domain names or",
"modify existing registrations; the Data in VeriSign Global Registry",
"Services' (\"VeriSign\") Whois database is provided by VeriSign for",
"information purposes only, and to assist persons in obtaining information",
"about or related to a domain name registration record. VeriSign does not",
"guarantee its accuracy. By submitting a Whois query, you agree to abide",
"by the following terms of use: You agree that you may use this Data only",
"for lawful purposes and that under no circumstances will you use this Data",
"to: (1) allow, enable, or otherwise support the transmission of mass",
"unsolicited, commercial advertising or solicitations via e-mail, telephone,",
"or facsimile; or (2) enable high volume, automated, electronic processes",
"that apply to VeriSign (or its computer systems). The compilation,",
"repackaging, dissemination or other use of this Data is expressly",
"prohibited without the prior written consent of VeriSign. You agree not to",
"use electronic processes that are automated and high-volume to access or",
"query the Whois database except as reasonably necessary to register",
"domain names or modify existing registrations. VeriSign reserves the right",
"to restrict your access to the Whois database in its sole discretion to ensure",
"operational stability. VeriSign may restrict or terminate your access to the",
"Whois database for failure to abide by these terms of use. VeriSign",
"reserves the right to modify these terms at any time.",
"",
"The Registry database contains ONLY .COM, .NET, .EDU domains and",
"Registrars."
]
},
"whois.godaddy.com": {
"Domain Status": [],
"Name Server": [],
"Rate limit exceeded. Try again after": "15s",
"text": [
""
]
}
}