Skip to content

Commit 1c3bf75

Browse files
abusufyan-netizenabusufyan-netizen
committed
Asset #7: Enterprise robots.txt Templates - Full technical integration and 63rd page indexing
1 parent 0b586cc commit 1c3bf75

3 files changed

Lines changed: 212 additions & 1 deletion

File tree

‎app/tools/page.tsx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ const tools = [
2828
{ name: 'API Latency Cost Calculator', description: 'Calculate the financial impact of network latency on your revenue', icon: TrendingDown, href: '/tools/api-latency-calculator', color: 'from-red-600 to-red-800', category: 'SEO' },
2929
{ name: 'AdSense Revenue Estimator', description: 'Project potential earnings based on traffic, niche, and CPC/CTR', icon: DollarSign, href: '/tools/adsense-calculator', color: 'from-emerald-500 to-emerald-700', category: 'SEO' },
3030
{ name: 'Technical SEO Audit Checklist', description: 'Interactive checklist for comprehensive search engine optimization', icon: ClipboardList, href: '/tools/seo-audit-checklist', color: 'from-blue-600 to-blue-800', category: 'SEO' },
31+
{ name: 'Enterprise robots.txt Templates', description: 'Pre-configured, battle-tested robots.txt for Next.js, WP, and E-commerce', icon: FileText, href: '/tools/robots-txt-templates', color: 'from-orange-600 to-orange-800', category: 'SEO', releaseDate: '2026-05-07' },
3132
{ name: 'WCAG Color Contrast Checker', description: 'Verify ADA and WCAG 2.1 accessibility compliance', icon: Palette, href: '/tools/color-contrast', color: 'from-pink-500 to-pink-700', category: 'Design', releaseDate: '2026-05-08' },
3233
{ name: 'Markdown to HTML Converter', description: 'Convert documentation to clean, semantic HTML code', icon: FileCode, href: '/tools/markdown-converter', color: 'from-violet-500 to-violet-700', category: 'Formatters', releaseDate: '2026-05-09' },
3334

‎app/tools/robots-txt-templates/page.tsx‎

Lines changed: 209 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,209 @@
1+
'use client'
2+
3+
import React, { useState } from 'react'
4+
import { FileText, Copy, CheckCircle2, Info, Layout, ShoppingCart, Cpu, Globe, AlertTriangle, Terminal } from 'lucide-react'
5+
import BreadcrumbSchema from '@/components/seo/BreadcrumbSchema'
6+
import AdSlot from '@/components/ads/AdSlot'
7+
8+
const templates = [
9+
{
10+
id: 'nextjs',
11+
name: 'Enterprise Next.js (App Router)',
12+
icon: Cpu,
13+
description: 'Optimized for modern Next.js 13+ applications with App Router and static export.',
14+
content: `User-agent: *
15+
Allow: /
16+
Disallow: /_next/
17+
Disallow: /404/
18+
Disallow: /api/
19+
20+
# Block common scrapers
21+
User-agent: GPTBot
22+
Disallow: /
23+
24+
Sitemap: https://yourdomain.com/sitemap.xml`,
25+
highlights: ['Blocks internal build files', 'Protects API routes', 'Blocks AI crawlers']
26+
},
27+
{
28+
id: 'wordpress',
29+
name: 'Advanced WordPress SEO',
30+
icon: Layout,
31+
description: 'Hardened config for WordPress sites using Yoast or RankMath.',
32+
content: `User-agent: *
33+
Allow: /
34+
Disallow: /wp-admin/
35+
Allow: /wp-admin/admin-ajax.php
36+
Disallow: /wp-login.php
37+
Disallow: /wp-content/plugins/
38+
Disallow: /readme.html
39+
Disallow: /refer/
40+
41+
Sitemap: https://yourdomain.com/sitemap_index.xml`,
42+
highlights: ['Protects admin areas', 'Allows AJAX functionality', 'Blocks plugin directory leakage']
43+
},
44+
{
45+
id: 'ecommerce',
46+
name: 'High-Authority E-commerce',
47+
icon: ShoppingCart,
48+
description: 'Global template for Shopify, Magento, or custom e-commerce platforms.',
49+
content: `User-agent: *
50+
Allow: /
51+
Disallow: /cart/
52+
Disallow: /checkout/
53+
Disallow: /account/
54+
Disallow: /search/
55+
Disallow: /orders/
56+
Disallow: /*?*filter*
57+
Disallow: /*?*sort*
58+
59+
Sitemap: https://yourdomain.com/sitemap_products.xml`,
60+
highlights: ['Prevents duplicate facet content', 'Protects checkout flow', 'Blocks search results index']
61+
},
62+
{
63+
id: 'minimal',
64+
name: 'Strict Technical Minimalist',
65+
icon: Terminal,
66+
description: 'Maximum crawl budget efficiency for content-heavy sites.',
67+
content: `User-agent: *
68+
Allow: /
69+
Disallow: /temp/
70+
Disallow: /staging/
71+
Disallow: /drafts/
72+
73+
User-agent: Googlebot
74+
Allow: /
75+
76+
Sitemap: https://yourdomain.com/sitemap.xml`,
77+
highlights: ['Optimizes crawl budget', 'Protects staging environments', 'Prioritizes Googlebot']
78+
}
79+
]
80+
81+
export default function RobotsTemplates() {
82+
const [activeTab, setActiveTab] = useState('nextjs')
83+
const [copied, setCopied] = useState(false)
84+
85+
const activeTemplate = templates.find(t => t.id === activeTab) || templates[0]
86+
87+
const handleCopy = () => {
88+
navigator.clipboard.writeText(activeTemplate.content)
89+
setCopied(true)
90+
setTimeout(() => setCopied(false), 2000)
91+
}
92+
93+
return (
94+
<div className="py-12 px-4 sm:px-6 lg:px-8 bg-gray-50/50 dark:bg-slate-950/50 min-h-screen">
95+
<BreadcrumbSchema name="Enterprise robots.txt Templates" slug="tools/robots-txt-templates" />
96+
97+
<div className="max-w-5xl mx-auto">
98+
{/* Header */}
99+
<div className="flex items-center gap-4 mb-12">
100+
<div className="w-14 h-14 bg-orange-600 rounded-2xl flex items-center justify-center shadow-lg shadow-orange-500/20">
101+
<FileText className="w-7 h-7 text-white" />
102+
</div>
103+
<div>
104+
<h1 className="text-3xl font-extrabold text-gray-900 dark:text-white tracking-tight">Enterprise robots.txt Templates</h1>
105+
<p className="text-gray-500 dark:text-slate-400">Battle-tested configurations for professional SEO & technical authority</p>
106+
</div>
107+
</div>
108+
109+
<div className="grid grid-cols-1 lg:grid-cols-12 gap-8">
110+
{/* Sidebar Tabs */}
111+
<div className="lg:col-span-4 space-y-3">
112+
{templates.map((t) => (
113+
<button
114+
key={t.id}
115+
onClick={() => setActiveTab(t.id)}
116+
className={`w-full text-left p-5 rounded-2xl border transition-all duration-300 group ${
117+
activeTab === t.id
118+
? 'bg-white dark:bg-slate-900 border-orange-200 dark:border-orange-900/30 shadow-md ring-1 ring-orange-100 dark:ring-orange-900/10'
119+
: 'bg-transparent border-transparent hover:bg-white/50 dark:hover:bg-slate-900/50 text-gray-500'
120+
}`}
121+
>
122+
<div className="flex items-center gap-3">
123+
<t.icon className={`w-5 h-5 ${activeTab === t.id ? 'text-orange-600' : 'text-gray-400 group-hover:text-orange-500'}`} />
124+
<div>
125+
<div className={`font-bold text-sm ${activeTab === t.id ? 'text-gray-900 dark:text-white' : 'text-gray-500 dark:text-slate-400'}`}>
126+
{t.name}
127+
</div>
128+
</div>
129+
</div>
130+
</button>
131+
))}
132+
133+
<div className="mt-8">
134+
<AdSlot />
135+
</div>
136+
</div>
137+
138+
{/* Template Content */}
139+
<div className="lg:col-span-8 space-y-6">
140+
<div className="bg-white dark:bg-slate-900 rounded-3xl border border-gray-100 dark:border-slate-800 shadow-sm overflow-hidden">
141+
<div className="p-6 border-b border-gray-50 dark:border-slate-800 flex items-center justify-between bg-gray-50/50 dark:bg-slate-900/50">
142+
<div className="flex items-center gap-2 text-xs font-bold text-gray-400 dark:text-slate-500 uppercase tracking-widest">
143+
<Globe className="w-4 h-4" /> Production-Ready Config
144+
</div>
145+
<button
146+
onClick={handleCopy}
147+
className="flex items-center gap-2 px-4 py-2 bg-white dark:bg-slate-800 text-gray-700 dark:text-gray-200 text-xs font-bold rounded-xl border border-gray-200 dark:border-slate-700 hover:bg-gray-50 dark:hover:bg-slate-700 transition-all shadow-sm"
148+
>
149+
{copied ? <CheckCircle2 className="w-4 h-4 text-green-500" /> : <Copy className="w-4 h-4" />}
150+
{copied ? 'Copied!' : 'Copy Code'}
151+
</button>
152+
</div>
153+
154+
<div className="p-8">
155+
<p className="text-gray-600 dark:text-slate-400 text-sm leading-relaxed mb-6">
156+
{activeTemplate.description}
157+
</p>
158+
159+
<div className="relative">
160+
<pre className="bg-slate-950 text-slate-200 p-8 rounded-2xl text-sm font-mono leading-relaxed overflow-x-auto border border-slate-800 shadow-inner">
161+
{activeTemplate.content}
162+
</pre>
163+
</div>
164+
165+
<div className="mt-8 grid grid-cols-1 md:grid-cols-3 gap-4">
166+
{activeTemplate.highlights.map((h, i) => (
167+
<div key={i} className="flex items-center gap-2 text-[10px] font-bold text-gray-500 dark:text-slate-500 uppercase tracking-wider bg-gray-50 dark:bg-slate-800/50 px-3 py-2 rounded-lg border border-gray-100 dark:border-slate-800">
168+
<CheckCircle2 className="w-3 h-3 text-orange-500" /> {h}
169+
</div>
170+
))}
171+
</div>
172+
</div>
173+
</div>
174+
175+
<div className="bg-amber-50 dark:bg-amber-900/10 p-6 rounded-3xl border border-amber-100 dark:border-amber-900/30">
176+
<div className="flex items-center gap-2 text-amber-700 dark:text-amber-400 mb-3 font-bold">
177+
<AlertTriangle className="w-5 h-5" />
178+
Technical Warning
179+
</div>
180+
<p className="text-sm text-amber-700/80 dark:text-amber-400/80 leading-relaxed">
181+
Always replace <strong>yourdomain.com</strong> with your actual production URL. Misconfiguring your robots.txt can lead to immediate de-indexing of your entire site.
182+
</p>
183+
</div>
184+
</div>
185+
</div>
186+
187+
<div className="mt-16 text-center">
188+
<h2 className="text-2xl font-bold text-gray-900 dark:text-white mb-6">Frequently Asked Questions</h2>
189+
<div className="grid grid-cols-1 md:grid-cols-2 gap-6 text-left">
190+
<div className="bg-white dark:bg-slate-900 p-6 rounded-2xl border border-gray-100 dark:border-slate-800 shadow-sm">
191+
<h3 className="font-bold text-gray-900 dark:text-white mb-2">Why block AI bots?</h3>
192+
<p className="text-sm text-gray-500 dark:text-slate-400 leading-relaxed">
193+
Bots like GPTBot crawl your site to train models. If you want to protect your unique data or research from being used without attribution, blocking them in robots.txt is the standard procedure.
194+
</p>
195+
</div>
196+
<div className="bg-white dark:bg-slate-900 p-6 rounded-2xl border border-gray-100 dark:border-slate-800 shadow-sm">
197+
<h3 className="font-bold text-gray-900 dark:text-white mb-2">What is a Crawl Budget?</h3>
198+
<p className="text-sm text-gray-500 dark:text-slate-400 leading-relaxed">
199+
Google allocates a specific amount of time to crawl your site. By blocking useless pages (like search results or cart pages), you ensure Google spends its time on your valuable articles and tools.
200+
</p>
201+
</div>
202+
</div>
203+
</div>
204+
205+
<AdSlot className="mt-16" />
206+
</div>
207+
</div>
208+
)
209+
}

‎public/sitemap.xml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,8 @@
3838
<url><loc>https://wtkpro.site/blog/ui-ux-design-trends/</loc><lastmod>2026-05-07</lastmod><priority>0.7</priority></url>
3939
<url><loc>https://wtkpro.site/blog/web-accessibility-best-practices/</loc><lastmod>2026-05-07</lastmod><priority>0.7</priority></url>
4040

41-
<!-- Essential Tools (27 Pages) -->
41+
<!-- Essential Tools (28 Pages) -->
42+
<url><loc>https://wtkpro.site/tools/robots-txt-templates/</loc><lastmod>2026-05-07</lastmod><priority>0.9</priority></url>
4243
<url><loc>https://wtkpro.site/tools/social-preview-tester/</loc><lastmod>2026-05-07</lastmod><priority>0.9</priority></url>
4344
<url><loc>https://wtkpro.site/tools/api-latency-calculator/</loc><lastmod>2026-05-07</lastmod><priority>0.9</priority></url>
4445
<url><loc>https://wtkpro.site/tools/adsense-calculator/</loc><lastmod>2026-05-07</lastmod><priority>0.9</priority></url>

0 commit comments

Comments
 (0)