Skip to content

Commit ca73a4c

Browse files
ColDog5044ColDog5044
committed
Enhance SEO meta tags and update robots.txt directives
Added dynamic canonical, robots, and structured data meta tags to scripts.js for improved SEO and search engine understanding. Updated robots.txt to provide specific directives for major search engines and block AI training bots, ensuring better control over site indexing and privacy.
1 parent e5cabb0 commit ca73a4c

2 files changed

Lines changed: 252 additions & 3 deletions

File tree

‎assets/js/scripts.js‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,117 @@ document.addEventListener("DOMContentLoaded", function () {
4141
document.head.appendChild(meta);
4242
}
4343

44+
// Function to add canonical link
45+
function addCanonical(url) {
46+
const link = document.createElement("link");
47+
link.rel = "canonical";
48+
link.href = url;
49+
document.head.appendChild(link);
50+
}
51+
52+
// Function to add search engine specific meta tags
53+
function addSearchEngineMetaTags(path) {
54+
// Add canonical URL
55+
addCanonical(window.location.href);
56+
57+
// Default robots directive for all pages
58+
let robotsDirective = "index, follow";
59+
60+
// Page-specific robots directives
61+
if (path.includes("/legal/")) {
62+
robotsDirective = "noindex, nofollow"; // Block legal pages from indexing
63+
} else if (path.includes("/branding/")) {
64+
robotsDirective = "noindex, nofollow"; // Keep internal branding private
65+
} else if (path === "/404.html") {
66+
robotsDirective = "noindex, nofollow"; // Don't index 404 pages
67+
} else if (path.includes("/contact/")) {
68+
robotsDirective = "index, follow"; // Good for local SEO
69+
}
70+
71+
// General robots meta tag
72+
addMetaTag("robots", robotsDirective);
73+
74+
// Google-specific directives
75+
addMetaTag("googlebot", `${robotsDirective}, max-snippet:160, max-image-preview:large`);
76+
77+
// Bing-specific directives
78+
addMetaTag("bingbot", robotsDirective);
79+
80+
// Additional meta tags for better indexing
81+
addMetaTag("referrer", "no-referrer-when-downgrade");
82+
addMetaTag("format-detection", "telephone=no");
83+
84+
// Add hreflang for English (assuming your site is English-only)
85+
const hrefLang = document.createElement("link");
86+
hrefLang.rel = "alternate";
87+
hrefLang.hreflang = "en";
88+
hrefLang.href = window.location.href;
89+
document.head.appendChild(hrefLang);
90+
}
91+
92+
// Function to add JSON-LD structured data
93+
function addStructuredData(pageMeta, path) {
94+
const script = document.createElement("script");
95+
script.type = "application/ld+json";
96+
97+
let structuredData;
98+
99+
if (path === "/") {
100+
// Organization schema for homepage
101+
structuredData = {
102+
"@context": "https://schema.org",
103+
"@type": "Organization",
104+
name: "ColDog Studios",
105+
url: "https://www.coldogstudios.com",
106+
logo: "https://www.coldogstudios.com/assets/images/cds/logo/cdsLogo.png",
107+
description: pageMeta.description,
108+
foundingDate: "2023",
109+
contactPoint: {
110+
"@type": "ContactPoint",
111+
url: "https://www.coldogstudios.com/contact/",
112+
},
113+
sameAs: [],
114+
};
115+
} else if (path.includes("/projects/")) {
116+
// SoftwareApplication schema for projects
117+
structuredData = {
118+
"@context": "https://schema.org",
119+
"@type": "SoftwareApplication",
120+
name: pageMeta.title,
121+
description: pageMeta.description,
122+
url: window.location.href,
123+
author: {
124+
"@type": "Organization",
125+
name: "ColDog Studios",
126+
},
127+
publisher: {
128+
"@type": "Organization",
129+
name: "ColDog Studios",
130+
},
131+
};
132+
} else {
133+
// WebPage schema for other pages
134+
structuredData = {
135+
"@context": "https://schema.org",
136+
"@type": "WebPage",
137+
name: pageMeta.title,
138+
description: pageMeta.description,
139+
url: window.location.href,
140+
author: {
141+
"@type": "Organization",
142+
name: "ColDog Studios",
143+
},
144+
publisher: {
145+
"@type": "Organization",
146+
name: "ColDog Studios",
147+
},
148+
};
149+
}
150+
151+
script.textContent = JSON.stringify(structuredData);
152+
document.head.appendChild(script);
153+
}
154+
44155
// Fetch and apply metadata from metadata.json
45156
fetch("https://www.coldogstudios.com/metadata.json")
46157
.then((response) => {
@@ -64,6 +175,9 @@ document.addEventListener("DOMContentLoaded", function () {
64175

65176
metaTags.forEach((tag) => addMetaTag(tag.name, tag.content));
66177

178+
// Add Search Engine Specific Meta Tags
179+
addSearchEngineMetaTags(currentPath);
180+
67181
// Add Social Media Meta Tags
68182
const currentUrl = window.location.href;
69183
const siteTitle = document.title;
@@ -80,6 +194,9 @@ document.addEventListener("DOMContentLoaded", function () {
80194
addMetaTag("twitter:title", siteTitle);
81195
addMetaTag("twitter:description", pageMeta.description);
82196
addMetaTag("twitter:image", "https://www.coldogstudios.com/assets/images/cds/cdsWallpaperLite.png");
197+
198+
// Add JSON-LD structured data for better search engine understanding
199+
addStructuredData(pageMeta, currentPath);
83200
} else {
84201
console.error("No metadata found for the current path:", currentPath);
85202
}

‎robots.txt‎

Lines changed: 135 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,146 @@
1+
# Allow Googlebot-Image to index images
12
User-agent: Googlebot-Image
23
Disallow: /
34
Allow: /assets/images/
45

5-
User-agent: *
6+
# Google Search Bot
7+
User-agent: Googlebot
8+
Disallow: /assets/html/
9+
Disallow: /sitemap.html
10+
Disallow: /branding/
11+
Disallow: /404.html
12+
Disallow: /metadata.json
13+
Disallow: /.legacy/
14+
Disallow: /contact/
15+
Disallow: /projects/coldog-locker/
16+
Disallow: /projects/dj-coldog/
17+
Disallow: /legal/
18+
Allow: /assets/images/
19+
Allow: /assets/css/
20+
Allow: /assets/js/
21+
22+
# Bing Search Bot
23+
User-agent: Bingbot
24+
Disallow: /assets/html/
25+
Disallow: /sitemap.html
26+
Disallow: /branding/
27+
Disallow: /404.html
28+
Disallow: /metadata.json
29+
Disallow: /.legacy/
30+
Disallow: /contact/
31+
Disallow: /projects/coldog-locker/
32+
Disallow: /projects/dj-coldog/
33+
Disallow: /legal/
34+
Allow: /assets/images/
35+
Allow: /assets/css/
36+
Allow: /assets/js/
37+
38+
# DuckDuckGo Bot
39+
User-agent: DuckDuckBot
40+
Disallow: /assets/html/
41+
Disallow: /sitemap.html
42+
Disallow: /branding/
43+
Disallow: /404.html
44+
Disallow: /metadata.json
45+
Disallow: /.legacy/
46+
Disallow: /contact/
47+
Disallow: /projects/coldog-locker/
48+
Disallow: /projects/dj-coldog/
49+
Disallow: /legal/
50+
Allow: /assets/images/
51+
Allow: /assets/css/
52+
Allow: /assets/js/
53+
54+
# Brave Search Bot
55+
User-agent: Brave-Indexer
656
Disallow: /assets/html/
757
Disallow: /sitemap.html
858
Disallow: /branding/
59+
Disallow: /404.html
60+
Disallow: /metadata.json
61+
Disallow: /.legacy/
62+
Disallow: /contact/
63+
Disallow: /projects/coldog-locker/
64+
Disallow: /projects/dj-coldog/
965
Disallow: /legal/
66+
Allow: /assets/images/
67+
Allow: /assets/css/
68+
Allow: /assets/js/
69+
70+
# Block AI Training Bots - OpenAI
71+
User-agent: GPTBot
72+
Disallow: /
73+
74+
User-agent: ChatGPT-User
75+
Disallow: /
76+
77+
User-agent: CCBot
78+
Disallow: /
79+
80+
User-agent: anthropic-ai
81+
Disallow: /
82+
83+
User-agent: Claude-Web
84+
Disallow: /
85+
86+
# Block AI Training Bots - Google
87+
User-agent: Google-Extended
88+
Disallow: /
89+
90+
# Block AI Training Bots - Common AI Crawlers
91+
User-agent: PerplexityBot
92+
Disallow: /
93+
94+
User-agent: YouBot
95+
Disallow: /
96+
97+
User-agent: Diffbot
98+
Disallow: /
99+
100+
User-agent: FacebookBot
101+
Disallow: /
102+
103+
# Block AI Training Bots - Academic/Research
104+
User-agent: ia_archiver
105+
Disallow: /
106+
107+
User-agent: archive.org_bot
108+
Disallow: /
109+
110+
# Block AI Training Bots - Others
111+
User-agent: Claude-Web
112+
Disallow: /
113+
114+
User-agent: Omgilibot
115+
Disallow: /
116+
117+
User-agent: Bytespider
118+
Disallow: /
119+
120+
User-agent: DataForSeoBot
121+
Disallow: /
122+
123+
User-agent: AhrefsBot
124+
Disallow: /
125+
126+
User-agent: SemrushBot
127+
Disallow: /
128+
129+
User-agent: MJ12bot
130+
Disallow: /
131+
132+
# All other bots (fallback)
133+
User-agent: *
134+
Disallow: /assets/html/
135+
Disallow: /sitemap.html
136+
Disallow: /branding/
10137
Disallow: /404.html
11-
Disallow: /CNAME
12-
Disallow: /BusinessIdeas.txt
138+
Disallow: /metadata.json
139+
Disallow: /.legacy/
140+
Disallow: /contact/
141+
Disallow: /projects/coldog-locker/
142+
Disallow: /projects/dj-coldog/
143+
Disallow: /legal/
13144

145+
# Sitemap location
14146
Sitemap: https://www.coldogstudios.com/sitemap.xml

0 commit comments

Comments
 (0)