diff --git a/app/scripts/generate-seo.mjs b/app/scripts/generate-seo.mjs new file mode 100644 index 0000000..7b64cf6 --- /dev/null +++ b/app/scripts/generate-seo.mjs @@ -0,0 +1,365 @@ +import { existsSync, mkdirSync, writeFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { TeknavData } from '../teknav-data.js'; + +const SITE_URL = (process.env.TEKNAV_SITE_URL || process.env.VITE_SITE_URL || 'https://www.teknav.ir').replace(/\/$/, ''); +const today = new Date().toISOString().slice(0, 10); +const SITEMAP_XMLNS = 'xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"'; +const XHTML_XMLNS = 'xmlns:xhtml="http://www.w3.org/1999/xhtml"'; +const IMAGE_XMLNS = 'xmlns:image="http://www.google.com/schemas/sitemap-image/1.1"'; +const NEWS_XMLNS = 'xmlns:news="http://www.google.com/schemas/sitemap-news/0.9"'; + +function xml(value) { + return String(value ?? '') + .replace(/\uFFFD/g, '') + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); +} + +function capDate(date) { + return String(date ?? today).slice(0, 10) > today ? today : String(date ?? today).slice(0, 10); +} + +function isPastOrTodayDate(date) { + const value = String(date ?? '').slice(0, 10); + return /^\d{4}-\d{2}-\d{2}$/.test(value) && value <= today; +} + +function assertBoundNamespacePrefixes(name, source) { + const declared = new Set([...source.matchAll(/\sxmlns:([A-Za-z_][\w.-]*)=/g)].map((match) => match[1])); + const used = new Set([...source.matchAll(/<\/?([A-Za-z_][\w.-]*):[A-Za-z_][\w.-]*(?:\s|>|\/>)/g)].map((match) => match[1])); + const missing = [...used].filter((prefix) => !declared.has(prefix)); + if (missing.length > 0) { + throw new Error(`[seo] ${name} has unbound XML namespace prefix(es): ${missing.join(', ')}`); + } +} + +function writeXml(path, source) { + assertBoundNamespacePrefixes(path, source); + writeFileSync(resolve(path), source, 'utf8'); +} + +function url(path, lastmod = today, priority = '0.7', changefreq = 'weekly', images = []) { + const lines = [ + ' ', + ` ${SITE_URL}${path}`, + ` ${capDate(lastmod)}`, + ` ${changefreq}`, + ` ${priority}`, + ]; + if (images && images.length > 0) { + for (const img of images) { + lines.push(' '); + lines.push(` ${SITE_URL}${img.url}`); + if (img.title) lines.push(` ${xml(img.title)}`); + if (img.caption) lines.push(` ${xml(img.caption)}`); + lines.push(' '); + } + } + lines.push(' '); + return lines.join('\n'); +} + +function newsLanguage(article) { + return String(article?.language || article?.inLanguage || 'fa-IR').toLocaleLowerCase('en-US').startsWith('en') ? 'en' : 'fa'; +} + +function newsUrl(path, title, date, name = 'تکنّاو', language = 'fa') { + return [ + ' ', + ` ${SITE_URL}${path}`, + ' ', + ' ', + ` ${xml(name)}`, + ` ${xml(language)}`, + ' ', + ` ${date}T09:00:00+03:30`, + ` ${xml(title)}`, + ' ', + ' ', + ].join('\n'); +} + +function text(value) { + return String(value ?? '').replace(/\uFFFD/g, '').replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim(); +} + +function cdata(value) { + return String(value ?? '').replace(/\uFFFD/g, '').replace(/]]>/g, ']]]]>'); +} + +function isPublishedArticle(article) { + if (article.status === 'منتشرشده') return true; + if (article.status === 'پیش‌نویس' || article.status === 'در انتظار بررسی') return false; + if (['rust-systems', 'venture-capital-ai'].includes(article.slug)) return false; + return !!article.dateEn; +} + +const publishedArticles = TeknavData.articles.filter(isPublishedArticle); +const indexableArticles = publishedArticles.filter((article) => isPastOrTodayDate(article.dateEn)); +const topicHubs = TeknavData.topicHubs ?? []; +const seriesPages = TeknavData.articleSeries ?? []; + +const englishSitemap = ` + +${url('/en/', today, '0.95', 'daily')} + +`; + +// Main Sitemap +const mainUrls = [ + url('/', today, '1.0', 'daily'), + url('/articles', today, '0.9', 'daily'), + url('/authors', today, '0.7', 'weekly'), + url('/newsletter', today, '0.7', 'weekly'), + url('/jobs', today, '0.55', 'weekly'), + url('/courses', today, '0.55', 'monthly'), + url('/membership', today, '0.45', 'monthly'), + url('/search', today, '0.4', 'monthly'), + url('/about', today, '0.6', 'monthly'), + url('/contact', today, '0.5', 'monthly'), + url('/tools', today, '0.55', 'weekly'), + url('/tools/nlp-normalizer', today, '0.55', 'weekly'), + url('/tools/rag-calculator', today, '0.55', 'weekly'), + url('/tekrag-persian', today, '0.7', 'weekly'), + url('/directory', today, '0.7', 'weekly'), + url('/compare', today, '0.7', 'weekly'), + url('/privacy', today, '0.4', 'yearly'), + url('/terms', today, '0.4', 'yearly'), + ...TeknavData.categories.map((category) => url(`/category/${category.slug}`, today, '0.8', 'weekly')), + ...TeknavData.tags.map((tag) => url(`/tag/${encodeURIComponent(tag.name)}`, today, '0.55', 'weekly')), + ...(TeknavData.glossary ?? []).map((term) => url(`/glossary/${term.slug}`, today, '0.55', 'monthly')), + ...topicHubs.map((topic) => url(`/topics/${topic.slug}`, today, '0.9', 'weekly')), + ...TeknavData.authors.map((author) => url(`/author/${author.slug}`, today, '0.7', 'weekly')), + ...TeknavData.authors.filter((author) => author.username).map((author) => url(`/profile/@${author.username}`, today, '0.6', 'weekly')), + ...seriesPages.map((series) => url(`/series/${series.slug}`, today, '0.75', 'weekly')), +]; + +const sitemap = ` + +${mainUrls.join('\n')} + +`; + +// Articles Sitemap (with Images) +const articleUrls = indexableArticles.map((article) => { + const slugImage = `/images/og/${article.slug}.jpg`; + const image = article.ogImage || (existsSync(resolve(`public${slugImage}`)) ? slugImage : null); + const images = image ? [{ url: image, title: article.title, caption: article.summary || article.subtitle }] : []; + return url(article.canonicalPath || `/article/${article.slug}`, String(article.dateModified || article.dateEn || today).slice(0, 10), '0.85', 'monthly', images); +}); + +const articlesSitemap = ` + +${articleUrls.join('\n')} + +`; + +// News Sitemap: Google News expects only recent, already-published articles. +const newsCutoff = new Date(); +newsCutoff.setUTCDate(newsCutoff.getUTCDate() - 7); +const newsCutoffDate = newsCutoff.toISOString().slice(0, 10); +const recentArticles = indexableArticles.filter(a => { + const date = String(a.dateEn ?? '').slice(0, 10); + return date >= newsCutoffDate && date <= today; +}); + +const newsSitemap = ` + +${recentArticles.map((a) => { + const language = newsLanguage(a); + return newsUrl(a.canonicalPath || `/article/${a.slug}`, a.title, a.dateEn, language === 'en' ? 'Teknav' : 'تکنّاو', language); +}).join('\n')} + +`; + +// Stories Sitemap +const storiesUrls = indexableArticles.map((article) => + url(`/stories/${article.slug}`, String(article.dateEn || today).slice(0, 10), '0.6', 'monthly'), +); +const storiesSitemap = ` + +${storiesUrls.join('\n')} + +`; + +// Comparison Pages Sitemap +const comparisonUrls = (TeknavData.comparisons ?? []).map((c) => + url(`/compare/${c.slug}`, String(c.dateEn || today).slice(0, 10), '0.7', 'monthly'), +); +const comparisonsSitemap = comparisonUrls.length > 0 ? ` + +${comparisonUrls.join('\n')} + +` : null; + +// Company Directory Sitemap +const companyUrls = (TeknavData.companies ?? []).map((c) => + url(`/directory/${c.slug}`, today, '0.6', 'weekly'), +); +const companiesSitemap = companyUrls.length > 0 ? ` + +${companyUrls.join('\n')} + +` : null; + +// Sitemap Index +const sitemapIndex = ` + + + ${SITE_URL}/sitemap-main.xml + ${today} + + + ${SITE_URL}/sitemap-en.xml + ${today} + + + ${SITE_URL}/sitemap-articles.xml + ${today} + + ${recentArticles.length > 0 ? ` + ${SITE_URL}/sitemap-news.xml + ${today} + ` : ''} + + ${SITE_URL}/sitemap-stories.xml + ${today} + + ${comparisonUrls.length > 0 ? ` + ${SITE_URL}/sitemap-comparisons.xml + ${today} + ` : ''} + ${companyUrls.length > 0 ? ` + ${SITE_URL}/sitemap-companies.xml + ${today} + ` : ''} + +`; + +const robots = `User-agent: * +Allow: / +Disallow: /admin +Disallow: /login +Disallow: /signup +Disallow: /forgot-password +Disallow: /reset-password +Disallow: /verify-email +Disallow: /writer +Disallow: /ACCOUNTS.md +Disallow: /api/ + +Sitemap: ${SITE_URL}/sitemap.xml +Sitemap: ${SITE_URL}/sitemap-main.xml +Sitemap: ${SITE_URL}/sitemap-en.xml +Sitemap: ${SITE_URL}/sitemap-articles.xml +Sitemap: ${SITE_URL}/sitemap-news.xml +Sitemap: ${SITE_URL}/sitemap-stories.xml +`; + +const feedItems = indexableArticles + .slice() + .sort((a, b) => String(b.dateEn || '').localeCompare(String(a.dateEn || ''))) + .slice(0, 20) + .map((article) => { + const link = `${SITE_URL}${article.canonicalPath || `/article/${article.slug}`}`; + const pubDate = article.dateEn ? new Date(`${article.dateEn}T09:00:00+03:30`).toUTCString() : new Date().toUTCString(); + return [ + ' ', + ` ${xml(article.title)}`, + ` ${xml(link)}`, + ` ${xml(link)}`, + ` ${xml(article.metaDescription || text(article.summary || article.subtitle))}`, + ` `, + ` ${xml(article.categoryName || article.category || 'فناوری')}`, + ` editor@teknav.ir (${xml(article.authorName || 'تکناو')})`, + ` ${pubDate}`, + ' ', + ].join('\n'); + }) + .join('\n'); + +const feed = ` + + + تکناو - تازه‌ترین تحلیل‌های فناوری + ${xml(SITE_URL)}/ + + مقاله‌ها و تحلیل‌های فارسی تکناو درباره هوش مصنوعی، علم داده، امنیت سایبری، نرم‌افزار، سخت‌افزار و آینده فناوری. + fa-IR + ${new Date().toUTCString()} +${feedItems} + + +`; + +function scopedFeed({ title, link, description, articles }) { + const scopedItems = articles + .slice() + .sort((a, b) => String(b.dateEn || '').localeCompare(String(a.dateEn || ''))) + .slice(0, 20) + .map((article) => { + const articleLink = `${SITE_URL}${article.canonicalPath || `/article/${article.slug}`}`; + const pubDate = article.dateEn ? new Date(`${article.dateEn}T09:00:00+03:30`).toUTCString() : new Date().toUTCString(); + return [ + ' ', + ` ${xml(article.title)}`, + ` ${xml(articleLink)}`, + ` ${xml(articleLink)}`, + ` ${xml(article.metaDescription || text(article.summary || article.subtitle))}`, + ` `, + ` ${xml(article.categoryName || article.category || 'فناوری')}`, + ` editor@teknav.ir (${xml(article.authorName || 'تکنّاو')})`, + ` ${pubDate}`, + ' ', + ].join('\n'); + }) + .join('\n'); + return ` + + + ${xml(title)} + ${xml(link)} + + ${xml(description)} + fa-IR + ${new Date().toUTCString()} +${scopedItems} + + +`; +} + +writeXml('public/sitemap-main.xml', sitemap); +writeXml('public/sitemap-en.xml', englishSitemap); +writeXml('public/sitemap-articles.xml', articlesSitemap); +writeXml('public/sitemap-news.xml', newsSitemap); +writeXml('public/sitemap-stories.xml', storiesSitemap); +if (comparisonsSitemap) writeXml('public/sitemap-comparisons.xml', comparisonsSitemap); +if (companiesSitemap) writeXml('public/sitemap-companies.xml', companiesSitemap); +writeXml('public/sitemap.xml', sitemapIndex); +writeFileSync(resolve('public/robots.txt'), robots, 'utf8'); +writeFileSync(resolve('public/feed.xml'), feed, 'utf8'); +mkdirSync(resolve('public/feeds'), { recursive: true }); +for (const topic of topicHubs) { + const topicArticles = indexableArticles.filter((article) => article.category === topic.categorySlug || article.categorySlug === topic.categorySlug); + writeFileSync(resolve(`public/feeds/topic-${topic.slug}.xml`), scopedFeed({ + title: `${topic.title} | خوراک موضوعی تکنّاو`, + link: `${SITE_URL}/feeds/topic-${topic.slug}.xml`, + description: topic.description || `تازه‌ترین نوشته‌های تکنّاو درباره ${topic.title}.`, + articles: topicArticles, + }), 'utf8'); +} +for (const author of TeknavData.authors) { + const authorArticles = indexableArticles.filter((article) => article.authorId === author.id || article.authorSlug === author.slug || article.authorName === author.name); + writeFileSync(resolve(`public/feeds/author-${author.slug}.xml`), scopedFeed({ + title: `${author.name} | خوراک نویسنده در تکنّاو`, + link: `${SITE_URL}/feeds/author-${author.slug}.xml`, + description: `آخرین نوشته‌های ${author.name} در تکنّاو.`, + articles: authorArticles, + }), 'utf8'); +} +console.log(`[seo] generated sitemap.xml (index), main, articles, ${recentArticles.length > 0 ? 'news, ' : ''}feed.xml, ${topicHubs.length} topic feeds, and ${TeknavData.authors.length} author feeds`);