248 lines
7.4 KiB
Plaintext
248 lines
7.4 KiB
Plaintext
// src/index.ts
|
|
import fs from "node:fs";
|
|
import { ZodError } from "zod";
|
|
|
|
// ../utils/src/is-valid-hostname.ts
|
|
var isValidHostname = (x) => {
|
|
if (typeof x !== "string") {
|
|
return false;
|
|
}
|
|
let value = x.toString();
|
|
const validHostnameChars = /^[a-zA-Z0-9-.]{1,253}\.?$/g;
|
|
if (!validHostnameChars.test(value)) {
|
|
return false;
|
|
}
|
|
if (value.endsWith(".")) {
|
|
value = value.slice(0, value.length - 1);
|
|
}
|
|
if (value.length > 253) {
|
|
return false;
|
|
}
|
|
return value.split(".").every((label) => /^([a-zA-Z0-9-]+)$/g.test(label) && label.length < 64 && !label.startsWith("-") && !label.endsWith("-"));
|
|
};
|
|
|
|
// ../utils/src/is-valid-http-url.ts
|
|
var isValidHttpUrl = (s) => {
|
|
if (typeof s !== "string" || !s) {
|
|
return false;
|
|
}
|
|
try {
|
|
const { protocol } = new URL(s);
|
|
return protocol === "http:" || protocol === "https:";
|
|
} catch {
|
|
return false;
|
|
}
|
|
};
|
|
|
|
// ../utils/src/logger.ts
|
|
var Logger = class {
|
|
constructor(packageName2) {
|
|
this.colors = {
|
|
reset: "\x1B[0m",
|
|
fg: {
|
|
red: "\x1B[31m",
|
|
green: "\x1B[32m",
|
|
yellow: "\x1B[33m"
|
|
}
|
|
};
|
|
this.packageName = packageName2;
|
|
}
|
|
log(msg, prefix = "") {
|
|
const s = msg.join("\n");
|
|
console.log(`%s${this.packageName}:%s ${s}
|
|
`, prefix, prefix ? this.colors.reset : "");
|
|
}
|
|
info(...msg) {
|
|
this.log(msg);
|
|
}
|
|
success(...msg) {
|
|
this.log(msg, this.colors.fg.green);
|
|
}
|
|
warn(...msg) {
|
|
this.log(["Skipped!", ...msg], this.colors.fg.yellow);
|
|
}
|
|
error(...msg) {
|
|
this.log(["Failed!", ...msg], this.colors.fg.red);
|
|
}
|
|
};
|
|
|
|
// ../utils/src/error-helpers.ts
|
|
function getErrorMessage(err) {
|
|
return err instanceof Error ? err.message : String(err);
|
|
}
|
|
|
|
// src/validate-options.ts
|
|
import { z as z2 } from "zod";
|
|
|
|
// src/schema.ts
|
|
import { z } from "zod";
|
|
import isValidFilename from "valid-filename";
|
|
|
|
// src/config-defaults.ts
|
|
var ROBOTS_TXT_CONFIG_DEFAULTS = {
|
|
sitemap: true,
|
|
policy: [
|
|
{
|
|
allow: "/",
|
|
userAgent: "*"
|
|
}
|
|
],
|
|
sitemapBaseFileName: "sitemap-index"
|
|
};
|
|
|
|
// src/schema.ts
|
|
var schemaSitemapItem = z.string().min(1).refine((val) => !val || isValidHttpUrl(val), {
|
|
message: "Only valid URLs with `http` or `https` protocol allowed"
|
|
});
|
|
var schemaCleanParam = z.string().max(500);
|
|
var schemaPath = z.string().or(z.string().array()).optional();
|
|
var RobotsTxtOptionsSchema = z.object({
|
|
host: z.string().or(z.boolean()).optional().refine((val) => !val || typeof val === "boolean" || isValidHostname(val), {
|
|
message: "Not valid host"
|
|
}),
|
|
sitemap: schemaSitemapItem.or(schemaSitemapItem.array()).or(z.boolean()).optional().default(ROBOTS_TXT_CONFIG_DEFAULTS.sitemap),
|
|
policy: z.object({
|
|
userAgent: z.string().min(1),
|
|
allow: schemaPath,
|
|
disallow: schemaPath,
|
|
cleanParam: schemaCleanParam.or(schemaCleanParam.array()).optional(),
|
|
crawlDelay: z.number().nonnegative().optional().refine((val) => typeof val === "undefined" || Number.isFinite(val), { message: "Must be finite number" })
|
|
}).array().nonempty().optional().default(ROBOTS_TXT_CONFIG_DEFAULTS.policy),
|
|
sitemapBaseFileName: z.string().min(1).optional().refine((val) => !val || isValidFilename(val), { message: "Not valid file name" }).default(ROBOTS_TXT_CONFIG_DEFAULTS.sitemapBaseFileName),
|
|
transform: z.function().args(z.string()).returns(z.any()).optional()
|
|
}).default(ROBOTS_TXT_CONFIG_DEFAULTS);
|
|
|
|
// src/validate-options.ts
|
|
var validateOptions = (site, opts) => {
|
|
const siteSchema = z2.string().min(1, {
|
|
message: "`site` property is required in `astro.config.*`."
|
|
});
|
|
siteSchema.parse(site);
|
|
const result = RobotsTxtOptionsSchema.parse(opts);
|
|
return result;
|
|
};
|
|
|
|
// src/get-robots-txt-content.ts
|
|
var capitaliseFirstLetter = (s) => s.charAt(0).toUpperCase() + s.slice(1);
|
|
var addBackSlash = (s) => s.endsWith("/") ? s : `${s}/`;
|
|
var addLine = (name, rule) => {
|
|
if (rule && Array.isArray(rule) && rule.length > 0) {
|
|
let content = "";
|
|
rule.forEach((item) => {
|
|
content += addLine(name, item);
|
|
});
|
|
return content;
|
|
}
|
|
const ruleContent = name === "Allow" || name === "Disallow" ? encodeURI(rule.toString()) : rule.toString();
|
|
return `${capitaliseFirstLetter(name.replace(/([a-z])([A-Z])/g, "$1-$2").toLowerCase())}:${ruleContent.length > 0 ? ` ${ruleContent}` : ""}
|
|
`;
|
|
};
|
|
var generatePoliceItem = (item, index) => {
|
|
let content = "";
|
|
if (index !== 0) {
|
|
content += "\n";
|
|
}
|
|
content += addLine("User-agent", item.userAgent);
|
|
if (typeof item.disallow === "string" || Array.isArray(item.disallow)) {
|
|
content += addLine("Disallow", item.disallow);
|
|
}
|
|
if (item.allow) {
|
|
content += addLine("Allow", item.allow);
|
|
}
|
|
if (item.crawlDelay) {
|
|
content += addLine("Crawl-delay", item.crawlDelay);
|
|
}
|
|
if (item.cleanParam && item.cleanParam.length > 0) {
|
|
content += addLine("Clean-param", item.cleanParam);
|
|
}
|
|
return content;
|
|
};
|
|
var getSitemapArr = (sitemap, finalSiteHref, sitemapBaseFileName) => {
|
|
if (typeof sitemap !== "undefined") {
|
|
if (!sitemap) {
|
|
return void 0;
|
|
}
|
|
if (Array.isArray(sitemap)) {
|
|
return sitemap;
|
|
}
|
|
if (typeof sitemap === "string") {
|
|
return [sitemap];
|
|
}
|
|
}
|
|
return [`${addBackSlash(finalSiteHref)}${sitemapBaseFileName}.xml`];
|
|
};
|
|
var getRobotsTxtContent = (finalSiteHref, opts, site) => {
|
|
var _a;
|
|
const { host, sitemap, policy, sitemapBaseFileName } = opts;
|
|
let result = "";
|
|
policy == null ? void 0 : policy.forEach((item, index) => {
|
|
result += generatePoliceItem(item, index);
|
|
});
|
|
(_a = getSitemapArr(sitemap, finalSiteHref, sitemapBaseFileName)) == null ? void 0 : _a.forEach((item) => {
|
|
result += addLine("Sitemap", item);
|
|
});
|
|
if (host) {
|
|
let hostStr;
|
|
if (typeof host === "boolean") {
|
|
const { hostname } = new URL(site);
|
|
hostStr = hostname;
|
|
} else {
|
|
hostStr = host;
|
|
}
|
|
result += addLine("Host", hostStr);
|
|
}
|
|
return result;
|
|
};
|
|
|
|
// src/data/pkg-name.ts
|
|
var packageName = "astro-robots-txt";
|
|
|
|
// src/index.ts
|
|
function formatConfigErrorMessage(err) {
|
|
const errorList = err.issues.map((issue) => `${issue.path.join(".")} ${issue.message + "."}`);
|
|
return errorList.join("\n");
|
|
}
|
|
var createRobotsTxtIntegration = (options = {}) => {
|
|
let config;
|
|
return {
|
|
name: packageName,
|
|
hooks: {
|
|
"astro:config:done": async ({ config: cfg }) => {
|
|
config = cfg;
|
|
},
|
|
"astro:build:done": async ({ dir }) => {
|
|
const logger = new Logger(packageName);
|
|
try {
|
|
const opts = validateOptions(config.site, options);
|
|
const finalSiteHref = new URL(config.base, config.site).href;
|
|
let robotsTxtContent = getRobotsTxtContent(finalSiteHref, opts, config.site);
|
|
if (opts.transform) {
|
|
try {
|
|
robotsTxtContent = await Promise.resolve(opts.transform(robotsTxtContent));
|
|
if (!robotsTxtContent) {
|
|
logger.warn("No content after transform.");
|
|
return;
|
|
}
|
|
} catch (err) {
|
|
logger.error("Error transforming content", getErrorMessage(err));
|
|
return;
|
|
}
|
|
}
|
|
fs.writeFileSync(new URL("robots.txt", dir), robotsTxtContent);
|
|
logger.success("`robots.txt` is created.");
|
|
} catch (err) {
|
|
if (err instanceof ZodError) {
|
|
logger.warn(formatConfigErrorMessage(err));
|
|
} else {
|
|
throw err;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
};
|
|
};
|
|
var src_default = createRobotsTxtIntegration;
|
|
export {
|
|
src_default as default
|
|
};
|