331 lines
12 KiB
JavaScript
331 lines
12 KiB
JavaScript
/*!
|
|
* Sitemap
|
|
* Copyright(c) 2011 Eugene Kalinin
|
|
* MIT Licensed
|
|
*/
|
|
import { createWriteStream, existsSync, statSync, } from 'node:fs';
|
|
import { dirname, resolve } from 'node:path';
|
|
import { pipeline, Readable, Transform, PassThrough, } from 'node:stream';
|
|
import { createInterface, Interface } from 'node:readline';
|
|
import { createGzip } from 'node:zlib';
|
|
import { URL } from 'node:url';
|
|
import { EnumYesNo } from './types.js';
|
|
import { LIMITS } from './constants.js';
|
|
import { validateURL } from './validation.js';
|
|
/**
|
|
* Per ADR 0001, CDATA sections are ordinary character data: wire both the SAX
|
|
* `text` and `cdata` events to `append`. Character data is joined, trimmed and
|
|
* handed to `onText` once at the next tag boundary, attributed to the innermost
|
|
* open element (which is not the last opened tag once a child has closed).
|
|
* An element whose character data exceeds LIMITS.MAX_PARSER_CHAR_DATA_LENGTH is
|
|
* dropped and reported through `onOverflow` to bound memory use.
|
|
*/
|
|
export function createCharDataBuffer({ onText, onOverflow, }) {
|
|
let charData = '';
|
|
let overflow = false;
|
|
const openTags = [];
|
|
const flush = () => {
|
|
const tag = openTags[openTags.length - 1];
|
|
const text = charData.trim();
|
|
charData = '';
|
|
if (overflow) {
|
|
overflow = false;
|
|
onOverflow(`character data for ${tag} exceeds max length of ${LIMITS.MAX_PARSER_CHAR_DATA_LENGTH}`);
|
|
}
|
|
else if (text) {
|
|
onText(tag, text);
|
|
}
|
|
};
|
|
return {
|
|
openTag(name) {
|
|
flush();
|
|
openTags.push(name);
|
|
},
|
|
append(text) {
|
|
if (overflow) {
|
|
return;
|
|
}
|
|
if (charData.length + text.length > LIMITS.MAX_PARSER_CHAR_DATA_LENGTH) {
|
|
overflow = true;
|
|
charData = '';
|
|
return;
|
|
}
|
|
charData += text;
|
|
},
|
|
closeTag() {
|
|
flush();
|
|
openTags.pop();
|
|
},
|
|
};
|
|
}
|
|
/**
|
|
* Combines multiple streams into one
|
|
* @param streams the streams to combine
|
|
*/
|
|
export function mergeStreams(streams, options) {
|
|
let pass = new PassThrough(options);
|
|
let waiting = streams.length;
|
|
for (const stream of streams) {
|
|
pass = stream.pipe(pass, { end: false });
|
|
stream.once('end', () => --waiting === 0 && pass.emit('end'));
|
|
}
|
|
return pass;
|
|
}
|
|
/**
|
|
* Wraps node's ReadLine in a stream
|
|
*/
|
|
export class ReadlineStream extends Readable {
|
|
_source;
|
|
constructor(options) {
|
|
if (options.autoDestroy === undefined) {
|
|
options.autoDestroy = true;
|
|
}
|
|
options.objectMode = true;
|
|
super(options);
|
|
this._source = createInterface({
|
|
input: options.input,
|
|
terminal: false,
|
|
crlfDelay: Infinity,
|
|
});
|
|
// Every time there's data, push it into the internal buffer.
|
|
this._source.on('line', (chunk) => {
|
|
// If push() returns false, then stop reading from source.
|
|
if (!this.push(chunk))
|
|
this._source.pause();
|
|
});
|
|
// When the source ends, push the EOF-signaling `null` chunk.
|
|
this._source.on('close', () => {
|
|
this.push(null);
|
|
});
|
|
}
|
|
// _read() will be called when the stream wants to pull more data in.
|
|
// The advisory size argument is ignored in this case.
|
|
_read(size) {
|
|
this._source.resume();
|
|
}
|
|
}
|
|
/**
|
|
* Takes a stream likely from fs.createReadStream('./path') and returns a stream
|
|
* of sitemap items
|
|
* @param stream a stream of line separated urls.
|
|
* @param opts.isJSON is the stream line separated JSON. leave undefined to guess
|
|
*/
|
|
export function lineSeparatedURLsToSitemapOptions(stream, { isJSON } = {}) {
|
|
return new ReadlineStream({ input: stream }).pipe(new Transform({
|
|
objectMode: true,
|
|
transform: (line, encoding, cb) => {
|
|
if (isJSON || (isJSON === undefined && line[0] === '{')) {
|
|
cb(null, JSON.parse(line));
|
|
}
|
|
else {
|
|
cb(null, line);
|
|
}
|
|
},
|
|
}));
|
|
}
|
|
/**
|
|
* Returns `err` if it is an Error, or wraps it in one, for passing a caught
|
|
* value to a stream callback
|
|
*/
|
|
export function toError(err) {
|
|
return err instanceof Error ? err : new Error(String(err));
|
|
}
|
|
function boolToYESNO(bool) {
|
|
if (bool === undefined) {
|
|
return undefined;
|
|
}
|
|
if (typeof bool === 'boolean') {
|
|
return bool ? EnumYesNo.yes : EnumYesNo.no;
|
|
}
|
|
return bool;
|
|
}
|
|
/**
|
|
* Converts a numeric string priority, such as `'0.5'`, to a number. Returns
|
|
* any other value unchanged, for validateSMIOptions to report.
|
|
*/
|
|
function toPriority(priority) {
|
|
if (typeof priority === 'string' &&
|
|
LIMITS.PRIORITY_STRING_REGEX.test(priority)) {
|
|
return Number(priority);
|
|
}
|
|
// Not a number: passed through so validation reports it at WARN and THROW
|
|
return priority;
|
|
}
|
|
/**
|
|
* Converts the passed in sitemap entry into one capable of being consumed by SitemapItem
|
|
* @param {string | SitemapItemLoose} elem the string or object to be converted
|
|
* @param {string} hostname
|
|
* @returns SitemapItemOptions a strict sitemap item option
|
|
*/
|
|
export function normalizeURL(elem, hostname, lastmodDateOnly = false) {
|
|
// SitemapItem
|
|
// create object with url property
|
|
const smi = {
|
|
img: [],
|
|
video: [],
|
|
links: [],
|
|
url: '',
|
|
};
|
|
if (typeof elem === 'string') {
|
|
smi.url = new URL(elem, hostname).toString();
|
|
return smi;
|
|
}
|
|
const { url, img, links, video, lastmodfile, lastmodISO, lastmod, priority, ...other } = elem;
|
|
Object.assign(smi, other);
|
|
if (priority !== undefined) {
|
|
smi.priority = toPriority(priority);
|
|
}
|
|
smi.url = new URL(url, hostname).toString();
|
|
if (img) {
|
|
// prepend hostname to all image urls
|
|
smi.img = (Array.isArray(img) ? img : [img]).map((el) => typeof el === 'string'
|
|
? { url: new URL(el, hostname).toString() }
|
|
: { ...el, url: new URL(el.url, hostname).toString() });
|
|
}
|
|
if (links) {
|
|
smi.links = links.map((link) => ({
|
|
...link,
|
|
url: new URL(link.url, hostname).toString(),
|
|
}));
|
|
}
|
|
if (video) {
|
|
smi.video = (Array.isArray(video) ? video : [video]).map((video) => {
|
|
const nv = {
|
|
...video,
|
|
family_friendly: boolToYESNO(video.family_friendly),
|
|
live: boolToYESNO(video.live),
|
|
requires_subscription: boolToYESNO(video.requires_subscription),
|
|
tag: [],
|
|
rating: undefined,
|
|
};
|
|
if (video.tag !== undefined) {
|
|
nv.tag = !Array.isArray(video.tag) ? [video.tag] : video.tag;
|
|
}
|
|
if (video.rating !== undefined) {
|
|
if (typeof video.rating === 'string') {
|
|
const parsedRating = parseFloat(video.rating);
|
|
// Validate parsed rating is a valid number
|
|
if (Number.isNaN(parsedRating)) {
|
|
throw new Error(`Invalid video rating "${video.rating}" for URL "${elem.url}": must be a valid number`);
|
|
}
|
|
nv.rating = parsedRating;
|
|
}
|
|
else {
|
|
nv.rating = video.rating;
|
|
}
|
|
}
|
|
if (typeof video.view_count === 'string') {
|
|
const parsedViewCount = parseInt(video.view_count, 10);
|
|
// Validate parsed view count is a valid non-negative integer
|
|
if (Number.isNaN(parsedViewCount)) {
|
|
throw new Error(`Invalid video view_count "${video.view_count}" for URL "${elem.url}": must be a valid number`);
|
|
}
|
|
if (parsedViewCount < 0) {
|
|
throw new Error(`Invalid video view_count "${video.view_count}" for URL "${elem.url}": cannot be negative`);
|
|
}
|
|
nv.view_count = parsedViewCount;
|
|
}
|
|
else if (typeof video.view_count === 'number') {
|
|
nv.view_count = video.view_count;
|
|
}
|
|
return nv;
|
|
});
|
|
}
|
|
// If given a file to use for last modified date
|
|
if (lastmodfile) {
|
|
const { mtime } = statSync(lastmodfile);
|
|
const lastmodDate = new Date(mtime);
|
|
// Validate date is valid
|
|
if (Number.isNaN(lastmodDate.getTime())) {
|
|
throw new Error(`Invalid date from file stats for URL "${smi.url}": file modification time is invalid`);
|
|
}
|
|
smi.lastmod = lastmodDate.toISOString();
|
|
// The date of last modification (YYYY-MM-DD)
|
|
}
|
|
else if (lastmodISO) {
|
|
const lastmodDate = new Date(lastmodISO);
|
|
// Validate date is valid
|
|
if (Number.isNaN(lastmodDate.getTime())) {
|
|
throw new Error(`Invalid lastmodISO "${lastmodISO}" for URL "${smi.url}": must be a valid date string`);
|
|
}
|
|
smi.lastmod = lastmodDate.toISOString();
|
|
}
|
|
else if (lastmod) {
|
|
const lastmodDate = new Date(lastmod);
|
|
// Validate date is valid
|
|
if (Number.isNaN(lastmodDate.getTime())) {
|
|
throw new Error(`Invalid lastmod "${lastmod}" for URL "${smi.url}": must be a valid date string`);
|
|
}
|
|
smi.lastmod = lastmodDate.toISOString();
|
|
}
|
|
if (lastmodDateOnly && smi.lastmod) {
|
|
smi.lastmod = smi.lastmod.slice(0, 10);
|
|
}
|
|
return smi;
|
|
}
|
|
/**
|
|
* Connects a sitemap to the stream that writes its file, through gzip when
|
|
* `gzip` is true. Uses `pipeline()`, which destroys every stream when one
|
|
* fails, so a gzip failure also fails `writeStream`, which
|
|
* `SitemapAndIndexStream` watches.
|
|
*
|
|
* @param onError - Called with the error if the pipeline fails.
|
|
*/
|
|
export function pipeSitemapToFile(sitemap, writeStream, gzip, onError) {
|
|
const streams = gzip
|
|
? [sitemap, createGzip(), writeStream]
|
|
: [sitemap, writeStream];
|
|
pipeline(streams, (err) => {
|
|
if (err) {
|
|
onError(err);
|
|
}
|
|
});
|
|
}
|
|
/**
|
|
* Walks up from `fromDir` to the filesystem root and returns the first path
|
|
* where `relativePath` exists, or undefined if there is none.
|
|
*
|
|
* Used to find files that ship with the package (`package.json`, the schema)
|
|
* from a module's own directory, whatever the build's output layout.
|
|
*/
|
|
export function findUp(relativePath, fromDir) {
|
|
let dir = fromDir;
|
|
for (;;) {
|
|
const candidate = resolve(dir, relativePath);
|
|
if (existsSync(candidate)) {
|
|
return candidate;
|
|
}
|
|
const parent = dirname(dir);
|
|
if (parent === dir) {
|
|
return undefined;
|
|
}
|
|
dir = parent;
|
|
}
|
|
}
|
|
/**
|
|
* Opens sitemap number `index` as a file, for a `SitemapAndIndexStream`
|
|
* `getSitemapStream` callback: names it `sitemap-N.xml` (`.xml.gz` when
|
|
* gzipped), builds its URL, checks the URL before creating the file so a URL
|
|
* the index would reject leaves no empty file behind, and pipes `sitemap` into
|
|
* it.
|
|
*
|
|
* @returns the `[url, sitemap, writeStream]` tuple `getSitemapStream` returns
|
|
*/
|
|
export function openSitemapFile(index, sitemap, { destinationDir, gzip, toURL, urlParamName, onError }) {
|
|
const fileName = `sitemap-${index}.xml${gzip ? '.gz' : ''}`;
|
|
const url = toURL(fileName);
|
|
validateURL(url, urlParamName);
|
|
const writeStream = createWriteStream(resolve(destinationDir, fileName));
|
|
// SitemapAndIndexStream watches writeStream; onError reports any failure
|
|
// that comes after it stopped watching.
|
|
pipeSitemapToFile(sitemap, writeStream, gzip, onError);
|
|
return [url, sitemap, writeStream];
|
|
}
|
|
/**
|
|
* Message text of a caught value, for embedding in a wrapping error's message.
|
|
* @param err - anything caught by a `catch` clause
|
|
*/
|
|
export function errorMessage(err) {
|
|
return err instanceof Error ? err.message : String(err);
|
|
}
|