156 lines
6.0 KiB
JavaScript
156 lines
6.0 KiB
JavaScript
import { Readable, Transform, } from 'node:stream';
|
|
import { IndexTagNames, } from './types.js';
|
|
import { getURLError, isValidISODate, isValidMaxEntries, } from './validation.js';
|
|
import { LIMITS } from './constants.js';
|
|
import { SaxParserStream, collectEntries } from './sax-parser-stream.js';
|
|
function isValidTagName(tagName) {
|
|
// This only works because the enum name and value are the same
|
|
return tagName in IndexTagNames;
|
|
}
|
|
function tagTemplate() {
|
|
return {
|
|
url: '',
|
|
};
|
|
}
|
|
const defaultStreamOpts = {};
|
|
/**
|
|
* Takes a stream of xml and transforms it into a stream of IndexItems
|
|
* Use this to parse existing sitemap indices into config options compatible with this library
|
|
*
|
|
* Emits `'maxEntriesExceeded'` (with the cap) once, when the first entry past
|
|
* `maxEntries` is seen. If nothing listens for that event, the stream logs the
|
|
* overflow through its logger instead.
|
|
*/
|
|
export class XMLToSitemapIndexStream extends SaxParserStream {
|
|
currentItem = tagTemplate();
|
|
constructor(opts = defaultStreamOpts) {
|
|
super(opts, {
|
|
isKnownTag: isValidTagName,
|
|
overflowMessage: (max) => `Sitemap index exceeds maximum of ${max} entries`,
|
|
});
|
|
}
|
|
onCharData(tag, text) {
|
|
switch (tag) {
|
|
case IndexTagNames.loc:
|
|
// Validate URL for security: prevents protocol injection, checks length limits
|
|
{
|
|
const errMsg = getURLError(text, 'Sitemap index URL');
|
|
if (errMsg) {
|
|
this.report('warn', `Invalid URL in sitemap index: ${errMsg}`);
|
|
}
|
|
else {
|
|
this.currentItem.url = text;
|
|
}
|
|
}
|
|
break;
|
|
case IndexTagNames.lastmod:
|
|
// Validate date format for security and spec compliance
|
|
if (!isValidISODate(text)) {
|
|
this.report('warn', `Invalid lastmod date format in sitemap index: ${text}`);
|
|
}
|
|
else {
|
|
this.currentItem.lastmod = text;
|
|
}
|
|
break;
|
|
default:
|
|
this.logger('log', 'unhandled text for tag:', tag, `'${text}'`);
|
|
this.err(`unhandled text for tag: ${tag} '${text}'`);
|
|
break;
|
|
}
|
|
}
|
|
onAttribute(currentTag, attr) {
|
|
switch (currentTag) {
|
|
case IndexTagNames.sitemapindex:
|
|
break;
|
|
default:
|
|
this.logger('log', 'unhandled attr', currentTag, attr.name);
|
|
this.err(`unhandled attr: ${currentTag} ${attr.name}`);
|
|
}
|
|
}
|
|
onCloseTag(tag) {
|
|
if (tag === IndexTagNames.sitemap) {
|
|
// Only emit items with valid URLs (non-empty after validation). Invalid
|
|
// ones were already reported and don't count toward maxEntries.
|
|
if (this.currentItem.url) {
|
|
this.emitEntry(this.currentItem);
|
|
}
|
|
this.currentItem = tagTemplate();
|
|
}
|
|
}
|
|
}
|
|
/**
|
|
Read xml and resolve with the configuration that would produce it or reject with
|
|
an error
|
|
```
|
|
import { createReadStream } from 'fs'
|
|
import { Readable } from 'stream'
|
|
import { parseSitemapIndex, SitemapIndexStream, streamToPromise } from 'sitemap'
|
|
const items = await parseSitemapIndex(createReadStream('./example-index.xml'))
|
|
// produces the same xml
|
|
// you can, of course, more practically modify it or store it
|
|
const smis = new SitemapIndexStream()
|
|
Readable.from(items).pipe(smis)
|
|
console.log((await streamToPromise(smis)).toString())
|
|
```
|
|
@param {Readable} xml what to parse
|
|
@param {number} maxEntries Maximum number of sitemap entries to parse (default: 50,000 per sitemaps.org spec).
|
|
Values above 50,000 have no effect: the spec caps a sitemap index at 50,000 entries, and larger sets must be split.
|
|
0 rejects on the first entry. Anything other than a non-negative integer or Infinity rejects immediately.
|
|
@return {Promise<IndexItem[]>} resolves with list of index items that can be fed into a SitemapIndexStream. Rejects with an Error object.
|
|
*/
|
|
export async function parseSitemapIndex(xml, maxEntries = LIMITS.MAX_SITEMAP_ITEM_LIMIT) {
|
|
if (!isValidMaxEntries(maxEntries)) {
|
|
// Release the source before rejecting; nothing else will read it
|
|
xml.destroy();
|
|
throw new Error(`maxEntries must be a non-negative integer or Infinity, got ${maxEntries}`);
|
|
}
|
|
// The spec caps an index at 50,000 entries, so a larger maxEntries has no
|
|
// effect. The stream drops entries past this cap instead of pushing them.
|
|
const cap = Math.min(maxEntries, LIMITS.MAX_SITEMAP_ITEM_LIMIT);
|
|
// Listening for the overflow also stops the parser logging it, since this
|
|
// rejection already reports it.
|
|
return collectEntries(xml, new XMLToSitemapIndexStream({ maxEntries: cap }), () => new Error(`Sitemap index exceeds maximum allowed entries (${cap})`));
|
|
}
|
|
const defaultObjectStreamOpts = {
|
|
lineSeparated: false,
|
|
};
|
|
/**
|
|
* A Transform that converts a stream of objects into a JSON Array or a line
|
|
* separated stringified JSON
|
|
* @param [lineSeparated=false] whether to separate entries by a new line or comma
|
|
*/
|
|
export class IndexObjectStreamToJSON extends Transform {
|
|
lineSeparated;
|
|
firstWritten;
|
|
constructor(opts = defaultObjectStreamOpts) {
|
|
opts.writableObjectMode = true;
|
|
super(opts);
|
|
this.lineSeparated = opts.lineSeparated;
|
|
this.firstWritten = false;
|
|
}
|
|
_transform(chunk, encoding, cb) {
|
|
if (!this.firstWritten) {
|
|
this.firstWritten = true;
|
|
if (!this.lineSeparated) {
|
|
this.push('[');
|
|
}
|
|
}
|
|
else if (this.lineSeparated) {
|
|
this.push('\n');
|
|
}
|
|
else {
|
|
this.push(',');
|
|
}
|
|
if (chunk) {
|
|
this.push(JSON.stringify(chunk));
|
|
}
|
|
cb();
|
|
}
|
|
_flush(cb) {
|
|
if (!this.lineSeparated) {
|
|
this.push(']');
|
|
}
|
|
cb();
|
|
}
|
|
}
|