213 lines
7.6 KiB
JavaScript
213 lines
7.6 KiB
JavaScript
import sax from 'sax';
|
|
import { Transform } from 'node:stream';
|
|
import { ErrorLevel, } from './types.js';
|
|
import { isValidMaxEntries } from './validation.js';
|
|
import { LIMITS } from './constants.js';
|
|
import { createCharDataBuffer } from './utils.js';
|
|
import { resolveLogger } from './logger.js';
|
|
/**
|
|
* Shared plumbing for the XML parser streams: sax setup, character-data
|
|
* buffering, error collection, backpressure, and the entry cap. Subclasses
|
|
* supply tag semantics through the `on*` hooks and hand each finished
|
|
* entry to `emitEntry`.
|
|
*
|
|
* Emits `'maxEntriesExceeded'` (with the cap) once, when the first entry past
|
|
* `maxEntries` is seen. If nothing listens for that event, the stream logs the
|
|
* overflow through its logger instead.
|
|
*/
|
|
export class SaxParserStream extends Transform {
|
|
level;
|
|
logger;
|
|
/**
|
|
* Errors encountered during parsing, capped at LIMITS.MAX_PARSER_ERRORS entries
|
|
* to prevent memory DoS from malformed XML (BB-03).
|
|
* Use errorCount for the total number of errors regardless of the cap.
|
|
*/
|
|
errors;
|
|
/** Total number of errors seen, including those beyond the stored cap. */
|
|
errorCount;
|
|
/** Number of entries emitted or dropped past `maxEntries` */
|
|
entryCount;
|
|
saxStream;
|
|
maxEntries;
|
|
overflowMessage;
|
|
constructor(opts, config) {
|
|
super({ ...opts, objectMode: true });
|
|
this.errors = [];
|
|
this.errorCount = 0;
|
|
this.entryCount = 0;
|
|
this.overflowMessage = config.overflowMessage;
|
|
this.maxEntries = opts.maxEntries ?? LIMITS.MAX_SITEMAP_ITEM_LIMIT;
|
|
if (!isValidMaxEntries(this.maxEntries)) {
|
|
throw new Error(`maxEntries must be a non-negative integer or Infinity, got ${this.maxEntries}`);
|
|
}
|
|
this.level = opts.level || ErrorLevel.WARN;
|
|
this.logger = resolveLogger(this.level, opts.logger);
|
|
this.saxStream = sax.createStream(true, {
|
|
xmlns: true,
|
|
// @ts-expect-error - SAX types don't include strictEntities option
|
|
strictEntities: true,
|
|
// No sax trim: it trims each text chunk separately, dropping whitespace
|
|
// between text and CDATA. The char-data buffer trims the joined value.
|
|
trim: false,
|
|
});
|
|
// Per ADR 0001, CDATA sections are ordinary character data: both SAX
|
|
// events append to one buffer, which hands the joined value for an
|
|
// element to onCharData.
|
|
const charData = createCharDataBuffer({
|
|
onText: (tag, text) => this.onCharData(tag, text),
|
|
onOverflow: (msg) => {
|
|
this.report('warn', msg);
|
|
},
|
|
});
|
|
let currentTag = '';
|
|
this.saxStream.on('opentagstart', (tag) => {
|
|
charData.openTag(tag.name);
|
|
currentTag = tag.name;
|
|
});
|
|
this.saxStream.on('opentag', (tag) => {
|
|
if (!config.isKnownTag(tag.name)) {
|
|
this.logger('warn', 'unhandled tag', tag.name);
|
|
this.err(`unhandled tag: ${tag.name}`);
|
|
}
|
|
this.onOpenTag(tag);
|
|
});
|
|
this.saxStream.on('text', charData.append);
|
|
this.saxStream.on('cdata', charData.append);
|
|
this.saxStream.on('attribute', (attr) => {
|
|
this.onAttribute(currentTag, attr);
|
|
});
|
|
this.saxStream.on('closetag', (tag) => {
|
|
charData.closeTag();
|
|
this.onCloseTag(tag);
|
|
});
|
|
}
|
|
/** The first parse error, or null if there has been none */
|
|
get error() {
|
|
return this.errors[0] ?? null;
|
|
}
|
|
/** Called for every opening tag, after unknown tags have been reported */
|
|
onOpenTag(tag) { }
|
|
/** Count a finished entry and push it, unless it is past the entry cap */
|
|
emitEntry(item) {
|
|
this.entryCount++;
|
|
if (this.entryCount > this.maxEntries) {
|
|
// Report once rather than for every dropped entry
|
|
if (this.entryCount === this.maxEntries + 1) {
|
|
const msg = this.overflowMessage(this.maxEntries);
|
|
this.err(msg);
|
|
// A listener (such as parseSitemapIndex) handles the overflow
|
|
// itself; only log when nobody is listening
|
|
if (!this.emit('maxEntriesExceeded', this.maxEntries)) {
|
|
this.logger('error', msg);
|
|
}
|
|
}
|
|
}
|
|
else {
|
|
this.push(item);
|
|
}
|
|
}
|
|
_transform(data, encoding, callback) {
|
|
try {
|
|
const cb = () => callback(this.level === ErrorLevel.THROW ? this.error : null);
|
|
// correcting the type here can be done without making it a breaking change
|
|
// TODO fix this
|
|
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
// @ts-ignore
|
|
if (!this.saxStream.write(data, encoding)) {
|
|
this.saxStream.once('drain', cb);
|
|
}
|
|
else {
|
|
process.nextTick(cb);
|
|
}
|
|
}
|
|
catch (error) {
|
|
callback(error);
|
|
}
|
|
}
|
|
/**
|
|
* Tell sax the document has ended, so a truncated document (an unclosed
|
|
* element) fails the stream like any other malformed XML.
|
|
*/
|
|
_flush(callback) {
|
|
// Past the entry cap the rest of the document is ignored, and a caller
|
|
// such as parseSitemap may have stopped reading the source on purpose,
|
|
// so a missing end is not an error then.
|
|
if (this.entryCount > this.maxEntries) {
|
|
callback();
|
|
return;
|
|
}
|
|
try {
|
|
this.saxStream.end();
|
|
callback();
|
|
}
|
|
catch (error) {
|
|
callback(error);
|
|
}
|
|
}
|
|
/** Log a message and record it as a parse error, with the same text */
|
|
report(level, msg) {
|
|
this.logger(level, msg);
|
|
this.err(msg);
|
|
}
|
|
/** Record a parse error, keeping at most LIMITS.MAX_PARSER_ERRORS */
|
|
err(msg) {
|
|
this.errorCount++;
|
|
if (this.errors.length < LIMITS.MAX_PARSER_ERRORS) {
|
|
this.errors.push(new Error(msg));
|
|
}
|
|
}
|
|
}
|
|
/**
|
|
* Pipe `xml` through `parser` and resolve with every entry it emits. Rejects
|
|
* if either stream fails.
|
|
*
|
|
* Entries past the parser's cap are dropped, so once the cap is crossed this
|
|
* stops reading `xml` and destroys it instead of parsing the rest (BB-05).
|
|
* `onOverflow` then picks the outcome: return an Error to reject with it, or
|
|
* nothing to resolve with the entries emitted before the cap.
|
|
*/
|
|
export function collectEntries(xml, parser, onOverflow) {
|
|
return new Promise((resolve, reject) => {
|
|
const items = [];
|
|
let settled = false;
|
|
const settle = (error) => {
|
|
if (settled)
|
|
return;
|
|
settled = true;
|
|
if (error) {
|
|
reject(error);
|
|
}
|
|
else {
|
|
resolve(items);
|
|
}
|
|
};
|
|
// pipe() does not forward source errors, so watch the source too
|
|
xml.on('error', (error) => {
|
|
settle(error);
|
|
parser.destroy();
|
|
});
|
|
parser.once('maxEntriesExceeded', (max) => {
|
|
xml.unpipe(parser);
|
|
xml.destroy();
|
|
const error = onOverflow(max);
|
|
if (error) {
|
|
settle(error);
|
|
parser.destroy();
|
|
}
|
|
else {
|
|
// Ending the parser still delivers the entries it already emitted
|
|
parser.end();
|
|
}
|
|
});
|
|
parser
|
|
.on('data', (item) => {
|
|
if (!settled)
|
|
items.push(item);
|
|
})
|
|
.on('end', () => settle())
|
|
.on('error', (error) => settle(error));
|
|
xml.pipe(parser);
|
|
});
|
|
}
|