239 lines
8.9 KiB
JavaScript
Executable File
239 lines
8.9 KiB
JavaScript
Executable File
#!/usr/bin/env node
|
|
import { createReadStream, readFileSync } from 'node:fs';
|
|
import { Readable, pipeline } from 'node:stream';
|
|
import { URL } from 'node:url';
|
|
import { createGzip } from 'node:zlib';
|
|
import { parseArgs } from 'node:util';
|
|
import { xmlLint } from './lib/xmllint.js';
|
|
import { InvalidLimitError, XMLLintUnavailable } from './lib/errors.js';
|
|
import { ObjectStreamToJSON, XMLToSitemapItemStream, } from './lib/sitemap-parser.js';
|
|
import { findUp, lineSeparatedURLsToSitemapOptions, openSitemapFile, } from './lib/utils.js';
|
|
import { SitemapStream } from './lib/sitemap-stream.js';
|
|
import { SitemapAndIndexStream } from './lib/sitemap-index-stream.js';
|
|
import { ErrorLevel } from './lib/types.js';
|
|
import { validateLimit } from './lib/validation.js';
|
|
/**
|
|
* Locate the package manifest by walking up from this module's directory.
|
|
*
|
|
* Resolving it at a hardcoded relative depth has broken twice already, because
|
|
* the correct depth depends on the build's output layout (`dist/esm/cli.js`
|
|
* needed `../../`, `dist/cli.js` needs `../`). Walking up finds it whichever
|
|
* layout this file is running from, source or built.
|
|
*/
|
|
const findPackageJson = () => {
|
|
const packageJson = findUp('package.json', import.meta.dirname);
|
|
if (!packageJson) {
|
|
throw new Error('Could not locate package.json');
|
|
}
|
|
return packageJson;
|
|
};
|
|
const packageJson = JSON.parse(readFileSync(findPackageJson(), 'utf8'));
|
|
const parseCliArgs = () => {
|
|
try {
|
|
return parseArgs({
|
|
allowPositionals: true,
|
|
options: {
|
|
help: { type: 'boolean', short: 'h' },
|
|
version: { type: 'boolean' },
|
|
validate: { type: 'boolean' },
|
|
index: { type: 'boolean' },
|
|
'index-base-url': { type: 'string' },
|
|
limit: { type: 'string' },
|
|
parse: { type: 'boolean' },
|
|
'single-line-json': { type: 'boolean' },
|
|
prepend: { type: 'string' },
|
|
gzip: { type: 'boolean' },
|
|
},
|
|
});
|
|
}
|
|
catch (err) {
|
|
console.error(err instanceof Error ? err.message : String(err));
|
|
console.error('Run `sitemap --help` for a list of options.');
|
|
process.exit(2);
|
|
}
|
|
};
|
|
const { values: argv, positionals } = parseCliArgs();
|
|
const pickStreamOrArg = () => {
|
|
if (!positionals.length) {
|
|
return process.stdin;
|
|
}
|
|
else {
|
|
return createReadStream(positionals[0], { encoding: 'utf8' });
|
|
}
|
|
};
|
|
/**
|
|
* Pipe `streams` together and write the result to stdout. On failure, print
|
|
* the error message (not an unhandled-error stack trace) and exit with 1.
|
|
*
|
|
* With `quietEPIPE`, a reader that closes stdout early (such as `| head`) is
|
|
* not a failure. Leave it off when the pipeline writes files as it goes, since
|
|
* stopping early leaves them incomplete.
|
|
*/
|
|
function pipeToStdout(streams, { quietEPIPE }) {
|
|
pipeline([...streams, process.stdout], (err) => {
|
|
if (!err ||
|
|
(quietEPIPE && err.code === 'EPIPE')) {
|
|
return;
|
|
}
|
|
console.error(err.message);
|
|
process.exitCode = 1;
|
|
});
|
|
}
|
|
/**
|
|
* Yield every item of each source in turn, so later sources start only after
|
|
* earlier ones have ended
|
|
*/
|
|
async function* concatSources(...sources) {
|
|
for (const source of sources) {
|
|
yield* source();
|
|
}
|
|
}
|
|
function getStream() {
|
|
if (positionals.length) {
|
|
return createReadStream(positionals[0]);
|
|
}
|
|
else {
|
|
console.warn('Reading from stdin. If you are not piping anything in, this command is not doing anything');
|
|
return process.stdin;
|
|
}
|
|
}
|
|
if (argv.version) {
|
|
console.log(packageJson.version);
|
|
}
|
|
else if (argv.help) {
|
|
console.log(`
|
|
Turn a list of urls into a sitemap xml.
|
|
Options:
|
|
--help Print this text
|
|
--version Print the version
|
|
--validate Ensure the passed in file is conforms to the sitemap spec
|
|
--index Create an index and stream that out. Writes out sitemaps along the way.
|
|
--index-base-url Base url the sitemaps will be hosted eg. https://example.com/sitemaps/
|
|
--limit=45000 Set a custom limit to the items per sitemap
|
|
--parse Parse fed xml and spit out config
|
|
--prepend=sitemap.xml Prepend the streamed in sitemap configs to sitemap.xml
|
|
--gzip Compress output. With --index, also gzips each sitemap
|
|
and names it sitemap-N.xml.gz
|
|
--single-line-json When used with parse, it spits out each entry as json rather than the whole json.
|
|
|
|
# examples
|
|
|
|
Generate a sitemap index file as well as sitemaps
|
|
npx sitemap --gzip --index --index-base-url https://example.com/path/to/sitemaps/ < listofurls.txt > sitemap-index.xml.gz
|
|
|
|
Add to a sitemap
|
|
npx sitemap --prepend sitemap.xml < listofurls.json
|
|
|
|
Turn an existing sitemap into configuration understood by the sitemap library
|
|
npx sitemap --parse sitemap.xml
|
|
|
|
Use XMLLib to validate your sitemap (requires xmllib)
|
|
npx sitemap --validate sitemap.xml
|
|
`);
|
|
}
|
|
else if (argv.parse) {
|
|
const streams = [
|
|
getStream(),
|
|
new XMLToSitemapItemStream({ level: ErrorLevel.THROW }),
|
|
new ObjectStreamToJSON({ lineSeparated: !argv['single-line-json'] }),
|
|
];
|
|
if (argv.gzip) {
|
|
streams.push(createGzip());
|
|
}
|
|
pipeToStdout(streams, { quietEPIPE: true });
|
|
}
|
|
else if (argv.validate) {
|
|
xmlLint(getStream())
|
|
.then(() => console.log('valid'))
|
|
.catch(([error, stderr]) => {
|
|
if (error instanceof XMLLintUnavailable || stderr === undefined) {
|
|
// No xmllint output to show (xmllint missing, or the schema could
|
|
// not be found), so report the error itself. Exit 2: validation
|
|
// could not run, as distinct from a file that failed it.
|
|
console.error(error?.message);
|
|
process.exitCode = 2;
|
|
return;
|
|
}
|
|
else {
|
|
console.log(stderr);
|
|
// Exit 1 so scripts and CI can detect an invalid sitemap
|
|
process.exitCode = 1;
|
|
}
|
|
});
|
|
}
|
|
else if (argv.index) {
|
|
// Decimal digits only: Number() alone would also accept forms such as
|
|
// "1e3", "0x10" or " 250 ". Anything else becomes NaN, which fails below.
|
|
const limit = argv.limit === undefined
|
|
? undefined
|
|
: /^\d+$/.test(argv.limit)
|
|
? Number(argv.limit)
|
|
: NaN;
|
|
try {
|
|
if (limit !== undefined) {
|
|
validateLimit(limit);
|
|
}
|
|
}
|
|
catch {
|
|
// Report the value as typed, rather than the NaN it parsed to
|
|
console.error(`--limit: ${new InvalidLimitError(argv.limit).message}`);
|
|
process.exit(2);
|
|
}
|
|
const baseURL = argv['index-base-url'];
|
|
if (!baseURL) {
|
|
throw new Error("You must specify where the sitemaps will be hosted. use --index-base-url 'https://example.com/path'");
|
|
}
|
|
const sms = new SitemapAndIndexStream({
|
|
limit,
|
|
getSitemapStream: (i) =>
|
|
// Gzipped files are named .xml.gz, as simpleSitemapAndIndex names them
|
|
openSitemapFile(i, new SitemapStream(), {
|
|
destinationDir: '.',
|
|
gzip: Boolean(argv.gzip),
|
|
toURL: (fileName) => new URL(fileName, baseURL).toString(),
|
|
urlParamName: '--index-base-url',
|
|
onError: (err) => sms.destroy(err),
|
|
}),
|
|
});
|
|
const streams = [
|
|
lineSeparatedURLsToSitemapOptions(pickStreamOrArg()),
|
|
sms,
|
|
];
|
|
if (argv.gzip) {
|
|
streams.push(createGzip());
|
|
}
|
|
// sitemap files are written as the index streams out, so a reader that
|
|
// stops early leaves them incomplete: report it
|
|
pipeToStdout(streams, { quietEPIPE: false });
|
|
}
|
|
else {
|
|
const sources = [];
|
|
const prepend = argv.prepend;
|
|
if (prepend) {
|
|
// Read the existing sitemap to the end before the new URLs, so its
|
|
// entries come first and one source ending can't end the sitemap early
|
|
sources.push(() => {
|
|
const parser = new XMLToSitemapItemStream();
|
|
// Past the cap the parser drops entries and only logs it, which would
|
|
// write a truncated sitemap and exit 0. Fail instead: the output can't
|
|
// hold more than a sitemap may.
|
|
parser.once('maxEntriesExceeded', (max) => {
|
|
parser.destroy(new Error(`--prepend file ${prepend} has more than ${max} URLs, the most a sitemap may hold`));
|
|
});
|
|
// pipeline() destroys both streams if either fails, and fails the
|
|
// parser with the error, which concatSources then rethrows. The
|
|
// callback has nothing to add.
|
|
return pipeline(createReadStream(prepend), parser, () => { });
|
|
});
|
|
}
|
|
sources.push(() => lineSeparatedURLsToSitemapOptions(pickStreamOrArg()));
|
|
const streams = [
|
|
Readable.from(concatSources(...sources)),
|
|
new SitemapStream(),
|
|
];
|
|
if (argv.gzip) {
|
|
streams.push(createGzip());
|
|
}
|
|
pipeToStdout(streams, { quietEPIPE: true });
|
|
}
|