tor-browser

The Tor Browser
git clone https://git.dasho.dev/tor-browser.git
Log | Files | Refs | README | LICENSE

doctest.ts (9856B)

      1 #! /usr/bin/env -S node --test-reporter spec
      2 
      3 /**
      4 * @license
      5 * Copyright 2023 Google Inc.
      6 * SPDX-License-Identifier: Apache-2.0
      7 */
      8 
      9 /**
     10 * `@puppeteer/doctest` tests `@example` code within a JavaScript file.
     11 *
     12 * There are a few reasonable assumptions for this tool to work:
     13 *
     14 * 1. Examples are written in block comments, not line comments.
     15 * 2. Examples do not use packages that are not available to the file it exists
     16 *    in. (Note the package will always be available).
     17 * 3. Examples are strictly written between code fences (\`\`\`) on separate
     18 *    lines. For example, \`\`\`console.log(1)\`\`\` is not allowed.
     19 * 4. Code is written using ES modules.
     20 *
     21 * By default, code blocks are interpreted as JavaScript. Use \`\`\`ts to change
     22 * the language. In general, the format is "\`\`\`[language] [ignore] [fail]".
     23 *
     24 * If there are several code blocks within an example, they are concatenated.
     25 */
     26 import 'source-map-support/register.js';
     27 
     28 import assert from 'node:assert';
     29 import {createHash} from 'node:crypto';
     30 import {mkdtemp, readFile, rm, writeFile} from 'node:fs/promises';
     31 import {basename, dirname, join, relative, resolve} from 'node:path';
     32 import {test} from 'node:test';
     33 import {pathToFileURL} from 'node:url';
     34 
     35 import {transform, type Output} from '@swc/core';
     36 import {parse as parseJs} from 'acorn';
     37 import {parse, type Tag} from 'doctrine';
     38 import {Glob} from 'glob';
     39 import {packageDirectory} from 'pkg-dir';
     40 import {
     41  SourceMapConsumer,
     42  SourceMapGenerator,
     43  type RawSourceMap,
     44 } from 'source-map';
     45 import yargs from 'yargs';
     46 import {hideBin} from 'yargs/helpers';
     47 
     48 // This is 1-indexed.
     49 interface Position {
     50  line: number;
     51  column: number;
     52 }
     53 
     54 interface Comment {
     55  file: string;
     56  text: string;
     57  position: Position;
     58 }
     59 
     60 interface ExtractedSourceLocation {
     61  // File path to the original source code.
     62  origin: string;
     63  // Mappings from the extracted code to the original code.
     64  positions: Array<{
     65    // The 1-indexed line number for the extracted code.
     66    extracted: number;
     67    // The position in the original code.
     68    original: Position;
     69  }>;
     70 }
     71 
     72 interface ExampleCode extends ExtractedSourceLocation {
     73  language: Language;
     74  code: string;
     75  fail: boolean;
     76 }
     77 
     78 const enum Language {
     79  JavaScript,
     80  TypeScript,
     81 }
     82 
     83 const CODE_FENCE = '```';
     84 const BLOCK_COMMENT_START = ' * ';
     85 
     86 const {files = []} = await yargs(hideBin(process.argv))
     87  .scriptName('@puppeteer/doctest')
     88  .command('* <files..>', `JSDoc @example code tester.`)
     89  .positional('files', {
     90    describe: 'Files to test',
     91    type: 'string',
     92  })
     93  .array('files')
     94  .version(false)
     95  .help()
     96  .parse();
     97 
     98 for await (const file of new Glob(files, {})) {
     99  void test(file, async context => {
    100    const testDirectory = await createTestDirectory(file);
    101    context.after(async () => {
    102      if (!process.env['KEEP_TESTS']) {
    103        await rm(testDirectory, {force: true, recursive: true});
    104      }
    105    });
    106    const tests = [];
    107    for (const example of await extractJSDocComments(file).then(
    108      extractExampleCode,
    109    )) {
    110      tests.push(
    111        context.test(
    112          `${file}:${example.positions[0]!.original.line}:${
    113            example.positions[0]!.original.column
    114          }`,
    115          async () => {
    116            await run(testDirectory, example);
    117          },
    118        ),
    119      );
    120    }
    121    await Promise.all(tests);
    122  });
    123 }
    124 
    125 async function createTestDirectory(file: string) {
    126  const dir = await packageDirectory({cwd: dirname(file)});
    127  if (!dir) {
    128    throw new Error(`Could not find package root for ${file}.`);
    129  }
    130 
    131  return await mkdtemp(join(dir, 'doctest-'));
    132 }
    133 
    134 async function run(tempdir: string, example: Readonly<ExampleCode>) {
    135  const path = getTestPath(tempdir, example.code);
    136  await compile(example.language, example.code, path, example);
    137  try {
    138    await import(pathToFileURL(path).toString());
    139    if (example.fail) {
    140      throw new Error(`Expected failure.`);
    141    }
    142  } catch (error) {
    143    if (!example.fail) {
    144      throw error;
    145    }
    146  }
    147 }
    148 
    149 function getTestPath(dir: string, code: string) {
    150  return join(
    151    dir,
    152    `doctest-${createHash('md5').update(code).digest('hex')}.js`,
    153  );
    154 }
    155 
    156 async function compile(
    157  language: Language,
    158  sourceCode: string,
    159  filePath: string,
    160  location: ExtractedSourceLocation,
    161 ) {
    162  const output = await compileCode(language, sourceCode);
    163  const map = await getExtractSourceMap(output.map, filePath, location);
    164  await writeFile(filePath, inlineSourceMap(output.code, map));
    165 }
    166 
    167 function inlineSourceMap(code: string, sourceMap: RawSourceMap) {
    168  return `${code}\n//# sourceMappingURL=data:application/json;base64,${Buffer.from(
    169    JSON.stringify(sourceMap),
    170  ).toString('base64')}`;
    171 }
    172 
    173 async function getExtractSourceMap(
    174  map: string,
    175  generatedFile: string,
    176  location: ExtractedSourceLocation,
    177 ) {
    178  const sourceMap = JSON.parse(map) as RawSourceMap;
    179  sourceMap.file = basename(generatedFile);
    180  sourceMap.sourceRoot = '';
    181  sourceMap.sources = [
    182    relative(dirname(generatedFile), resolve(location.origin)),
    183  ];
    184  const consumer = await new SourceMapConsumer(sourceMap);
    185  const generator = new SourceMapGenerator({
    186    file: consumer.file,
    187    sourceRoot: consumer.sourceRoot,
    188  });
    189  // We want descending order of the `generated` property.
    190  const positions = [...location.positions].reverse();
    191  consumer.eachMapping(mapping => {
    192    // Note `mapping.originalLine` is the line number with respect to the
    193    // extracted, raw code.
    194    const {extracted, original} = positions.find(({extracted}) => {
    195      return mapping.originalLine >= extracted;
    196    })!;
    197 
    198    // `original.line` will account for `extracted`, so we need to subtract
    199    // `extracted` to avoid duplicity. We also subtract 1 because `extracted` is
    200    // 1-indexed.
    201    mapping.originalLine -= extracted - 1;
    202 
    203    generator.addMapping({
    204      ...mapping,
    205      original: {
    206        line: mapping.originalLine + original.line - 1,
    207        column: mapping.originalColumn + original.column - 1,
    208      },
    209      generated: {
    210        line: mapping.generatedLine,
    211        column: mapping.generatedColumn,
    212      },
    213    });
    214  });
    215  return generator.toJSON();
    216 }
    217 
    218 const LANGUAGE_TO_SYNTAX = {
    219  [Language.TypeScript]: 'typescript',
    220  [Language.JavaScript]: 'ecmascript',
    221 } as const;
    222 
    223 async function compileCode(language: Language, code: string) {
    224  return (await transform(code, {
    225    sourceMaps: true,
    226    inlineSourcesContent: false,
    227    jsc: {
    228      parser: {
    229        syntax: LANGUAGE_TO_SYNTAX[language],
    230      },
    231      target: 'es2022',
    232    },
    233  })) as Required<Output>;
    234 }
    235 
    236 const enum Option {
    237  Ignore = 'ignore',
    238  Fail = 'fail',
    239 }
    240 
    241 function* extractExampleCode(
    242  comments: Iterable<Readonly<Comment>>,
    243 ): Iterable<Readonly<ExampleCode>> {
    244  interface Context {
    245    language: Language;
    246    fail: boolean;
    247    start: number;
    248  }
    249  for (const {file, text, position: loc} of comments) {
    250    const {tags} = parse(text, {
    251      unwrap: true,
    252      tags: ['example'],
    253      lineNumbers: true,
    254      preserveWhitespace: true,
    255    });
    256    for (const {description, lineNumber} of tags as Array<
    257      Tag & {lineNumber: number}
    258    >) {
    259      if (!description) {
    260        continue;
    261      }
    262      const lines = description.split('\n');
    263      const blocks: ExampleCode[] = [];
    264      let context: Context | undefined;
    265      for (let i = 0; i < lines.length; i++) {
    266        const line = lines[i]!;
    267        const borderIndex = line.indexOf(CODE_FENCE);
    268        if (borderIndex === -1) {
    269          continue;
    270        }
    271        if (context) {
    272          blocks.push({
    273            language: context.language,
    274            code: lines.slice(context.start, i).join('\n'),
    275            origin: file,
    276            positions: [
    277              {
    278                extracted: 1,
    279                original: {
    280                  line: loc.line + lineNumber + context.start,
    281                  column:
    282                    loc.column + borderIndex + BLOCK_COMMENT_START.length + 1,
    283                },
    284              },
    285            ],
    286            fail: context.fail,
    287          });
    288          context = undefined;
    289          continue;
    290        }
    291        const [tag, ...options] = line
    292          .slice(borderIndex + CODE_FENCE.length)
    293          .split(' ');
    294        if (options.includes(Option.Ignore)) {
    295          // Ignore the code sample.
    296          continue;
    297        }
    298        const fail = options.includes(Option.Fail);
    299        // Code starts on the next line.
    300        const start = i + 1;
    301        if (!tag || tag.match(/js|javascript/)) {
    302          context = {language: Language.JavaScript, fail, start};
    303        } else if (tag.match(/ts|typescript/)) {
    304          context = {language: Language.TypeScript, fail, start};
    305        }
    306      }
    307      // Merging the blocks into a single block.
    308      yield blocks.reduce(
    309        (context, {language, code, positions: [position], fail}, index) => {
    310          assert(position);
    311          return {
    312            origin: file,
    313            language: language || context.language,
    314            code: `${context.code}\n${code}`,
    315            positions: [
    316              ...context.positions,
    317              {
    318                ...position,
    319                extracted:
    320                  context.code.split('\n').length +
    321                  context.positions.at(-1)!.extracted -
    322                  // We subtract this because of the accumulated '\n'.
    323                  (index - 1),
    324              },
    325            ],
    326            fail: fail || context.fail,
    327          };
    328        },
    329      );
    330    }
    331  }
    332 }
    333 
    334 async function extractJSDocComments(file: string) {
    335  const contents = await readFile(file, 'utf8');
    336  const comments: Comment[] = [];
    337  parseJs(contents, {
    338    ecmaVersion: 'latest',
    339    sourceType: 'module',
    340    locations: true,
    341    sourceFile: file,
    342    onComment(isBlock, text, _, __, loc) {
    343      if (isBlock) {
    344        comments.push({file, text, position: loc!});
    345      }
    346    },
    347  });
    348  return comments;
    349 }