tor-browser

gen_cache.ts (8481B)

---

import * as fs from 'fs';
import * as path from 'path';
import * as process from 'process';

import { Cacheable, dataCache, setIsBuildingDataCache } from '../framework/data_cache.js';
import { crc32, toHexString } from '../util/crc32.js';
import { parseImports } from '../util/parse_imports.js';

function usage(rc: number): void {
 console.error(`Usage: tools/gen_cache [options] [SUITE_DIRS...]

For each suite in SUITE_DIRS, pre-compute data that is expensive to generate
at runtime and store it under 'src/resources/cache'. If the data file is found
then the DataCache will load this instead of building the expensive data at CTS
runtime.
Note: Due to differences in gzip compression, different versions of node can
produce radically different binary cache files. gen_cache uses the hashes of the
source files to determine whether a cache file is 'up to date'. This is faster
and does not depend on the compressed output.

Options:
 --help          Print this message and exit.
 --list          Print the list of output files without writing them.
 --force         Rebuild cache even if they're up to date
 --validate      Check the cache is up to date
 --verbose       Print each action taken.
`);
 process.exit(rc);
}

// Where the cache is generated
const outDir = 'src/resources/cache';

let forceRebuild = false;
let mode: 'emit' | 'list' | 'validate' = 'emit';
let verbose = false;

const nonFlagsArgs: string[] = [];

for (const arg of process.argv) {
 if (arg.startsWith('-')) {
   switch (arg) {
     case '--list': {
       mode = 'list';
       break;
     }
     case '--help': {
       usage(0);
       break;
     }
     case '--force': {
       forceRebuild = true;
       break;
     }
     case '--verbose': {
       verbose = true;
       break;
     }
     case '--validate': {
       mode = 'validate';
       break;
     }
     default: {
       console.log('unrecognized flag: ', arg);
       usage(1);
     }
   }
 } else {
   nonFlagsArgs.push(arg);
 }
}

if (nonFlagsArgs.length < 3) {
 usage(0);
}

dataCache.setStore({
 load: (path: string) => {
   return new Promise<Uint8Array>((resolve, reject) => {
     fs.readFile(`data/${path}`, (err, data) => {
       if (err !== null) {
         reject(err.message);
       } else {
         resolve(data);
       }
     });
   });
 },
});
setIsBuildingDataCache();

const cacheFileSuffix = __filename.endsWith('.ts') ? '.cache.ts' : '.cache.js';

/**
* @returns a list of all the files under 'dir' that has the given extension
* @param dir the directory to search
* @param ext the extension of the files to find
*/
function glob(dir: string, ext: string) {
 const files: string[] = [];
 for (const file of fs.readdirSync(dir)) {
   const path = `${dir}/${file}`;
   if (fs.statSync(path).isDirectory()) {
     for (const child of glob(path, ext)) {
       files.push(`${file}/${child}`);
     }
   }

   if (path.endsWith(ext) && fs.statSync(path).isFile()) {
     files.push(file);
   }
 }
 return files;
}

/**
* Exception type thrown by SourceHasher.hashFile() when a file annotated with
* MUST_NOT_BE_IMPORTED_BY_DATA_CACHE is transitively imported by a .cache.ts file.
*/
class InvalidImportException {
 constructor(path: string) {
   this.stack = [path];
 }
 toString(): string {
   return `invalid transitive import for cache:\n  ${this.stack.join('\n  ')}`;
 }
 readonly stack: string[];
}
/**
* SourceHasher is a utility for producing a hash of a source .ts file and its imported source files.
*/
class SourceHasher {
 /**
  * @param path the source file path
  * @returns a hash of the source file and all of its imported dependencies.
  */
 public hashOf(path: string) {
   this.u32Array[0] = this.hashFile(path);
   return this.u32Array[0].toString(16);
 }

 hashFile(path: string): number {
   if (!fs.existsSync(path) && path.endsWith('.js')) {
     path = path.substring(0, path.length - 2) + 'ts';
   }

   const cached = this.hashes.get(path);
   if (cached !== undefined) {
     return cached;
   }

   this.hashes.set(path, 0); // Store a zero hash to handle cyclic imports

   const content = fs.readFileSync(path, { encoding: 'utf-8' });
   const normalized = content.replace('\r\n', '\n');
   let hash = crc32(normalized);
   for (const importPath of parseImports(path, normalized)) {
     try {
       const importHash = this.hashFile(importPath);
       hash = this.hashCombine(hash, importHash);
     } catch (ex) {
       if (ex instanceof InvalidImportException) {
         ex.stack.push(path);
         throw ex;
       }
     }
   }

   if (content.includes('MUST_NOT_BE_IMPORTED_BY_DATA_CACHE')) {
     throw new InvalidImportException(path);
   }

   this.hashes.set(path, hash);
   return hash;
 }

 /** Simple non-cryptographic hash combiner */
 hashCombine(a: number, b: number): number {
   return crc32(`${toHexString(a)} ${toHexString(b)}`);
 }

 private hashes = new Map<string, number>();
 private u32Array = new Uint32Array(1);
}

void (async () => {
 const suiteDirs = nonFlagsArgs.slice(2); // skip <exe> <js>
 for (const suiteDir of suiteDirs) {
   await build(suiteDir);
 }
})();

async function build(suiteDir: string) {
 if (!fs.existsSync(suiteDir)) {
   console.error(`Could not find ${suiteDir}`);
   process.exit(1);
 }

 // Load  hashes.json
 const fileHashJsonPath = `${outDir}/hashes.json`;
 let fileHashes: Record<string, string> = {};
 if (fs.existsSync(fileHashJsonPath)) {
   const json = fs.readFileSync(fileHashJsonPath, { encoding: 'utf8' });
   fileHashes = JSON.parse(json);
 }

 // Crawl files and convert paths to be POSIX-style, relative to suiteDir.
 const filesToEnumerate = glob(suiteDir, cacheFileSuffix)
   .map(p => `${suiteDir}/${p}`)
   .sort();

 const fileHasher = new SourceHasher();
 const cacheablePathToTS = new Map<string, string>();
 const errors: Array<string> = [];

 for (const file of filesToEnumerate) {
   const pathWithoutExtension = file.substring(0, file.length - 3);
   const mod = await import(`../../../${pathWithoutExtension}.js`);
   if (mod.d?.serialize !== undefined) {
     const cacheable = mod.d as Cacheable<unknown>;

     {
       // Check for collisions
       const existing = cacheablePathToTS.get(cacheable.path);
       if (existing !== undefined) {
         errors.push(
           `'${cacheable.path}' is emitted by both:
   '${existing}'
and
   '${file}'`
         );
       }
       cacheablePathToTS.set(cacheable.path, file);
     }

     const outPath = `${outDir}/${cacheable.path}`;
     const fileHash = fileHasher.hashOf(file);

     switch (mode) {
       case 'emit': {
         if (!forceRebuild && fileHashes[cacheable.path] === fileHash) {
           if (verbose) {
             console.log(`'${outPath}' is up to date`);
           }
           continue;
         }
         console.log(`building '${outPath}'`);
         const data = await cacheable.build();
         const serialized = cacheable.serialize(data);
         fs.mkdirSync(path.dirname(outPath), { recursive: true });
         fs.writeFileSync(outPath, serialized, 'binary');
         fileHashes[cacheable.path] = fileHash;
         break;
       }
       case 'list': {
         console.log(outPath);
         break;
       }
       case 'validate': {
         if (fileHashes[cacheable.path] !== fileHash) {
           errors.push(
             `'${outPath}' needs rebuilding. Generate with 'npx grunt run:generate-cache'`
           );
         } else if (verbose) {
           console.log(`'${outPath}' is up to date`);
         }
       }
     }
   }
 }

 // Check that there aren't stale files in the cache directory
 for (const file of glob(outDir, '.bin')) {
   if (cacheablePathToTS.get(file) === undefined) {
     switch (mode) {
       case 'emit':
         fs.rmSync(file);
         break;
       case 'validate':
         errors.push(
           `cache file '${outDir}/${file}' is no longer generated. Remove with 'npx grunt run:generate-cache'`
         );
         break;
     }
   }
 }

 // Update hashes.json
 if (mode === 'emit') {
   const json = JSON.stringify(fileHashes, undefined, '  ');
   fs.writeFileSync(fileHashJsonPath, json, { encoding: 'utf8' });
 }

 if (errors.length > 0) {
   for (const error of errors) {
     console.error(error);
   }
   process.exit(1);
 }
}