base.js (12998B)
1 // Copyright 2013 the V8 project authors. All rights reserved. 2 // Redistribution and use in source and binary forms, with or without 3 // modification, are permitted provided that the following conditions are 4 // met: 5 // 6 // * Redistributions of source code must retain the above copyright 7 // notice, this list of conditions and the following disclaimer. 8 // * Redistributions in binary form must reproduce the above 9 // copyright notice, this list of conditions and the following 10 // disclaimer in the documentation and/or other materials provided 11 // with the distribution. 12 // * Neither the name of Google Inc. nor the names of its 13 // contributors may be used to endorse or promote products derived 14 // from this software without specific prior written permission. 15 // 16 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 28 29 // Performance.now is used in latency benchmarks, the fallback is Date.now. 30 var performance = performance || {}; 31 performance.now = (function() { 32 return performance.now || 33 performance.mozNow || 34 performance.msNow || 35 performance.oNow || 36 performance.webkitNow || 37 Date.now; 38 })(); 39 40 // Simple framework for running the benchmark suites and 41 // computing a score based on the timing measurements. 42 43 44 // A benchmark has a name (string) and a function that will be run to 45 // do the performance measurement. The optional setup and tearDown 46 // arguments are functions that will be invoked before and after 47 // running the benchmark, but the running time of these functions will 48 // not be accounted for in the benchmark score. 49 function Benchmark(name, doWarmup, doDeterministic, deterministicIterations, 50 run, setup, tearDown, rmsResult, minIterations) { 51 this.name = name; 52 this.doWarmup = doWarmup; 53 this.doDeterministic = doDeterministic; 54 this.deterministicIterations = deterministicIterations; 55 this.run = run; 56 this.Setup = setup ? setup : function() { }; 57 this.TearDown = tearDown ? tearDown : function() { }; 58 this.rmsResult = rmsResult ? rmsResult : null; 59 this.minIterations = minIterations ? minIterations : 32; 60 } 61 62 63 // Benchmark results hold the benchmark and the measured time used to 64 // run the benchmark. The benchmark score is computed later once a 65 // full benchmark suite has run to completion. If latency is set to 0 66 // then there is no latency score for this benchmark. 67 function BenchmarkResult(benchmark, time, latency) { 68 this.benchmark = benchmark; 69 this.time = time; 70 this.latency = latency; 71 } 72 73 74 // Automatically convert results to numbers. Used by the geometric 75 // mean computation. 76 BenchmarkResult.prototype.valueOf = function() { 77 return this.time; 78 } 79 80 81 // Suites of benchmarks consist of a name and the set of benchmarks in 82 // addition to the reference timing that the final score will be based 83 // on. This way, all scores are relative to a reference run and higher 84 // scores implies better performance. 85 function BenchmarkSuite(name, reference, benchmarks) { 86 this.name = name; 87 this.reference = reference; 88 this.benchmarks = benchmarks; 89 BenchmarkSuite.suites.push(this); 90 } 91 92 93 // Keep track of all declared benchmark suites. 94 BenchmarkSuite.suites = []; 95 96 // Scores are not comparable across versions. Bump the version if 97 // you're making changes that will affect that scores, e.g. if you add 98 // a new benchmark or change an existing one. 99 BenchmarkSuite.version = '9'; 100 101 102 // Defines global benchsuite running mode that overrides benchmark suite 103 // behavior. Intended to be set by the benchmark driver. Undefined 104 // values here allow a benchmark to define behaviour itself. 105 BenchmarkSuite.config = { 106 doWarmup: undefined, 107 doDeterministic: undefined 108 }; 109 110 111 // Override the alert function to throw an exception instead. 112 alert = function(s) { 113 throw "Alert called with argument: " + s; 114 }; 115 116 117 // To make the benchmark results predictable, we replace Math.random 118 // with a 100% deterministic alternative. 119 BenchmarkSuite.ResetRNG = function() { 120 Math.random = (function() { 121 var seed = 49734321; 122 return function() { 123 // Robert Jenkins' 32 bit integer hash function. 124 seed = ((seed + 0x7ed55d16) + (seed << 12)) & 0xffffffff; 125 seed = ((seed ^ 0xc761c23c) ^ (seed >>> 19)) & 0xffffffff; 126 seed = ((seed + 0x165667b1) + (seed << 5)) & 0xffffffff; 127 seed = ((seed + 0xd3a2646c) ^ (seed << 9)) & 0xffffffff; 128 seed = ((seed + 0xfd7046c5) + (seed << 3)) & 0xffffffff; 129 seed = ((seed ^ 0xb55a4f09) ^ (seed >>> 16)) & 0xffffffff; 130 return (seed & 0xfffffff) / 0x10000000; 131 }; 132 })(); 133 } 134 135 136 // Runs all registered benchmark suites and optionally yields between 137 // each individual benchmark to avoid running for too long in the 138 // context of browsers. Once done, the final score is reported to the 139 // runner. 140 BenchmarkSuite.RunSuites = function(runner, skipBenchmarks) { 141 skipBenchmarks = typeof skipBenchmarks === 'undefined' ? [] : skipBenchmarks; 142 var continuation = null; 143 var suites = BenchmarkSuite.suites; 144 var length = suites.length; 145 BenchmarkSuite.scores = []; 146 var index = 0; 147 function RunStep() { 148 while (continuation || index < length) { 149 if (continuation) { 150 continuation = continuation(); 151 } else { 152 var suite = suites[index++]; 153 if (runner.NotifyStart) runner.NotifyStart(suite.name); 154 if (skipBenchmarks.indexOf(suite.name) > -1) { 155 suite.NotifySkipped(runner); 156 } else { 157 continuation = suite.RunStep(runner); 158 } 159 } 160 if (continuation && typeof window != 'undefined' && window.setTimeout) { 161 window.setTimeout(RunStep, 25); 162 return; 163 } 164 } 165 166 // show final result 167 if (runner.NotifyScore) { 168 var score = BenchmarkSuite.GeometricMean(BenchmarkSuite.scores); 169 var formatted = BenchmarkSuite.FormatScore(100 * score); 170 runner.NotifyScore(formatted); 171 } 172 } 173 RunStep(); 174 } 175 176 177 // Counts the total number of registered benchmarks. Useful for 178 // showing progress as a percentage. 179 BenchmarkSuite.CountBenchmarks = function() { 180 var result = 0; 181 var suites = BenchmarkSuite.suites; 182 for (var i = 0; i < suites.length; i++) { 183 result += suites[i].benchmarks.length; 184 } 185 return result; 186 } 187 188 189 // Computes the geometric mean of a set of numbers. 190 BenchmarkSuite.GeometricMean = function(numbers) { 191 var log = 0; 192 for (var i = 0; i < numbers.length; i++) { 193 log += Math.log(numbers[i]); 194 } 195 return Math.pow(Math.E, log / numbers.length); 196 } 197 198 199 // Computes the geometric mean of a set of throughput time measurements. 200 BenchmarkSuite.GeometricMeanTime = function(measurements) { 201 var log = 0; 202 for (var i = 0; i < measurements.length; i++) { 203 log += Math.log(measurements[i].time); 204 } 205 return Math.pow(Math.E, log / measurements.length); 206 } 207 208 209 // Computes the geometric mean of a set of rms measurements. 210 BenchmarkSuite.GeometricMeanLatency = function(measurements) { 211 var log = 0; 212 var hasLatencyResult = false; 213 for (var i = 0; i < measurements.length; i++) { 214 if (measurements[i].latency != 0) { 215 log += Math.log(measurements[i].latency); 216 hasLatencyResult = true; 217 } 218 } 219 if (hasLatencyResult) { 220 return Math.pow(Math.E, log / measurements.length); 221 } else { 222 return 0; 223 } 224 } 225 226 227 // Converts a score value to a string with at least three significant 228 // digits. 229 BenchmarkSuite.FormatScore = function(value) { 230 if (value > 100) { 231 return value.toFixed(0); 232 } else { 233 return value.toPrecision(3); 234 } 235 } 236 237 // Notifies the runner that we're done running a single benchmark in 238 // the benchmark suite. This can be useful to report progress. 239 BenchmarkSuite.prototype.NotifyStep = function(result) { 240 this.results.push(result); 241 if (this.runner.NotifyStep) this.runner.NotifyStep(result.benchmark.name); 242 } 243 244 245 // Notifies the runner that we're done with running a suite and that 246 // we have a result which can be reported to the user if needed. 247 BenchmarkSuite.prototype.NotifyResult = function() { 248 var mean = BenchmarkSuite.GeometricMeanTime(this.results); 249 var score = this.reference[0] / mean; 250 BenchmarkSuite.scores.push(score); 251 if (this.runner.NotifyResult) { 252 var formatted = BenchmarkSuite.FormatScore(100 * score); 253 this.runner.NotifyResult(this.name, formatted); 254 } 255 if (this.reference.length == 2) { 256 var meanLatency = BenchmarkSuite.GeometricMeanLatency(this.results); 257 if (meanLatency != 0) { 258 var scoreLatency = this.reference[1] / meanLatency; 259 BenchmarkSuite.scores.push(scoreLatency); 260 if (this.runner.NotifyResult) { 261 var formattedLatency = BenchmarkSuite.FormatScore(100 * scoreLatency) 262 this.runner.NotifyResult(this.name + "Latency", formattedLatency); 263 } 264 } 265 } 266 } 267 268 269 BenchmarkSuite.prototype.NotifySkipped = function(runner) { 270 BenchmarkSuite.scores.push(1); // push default reference score. 271 if (runner.NotifyResult) { 272 runner.NotifyResult(this.name, "Skipped"); 273 } 274 } 275 276 277 // Notifies the runner that running a benchmark resulted in an error. 278 BenchmarkSuite.prototype.NotifyError = function(error) { 279 if (this.runner.NotifyError) { 280 this.runner.NotifyError(this.name, error); 281 } 282 if (this.runner.NotifyStep) { 283 this.runner.NotifyStep(this.name); 284 } 285 } 286 287 288 // Runs a single benchmark for at least a second and computes the 289 // average time it takes to run a single iteration. 290 BenchmarkSuite.prototype.RunSingleBenchmark = function(benchmark, data) { 291 var config = BenchmarkSuite.config; 292 var doWarmup = config.doWarmup !== undefined 293 ? config.doWarmup 294 : benchmark.doWarmup; 295 var doDeterministic = config.doDeterministic !== undefined 296 ? config.doDeterministic 297 : benchmark.doDeterministic; 298 299 function Measure(data) { 300 var elapsed = 0; 301 var start = new Date(); 302 303 // Run either for 1 second or for the number of iterations specified 304 // by minIterations, depending on the config flag doDeterministic. 305 for (var i = 0; (doDeterministic ? 306 i<benchmark.deterministicIterations : elapsed < 1000); i++) { 307 benchmark.run(); 308 elapsed = new Date() - start; 309 } 310 if (data != null) { 311 data.runs += i; 312 data.elapsed += elapsed; 313 } 314 } 315 316 // Sets up data in order to skip or not the warmup phase. 317 if (!doWarmup && data == null) { 318 data = { runs: 0, elapsed: 0 }; 319 } 320 321 if (data == null) { 322 Measure(null); 323 return { runs: 0, elapsed: 0 }; 324 } else { 325 Measure(data); 326 // If we've run too few iterations, we continue for another second. 327 if (data.runs < benchmark.minIterations) return data; 328 var usec = (data.elapsed * 1000) / data.runs; 329 var rms = (benchmark.rmsResult != null) ? benchmark.rmsResult() : 0; 330 this.NotifyStep(new BenchmarkResult(benchmark, usec, rms)); 331 return null; 332 } 333 } 334 335 336 // This function starts running a suite, but stops between each 337 // individual benchmark in the suite and returns a continuation 338 // function which can be invoked to run the next benchmark. Once the 339 // last benchmark has been executed, null is returned. 340 BenchmarkSuite.prototype.RunStep = function(runner) { 341 BenchmarkSuite.ResetRNG(); 342 this.results = []; 343 this.runner = runner; 344 var length = this.benchmarks.length; 345 var index = 0; 346 var suite = this; 347 var data; 348 349 // Run the setup, the actual benchmark, and the tear down in three 350 // separate steps to allow the framework to yield between any of the 351 // steps. 352 353 function RunNextSetup() { 354 if (index < length) { 355 try { 356 suite.benchmarks[index].Setup(); 357 } catch (e) { 358 suite.NotifyError(e); 359 return null; 360 } 361 return RunNextBenchmark; 362 } 363 suite.NotifyResult(); 364 return null; 365 } 366 367 function RunNextBenchmark() { 368 try { 369 data = suite.RunSingleBenchmark(suite.benchmarks[index], data); 370 } catch (e) { 371 suite.NotifyError(e); 372 return null; 373 } 374 // If data is null, we're done with this benchmark. 375 return (data == null) ? RunNextTearDown : RunNextBenchmark(); 376 } 377 378 function RunNextTearDown() { 379 try { 380 suite.benchmarks[index++].TearDown(); 381 } catch (e) { 382 suite.NotifyError(e); 383 return null; 384 } 385 return RunNextSetup; 386 } 387 388 // Start out running the setup. 389 return RunNextSetup(); 390 }