tor-browser

har-builder.js (19689B)

---

/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */

"use strict";

const appInfo = Services.appinfo;
const { LocalizationHelper } = require("resource://devtools/shared/l10n.js");
const { CurlUtils } = require("resource://devtools/client/shared/curl.js");
const {
 getFormDataSections,
 getUrlQuery,
 parseQueryString,
} = require("resource://devtools/client/netmonitor/src/utils/request-utils.js");
const {
 buildHarLog,
} = require("resource://devtools/client/netmonitor/src/har/har-builder-utils.js");
const L10N = new LocalizationHelper("devtools/client/locales/har.properties");
const {
 TIMING_KEYS,
} = require("resource://devtools/client/netmonitor/src/constants.js");

/**
* This object is responsible for building HAR file. See HAR spec:
* https://dvcs.w3.org/hg/webperf/raw-file/tip/specs/HAR/Overview.html
* http://www.softwareishard.com/blog/har-12-spec/
*/
class HarBuilder {
 /**
  * @param {object} options
  *        configuration object
  * @param {boolean} options.connector
  *        Set to true to include HTTP response bodies in the result data
  *        structure.
  * @param {string} options.id
  *        ID of the exported page.
  * @param {boolean} options.includeResponseBodies
  *        Set to true to include HTTP response bodies in the result data
  *        structure.
  * @param {Array} options.items
  *        List of network events to be exported.
  * @param {boolean} options.supportsMultiplePages
  *        Set to true to create distinct page entries for each navigation.
  */
 constructor(options) {
   this._connector = options.connector;
   this._id = options.id;
   this._includeResponseBodies = options.includeResponseBodies;
   this._items = options.items;
   // Page id counter, only used when options.supportsMultiplePages is true.
   this._pageId = options.supportsMultiplePages ? 0 : options.id;
   this._pageMap = [];
   this._supportsMultiplePages = options.supportsMultiplePages;
   this._url = this._connector.currentTarget.url;
 }

 // Public API

 /**
  * This is the main method used to build the entire result HAR data.
  * The process is asynchronous since it can involve additional RDP
  * communication (e.g. resolving long strings).
  *
  * @returns {Promise} A promise that resolves to the HAR object when
  * the entire build process is done.
  */
 async build() {
   this.promises = [];

   // Build basic structure for data.
   const harLog = buildHarLog(appInfo);

   // Build pages.
   this.buildPages(harLog.log);

   // Build entries.
   for (const request of this._items) {
     const entry = await this.buildEntry(harLog.log, request);
     if (entry) {
       harLog.log.entries.push(entry);
     }
   }

   // Some data needs to be fetched from the backend during the
   // build process, so wait till all is done.
   await Promise.all(this.promises);

   return harLog;
 }

 // Helpers
 buildPages(log) {
   if (this._supportsMultiplePages) {
     this.buildPagesFromTargetTitles(log);
   } else if (this._items.length) {
     const firstRequest = this._items[0];
     const page = this.buildPage(this._url, firstRequest);
     log.pages.push(page);
     this._pageMap[this._id] = page;
   }
 }

 buildPagesFromTargetTitles(log) {
   // Retrieve the additional HAR data collected by the connector.
   const { initialURL, navigationRequests } = this._connector.getHarData();
   const firstNavigationRequest = navigationRequests[0];
   const firstRequest = this._items[0];

   if (
     !firstNavigationRequest ||
     firstRequest.resourceId !== firstNavigationRequest.resourceId
   ) {
     // If the first request is not a navigation request, it must be related
     // to the initial page. Create a first page entry for such early requests.
     const initialPage = this.buildPage(initialURL, firstRequest);
     log.pages.push(initialPage);
   }

   for (const request of navigationRequests) {
     const page = this.buildPage(request.url, request);
     log.pages.push(page);
   }
 }

 buildPage(url, networkEvent) {
   const page = {};

   page.id = "page_" + this._pageId;
   page.pageTimings = this.buildPageTimings(page, networkEvent);
   page.startedDateTime = dateToHarString(new Date(networkEvent.startedMs));

   // To align with other existing implementations of HAR exporters, the title
   // should contain the page URL and not the page title.
   page.title = url;

   // Increase the pageId, for upcoming calls to buildPage.
   // If supportsMultiplePages is disabled this method is only called once.
   this._pageId++;

   return page;
 }

 getPage(log, entry) {
   const existingPage = log.pages.findLast(
     ({ startedDateTime }) => startedDateTime <= entry.startedDateTime
   );

   if (!existingPage) {
     throw new Error(
       "Could not find a page for request: " + entry.request.url
     );
   }

   return existingPage;
 }

 async buildEntry(log, networkEvent) {
   const entry = {};
   entry.startedDateTime = dateToHarString(new Date(networkEvent.startedMs));

   let { eventTimings, id } = networkEvent;
   try {
     if (!eventTimings && this._connector.requestData) {
       eventTimings = await this._connector.requestData(id, "eventTimings");
     }

     entry.request = await this.buildRequest(networkEvent);
     entry.response = await this.buildResponse(networkEvent);
     entry.cache = await this.buildCache(networkEvent);
   } catch (e) {
     // Ignore any request for which we can't retrieve lazy data
     // The request has most likely been destroyed on the server side,
     // either because persist is disabled or the request's target/WindowGlobal/process
     // has been destroyed.
     console.warn("HAR builder failed on", networkEvent.url, e, e.stack);
     return null;
   }
   entry.timings = eventTimings ? eventTimings.timings : {};

   // Calculate total time by summing all timings. Note that
   // `networkEvent.totalTime` can't be used since it doesn't have to
   // correspond to plain summary of individual timings.
   // With TCP Fast Open and TLS early data sending data can
   // start at the same time as connect (we can send data on
   // TCP syn packet). Also TLS handshake can carry application
   // data thereby overlapping a sending data period and TLS
   // handshake period.
   entry.time = TIMING_KEYS.reduce((sum, type) => {
     const time = entry.timings[type];
     return typeof time != "undefined" && time != -1 ? sum + time : sum;
   }, 0);

   // Security state isn't part of HAR spec, and so create
   // custom field that needs to use '_' prefix.
   entry._securityState = networkEvent.securityState;

   if (networkEvent.remoteAddress) {
     entry.serverIPAddress = networkEvent.remoteAddress;
   }

   if (networkEvent.remotePort) {
     entry.connection = networkEvent.remotePort + "";
   }

   const page = this.getPage(log, entry);
   entry.pageref = page.id;

   return entry;
 }

 buildPageTimings() {
   // Event timing info isn't available
   const timings = {
     onContentLoad: -1,
     onLoad: -1,
   };

   // TODO: This method currently ignores the networkEvent and always retrieves
   // the same timing markers for all pages. Seee Bug 1833806.
   if (this._connector.getTimingMarker) {
     timings.onContentLoad = this._connector.getTimingMarker(
       "firstDocumentDOMContentLoadedTimestamp"
     );
     timings.onLoad = this._connector.getTimingMarker(
       "firstDocumentLoadTimestamp"
     );
   }

   return timings;
 }

 async buildRequest(networkEvent) {
   // When using HarAutomation, HarCollector will automatically fetch requestHeaders
   // and requestCookies, but when we use it from netmonitor, FirefoxDataProvider
   // should fetch it itself lazily, via requestData.
   let { id, requestHeaders } = networkEvent;
   if (!requestHeaders && this._connector.requestData) {
     requestHeaders = await this._connector.requestData(id, "requestHeaders");
   }

   let { requestCookies } = networkEvent;
   if (!requestCookies && this._connector.requestData) {
     requestCookies = await this._connector.requestData(id, "requestCookies");
   }

   const request = {
     bodySize: 0,
   };
   request.method = networkEvent.method;
   request.url = networkEvent.url;
   request.httpVersion = networkEvent.httpVersion || "";
   request.headers = this.buildHeaders(requestHeaders);
   request.headers = this.appendHeadersPostData(request.headers, networkEvent);
   request.cookies = this.buildCookies(requestCookies);
   request.queryString = parseQueryString(getUrlQuery(networkEvent.url)) || [];
   request.headersSize = requestHeaders.headersSize;
   request.postData = await this.buildPostData(networkEvent);

   if (request.postData?.text) {
     request.bodySize = request.postData.text.length;
   }

   return request;
 }

 /**
  * Fetch all header values from the backend (if necessary) and
  * build the result HAR structure.
  *
  * @param {object} input Request or response header object.
  */
 buildHeaders(input) {
   if (!input) {
     return [];
   }

   return this.buildNameValuePairs(input.headers);
 }

 appendHeadersPostData(input = [], networkEvent) {
   if (!networkEvent.requestPostData) {
     return input;
   }

   this.fetchData(networkEvent.requestPostData.postData.text).then(value => {
     const multipartHeaders = CurlUtils.getHeadersFromMultipartText(value);
     for (const header of multipartHeaders) {
       input.push(header);
     }
   });

   return input;
 }

 buildCookies(input) {
   if (!input) {
     return [];
   }

   return this.buildNameValuePairs(input.cookies || input);
 }

 buildNameValuePairs(entries) {
   const result = [];

   // HAR requires headers array to be presented, so always
   // return at least an empty array.
   if (!entries) {
     return result;
   }

   // Make sure header values are fully fetched from the server.
   entries.forEach(entry => {
     this.fetchData(entry.value).then(value => {
       result.push({
         name: entry.name,
         value,
       });
     });
   });

   return result;
 }

 async buildPostData(networkEvent) {
   // When using HarAutomation, HarCollector will automatically fetch requestPostData
   // and requestHeaders, but when we use it from netmonitor, FirefoxDataProvider
   // should fetch it itself lazily, via requestData.
   let { id, requestHeaders, requestPostData } = networkEvent;
   let requestHeadersFromUploadStream;

   if (!requestPostData && this._connector.requestData) {
     requestPostData = await this._connector.requestData(
       id,
       "requestPostData"
     );
     requestHeadersFromUploadStream = requestPostData.uploadHeaders;
   }

   if (!requestPostData.postData.text) {
     return undefined;
   }

   if (!requestHeaders && this._connector.requestData) {
     requestHeaders = await this._connector.requestData(id, "requestHeaders");
   }

   const postData = {
     mimeType: findValue(requestHeaders.headers, "content-type"),
     params: [],
     text: requestPostData.postData.text,
   };

   if (requestPostData.postDataDiscarded) {
     postData.comment = L10N.getStr("har.requestBodyNotIncluded");
     return postData;
   }

   // If we are dealing with URL encoded body, parse parameters.
   if (
     CurlUtils.isUrlEncodedRequest({
       headers: requestHeaders.headers,
       postDataText: postData.text,
     })
   ) {
     postData.mimeType = "application/x-www-form-urlencoded";
     // Extract form parameters and produce nice HAR array.
     const formDataSections = await getFormDataSections(
       requestHeaders,
       requestHeadersFromUploadStream,
       requestPostData,
       this._connector.getLongString
     );

     formDataSections.forEach(section => {
       const paramsArray = parseQueryString(section);
       if (paramsArray) {
         postData.params = [...postData.params, ...paramsArray];
       }
     });
   }

   return postData;
 }

 async buildResponse(networkEvent) {
   // When using HarAutomation, HarCollector will automatically fetch responseHeaders
   // and responseCookies, but when we use it from netmonitor, FirefoxDataProvider
   // should fetch it itself lazily, via requestData.

   let { id, responseCookies, responseHeaders } = networkEvent;
   if (!responseHeaders && this._connector.requestData) {
     responseHeaders = await this._connector.requestData(
       id,
       "responseHeaders"
     );
   }

   if (!responseCookies && this._connector.requestData) {
     responseCookies = await this._connector.requestData(
       id,
       "responseCookies"
     );
   }

   const response = {
     status: 0,
   };

   // Arbitrary value if it's aborted to make sure status has a number
   if (networkEvent.status) {
     response.status = parseInt(networkEvent.status, 10);
   }
   response.statusText = networkEvent.statusText || "";
   response.httpVersion = networkEvent.httpVersion || "";

   response.headers = this.buildHeaders(responseHeaders);
   response.cookies = this.buildCookies(responseCookies);
   response.content = await this.buildContent(networkEvent);

   const headers = responseHeaders ? responseHeaders.headers : null;
   const headersSize = responseHeaders ? responseHeaders.headersSize : -1;

   response.redirectURL = findValue(headers, "Location");
   response.headersSize = headersSize;

   // 'bodySize' is size of the received response body in bytes.
   // Set to zero in case of responses coming from the cache (304).
   // Set to -1 if the info is not available.
   if (typeof networkEvent.transferredSize != "number") {
     response.bodySize = response.status == 304 ? 0 : -1;
   } else {
     response.bodySize = networkEvent.transferredSize;
   }

   return response;
 }

 async buildContent(networkEvent) {
   const content = {
     mimeType: networkEvent.mimeType,
     size: -1,
   };

   // When using HarAutomation, HarCollector will automatically fetch responseContent,
   // but when we use it from netmonitor, FirefoxDataProvider should fetch it itself
   // lazily, via requestData.
   let { responseContent } = networkEvent;
   if (!responseContent && this._connector.requestData) {
     responseContent = await this._connector.requestData(
       networkEvent.id,
       "responseContent"
     );
   }
   if (responseContent?.content) {
     content.size = responseContent.content.size;
     content.encoding = responseContent.content.encoding;
   }

   const includeBodies = this._includeResponseBodies;
   const contentDiscarded = responseContent
     ? responseContent.contentDiscarded
     : false;

   // The comment is appended only if the response content
   // is explicitly discarded.
   if (!includeBodies || contentDiscarded) {
     content.comment = L10N.getStr("har.responseBodyNotIncluded");
     return content;
   }

   if (responseContent) {
     const { text } = responseContent.content;
     this.fetchData(text).then(value => {
       content.text = value;
     });
   }

   return content;
 }

 async buildCache(networkEvent) {
   const cache = {};

   // if resource has changed, return early
   if (networkEvent.status != "304") {
     return cache;
   }

   if (networkEvent.responseCacheAvailable && this._connector.requestData) {
     const responseCache = await this._connector.requestData(
       networkEvent.id,
       "responseCache"
     );
     if (responseCache.cache) {
       cache.afterRequest = this.buildCacheEntry(responseCache.cache);
     }
   } else if (networkEvent.responseCache?.cache) {
     cache.afterRequest = this.buildCacheEntry(
       networkEvent.responseCache.cache
     );
   } else {
     cache.afterRequest = null;
   }

   return cache;
 }

 buildCacheEntry(cacheEntry) {
   const cache = {};

   if (typeof cacheEntry !== "undefined") {
     cache.expires = findKeys(cacheEntry, ["expirationTime", "expires"]);
     cache.lastFetched = findKeys(cacheEntry, ["lastFetched"]);

     // TODO: eTag support
     // Har format expects cache entries to provide information about eTag,
     // however this is not currently exposed on nsICacheEntry.
     // This should be stored under cache.eTag. See Bug 1799844.
     cache.fetchCount = findKeys(cacheEntry, ["fetchCount"]);

     // har-importer.js, along with other files, use buildCacheEntry
     // initial value comes from properties without underscores.
     // this checks for both in appropriate order.
     cache._dataSize = findKeys(cacheEntry, ["storageDataSize", "_dataSize"]);
     cache._lastModified = findKeys(cacheEntry, [
       "lastModified",
       "_lastModified",
     ]);
     cache._device = findKeys(cacheEntry, ["deviceID", "_device"]);
   }

   return cache;
 }

 // RDP Helpers
 fetchData(string) {
   const promise = this._connector.getLongString(string).then(value => {
     return value;
   });

   // Building HAR is asynchronous and not done till all
   // collected promises are resolved.
   this.promises.push(promise);

   return promise;
 }
}

// Helpers

/**
* Find specified keys within an object.
* Searches object for keys passed in, returns first value returned,
* or an empty string.
*
* @param obj (object)
* @param keys (array)
* @returns {string}
*/
function findKeys(obj, keys) {
 if (!keys) {
   return "";
 }

 const keyFound = keys.filter(key => obj[key]);
 if (!keys.length) {
   return "";
 }

 const value = obj[keyFound[0]];
 if (typeof value === "undefined" || typeof value === "object") {
   return "";
 }

 return String(value);
}

/**
* Find specified value within an array of name-value pairs
* (used for headers, cookies and cache entries)
*/
function findValue(arr, name) {
 if (!arr) {
   return "";
 }

 name = name.toLowerCase();
 const result = arr.find(entry => entry.name.toLowerCase() == name);
 return result ? result.value : "";
}

/**
* Generate HAR representation of a date.
* (YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.45+01:00)
* See also HAR Schema: http://janodvarko.cz/har/viewer/
*
* Note: it would be great if we could utilize Date.toJSON(), but
* it doesn't return proper time zone offset.
*
* An example:
* This helper returns:    2015-05-29T16:10:30.424+02:00
* Date.toJSON() returns:  2015-05-29T14:10:30.424Z
*
* @param date {Date} The date object we want to convert.
*/
function dateToHarString(date) {
 function f(n, c) {
   if (!c) {
     c = 2;
   }
   let s = String(n);
   while (s.length < c) {
     s = "0" + s;
   }
   return s;
 }

 const result =
   date.getFullYear() +
   "-" +
   f(date.getMonth() + 1) +
   "-" +
   f(date.getDate()) +
   "T" +
   f(date.getHours()) +
   ":" +
   f(date.getMinutes()) +
   ":" +
   f(date.getSeconds()) +
   "." +
   f(date.getMilliseconds(), 3);

 let offset = date.getTimezoneOffset();
 const positive = offset > 0;

 // Convert to positive number before using Math.floor (see issue 5512)
 offset = Math.abs(offset);
 const offsetHours = Math.floor(offset / 60);
 const offsetMinutes = Math.floor(offset % 60);
 const prettyOffset =
   (positive > 0 ? "-" : "+") + f(offsetHours) + ":" + f(offsetMinutes);

 return result + prettyOffset;
}

// Exports from this module
exports.HarBuilder = HarBuilder;