tor-browser

The Tor Browser
git clone https://git.dasho.dev/tor-browser.git
Log | Files | Refs | README | LICENSE

talos_from_code.py (4572B)

      1 #! /usr/bin/env python
      2 # This Source Code Form is subject to the terms of the Mozilla Public
      3 # License, v. 2.0. If a copy of the MPL was not distributed with this
      4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
      5 
      6 #
      7 # Script name:   talos_from_code.py
      8 # Purpose:       Read from a talos.json file the different files to download for a talos job
      9 # Author(s):     Zambrano Gasparnian, Armen <armenzg@mozilla.com>
     10 # Target:        Python 2.5
     11 #
     12 import json
     13 import os
     14 import re
     15 import sys
     16 import urllib.parse
     17 from optparse import OptionParser
     18 
     19 
     20 def main():
     21    """
     22    This script downloads a talos.json file which indicates which files to download
     23    for a talos job.
     24    See a talos.json file for a better understand:
     25    https://hg.mozilla.org/mozilla-central/raw-file/default/testing/talos/talos.json
     26    """
     27    parser = OptionParser()
     28    parser.add_option(
     29        "--talos-json-url",
     30        dest="talos_json_url",
     31        type="string",
     32        help="It indicates from where to download the talos.json file.",
     33    )
     34    (options, args) = parser.parse_args()
     35 
     36    # 1) check that the url was passed
     37    if options.talos_json_url is None:
     38        print("You need to specify --talos-json-url.")
     39        sys.exit(1)
     40 
     41    # 2) try to download the talos.json file
     42    try:
     43        jsonFilename = download_file(options.talos_json_url)
     44    except Exception as e:
     45        print("ERROR: We tried to download the talos.json file but something failed.")
     46        print("ERROR: %s" % str(e))
     47        sys.exit(1)
     48 
     49    # 3) download the necessary files
     50    print("INFO: talos.json URL: %s" % options.talos_json_url)
     51    try:
     52        key = "talos.zip"
     53        entity = get_value(jsonFilename, key)
     54        if passesRestrictions(options.talos_json_url, entity["url"]):
     55            # the key is at the same time the filename e.g. talos.zip
     56            print(
     57                "INFO: Downloading %s as %s"
     58                % (entity["url"], os.path.join(entity["path"], key))
     59            )
     60            download_file(entity["url"], entity["path"], key)
     61        else:
     62            print(
     63                "ERROR: You have tried to download a file "
     64                + "from: %s " % entity["url"]
     65                + "which is a location different than http://talos-bundles.pvt.build.mozilla.org/"
     66            )
     67            print("ERROR: This is only allowed for the certain branches.")
     68            sys.exit(1)
     69    except Exception as e:
     70        print("ERROR: %s" % str(e))
     71        sys.exit(1)
     72 
     73 
     74 def passesRestrictions(talosJsonUrl, fileUrl):
     75    """
     76    Only certain branches are exempted from having to host their downloadable files
     77    in talos-bundles.pvt.build.mozilla.org
     78    """
     79    if (
     80        talosJsonUrl.startswith("http://hg.mozilla.org/try/")
     81        or talosJsonUrl.startswith("https://hg.mozilla.org/try/")
     82        or talosJsonUrl.startswith("http://hg.mozilla.org/projects/pine/")
     83        or talosJsonUrl.startswith("https://hg.mozilla.org/projects/pine/")
     84        or talosJsonUrl.startswith("http://hg.mozilla.org/projects/ash/")
     85        or talosJsonUrl.startswith("https://hg.mozilla.org/projects/ash/")
     86    ):
     87        return True
     88    else:
     89        p = re.compile("^http://talos-bundles.pvt.build.mozilla.org/")
     90        m = p.match(fileUrl)
     91        if m is None:
     92            return False
     93        return True
     94 
     95 
     96 def get_filename_from_url(url):
     97    """
     98    This returns the filename of the file we're trying to download
     99    """
    100    parsed = urllib.parse.urlsplit(url.rstrip("/"))
    101    if parsed.path != "":
    102        return parsed.path.rsplit("/", 1)[-1]
    103    else:
    104        print(
    105            "ERROR: We were trying to download a file from %s "
    106            + "but the URL seems to be incorrect."
    107        )
    108        sys.exit(1)
    109 
    110 
    111 def download_file(url, path="", saveAs=None):
    112    """
    113    It downloads a file from URL to the indicated path
    114    """
    115    req = urllib.request.Request(url)
    116    f = urllib.request.urlopen(req)
    117    if path != "" and not os.path.isdir(path):
    118        try:
    119            os.makedirs(path)
    120            print("INFO: directory %s created" % path)
    121        except Exception as e:
    122            print("ERROR: %s" % str(e))
    123            sys.exit(1)
    124    filename = saveAs if saveAs else get_filename_from_url(url)
    125    local_file = open(os.path.join(path, filename), "wb")
    126    local_file.write(f.read())
    127    local_file.close()
    128    return filename
    129 
    130 
    131 def get_value(json_filename, key):
    132    """
    133    It loads up a JSON file and returns the value for the given string
    134    """
    135    f = open(json_filename)
    136    return json.load(f)[key]
    137 
    138 
    139 if __name__ == "__main__":
    140    main()