tor-browser

The Tor Browser
git clone https://git.dasho.dev/tor-browser.git
Log | Files | Refs | README | LICENSE

google_storage_helper.py (4379B)

      1 # Copyright 2017 The Chromium Authors
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 
      5 """Helper functions to upload data to Google Storage.
      6 
      7 Text data should be streamed to logdog using |logdog_helper| module.
      8 Due to logdog not having image or HTML viewer, those instead should be uploaded
      9 to Google Storage directly using this module.
     10 """
     11 
     12 import logging
     13 import os
     14 import sys
     15 import time
     16 try:
     17  from urllib.parse import urlparse
     18 except ImportError:
     19  from urlparse import urlparse
     20 
     21 from pylib.constants import host_paths
     22 from pylib.utils import decorators
     23 
     24 if host_paths.DEVIL_PATH not in sys.path:
     25  sys.path.append(host_paths.DEVIL_PATH)
     26 from devil.utils import cmd_helper
     27 
     28 _GSUTIL_PATH = os.path.join(host_paths.DIR_SOURCE_ROOT, 'third_party',
     29                            'catapult', 'third_party', 'gsutil', 'gsutil')
     30 _PUBLIC_URL = 'https://storage.googleapis.com/%s/'
     31 _AUTHENTICATED_URL = 'https://storage.cloud.google.com/%s/'
     32 
     33 
     34 @decorators.NoRaiseException(default_return_value='')
     35 def upload(name, filepath, bucket, gs_args=None, command_args=None,
     36           content_type=None, authenticated_link=True):
     37  """Uploads data to Google Storage.
     38 
     39  Args:
     40    name: Name of the file on Google Storage.
     41    filepath: Path to file you want to upload.
     42    bucket: Bucket to upload file to.
     43    content_type: Content type to upload as. If not specified, Google storage
     44        will attempt to infer content type from file extension.
     45    authenticated_link: Whether to return a link that requires user to
     46        authenticate with a Google account. Setting this to false will return
     47        a link that does not require user to be signed into Google account but
     48        will only work for completely public storage buckets.
     49  Returns:
     50    Web link to item uploaded to Google Storage bucket.
     51  """
     52  bucket = _format_bucket_name(bucket)
     53 
     54  gs_path = 'gs://%s/%s' % (bucket, name)
     55  logging.info('Uploading %s to %s', filepath, gs_path)
     56 
     57  cmd = [_GSUTIL_PATH, '-q']
     58  cmd.extend(gs_args or [])
     59  if content_type:
     60    cmd.extend(['-h', 'Content-Type:%s' % content_type])
     61  cmd.extend(['cp'] + (command_args or []) + [filepath, gs_path])
     62 
     63  cmd_helper.RunCmd(cmd)
     64 
     65  return get_url_link(name, bucket, authenticated_link)
     66 
     67 
     68 @decorators.NoRaiseException(default_return_value='')
     69 def read_from_link(link):
     70  # Note that urlparse returns the path with an initial '/', so we only need to
     71  # add one more after the 'gs;'
     72  gs_path = 'gs:/%s' % urlparse(link).path
     73  cmd = [_GSUTIL_PATH, '-q', 'cat', gs_path]
     74  return cmd_helper.GetCmdOutput(cmd)
     75 
     76 
     77 @decorators.NoRaiseException(default_return_value=False)
     78 def exists(name, bucket):
     79  bucket = _format_bucket_name(bucket)
     80  gs_path = 'gs://%s/%s' % (bucket, name)
     81 
     82  cmd = [_GSUTIL_PATH, '-q', 'stat', gs_path]
     83  return_code = cmd_helper.RunCmd(cmd)
     84  return return_code == 0
     85 
     86 
     87 # TODO(jbudorick): Delete this function. Only one user of it.
     88 def unique_name(basename, suffix='', timestamp=True, device=None):
     89  """Helper function for creating a unique name for a file to store in GS.
     90 
     91  Args:
     92    basename: Base of the unique filename.
     93    suffix: Suffix of filename.
     94    timestamp: Whether or not to add a timestamp to name.
     95    device: Device to add device serial of to name.
     96  """
     97  return '%s%s%s%s' % (
     98      basename,
     99      '_%s' % time.strftime('%Y_%m_%d_T%H_%M_%S-UTC', time.gmtime())
    100          if timestamp else '',
    101      '_%s' % device.serial if device else '',
    102      suffix)
    103 
    104 
    105 def get_url_link(name, bucket, authenticated_link=True):
    106  """Get url link before/without uploading.
    107 
    108  Args:
    109    name: Name of the file on Google Storage.
    110    bucket: Bucket to upload file to.
    111    authenticated_link: Whether to return a link that requires user to
    112        authenticate with a Google account. Setting this to false will return
    113        a link that does not require user to be signed into Google account but
    114        will only work for completely public storage buckets.
    115  Returns:
    116    Web link to item to be uploaded to Google Storage bucket
    117  """
    118  bucket = _format_bucket_name(bucket)
    119  url_template = _AUTHENTICATED_URL if authenticated_link else _PUBLIC_URL
    120  return os.path.join(url_template % bucket, name)
    121 
    122 
    123 def _format_bucket_name(bucket):
    124  if bucket.startswith('gs://'):
    125    bucket = bucket[len('gs://'):]
    126  if bucket.endswith('/'):
    127    bucket = bucket[:-1]
    128  return bucket