tor-browser

The Tor Browser
git clone https://git.dasho.dev/tor-browser.git
Log | Files | Refs | README | LICENSE

full.py (4887B)

      1 # This Source Code Form is subject to the terms of the Mozilla Public
      2 # License, v. 2.0. If a copy of the MPL was not distributed with this
      3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
      4 
      5 import asyncio
      6 import os
      7 import random
      8 
      9 from arsenic.errors import UnknownArsenicError, UnknownError
     10 
     11 from condprof.helpers import TabSwitcher, execute_async_script, is_mobile
     12 from condprof.util import get_credentials, logger
     13 
     14 BOOKMARK_FREQUENCY = 5
     15 MAX_URLS = 150
     16 MAX_BOOKMARKS = 250
     17 CallErrors = asyncio.TimeoutError, UnknownError, UnknownArsenicError
     18 
     19 
     20 class Builder:
     21    def __init__(self, options):
     22        self.options = options
     23        self.words = self._read_lines("words.txt")
     24        self.urls = self._build_url_list(self._read_lines("urls.txt"))
     25        self.sync_js = self._load_script("sync")
     26        self.max_bookmarks = options.get("max_bookmarks", MAX_BOOKMARKS)
     27        self.bookmark_js = self._load_script("bookmark")
     28        self.platform = options.get("platform", "")
     29        self.mobile = is_mobile(self.platform)
     30        self.max_urls = options.get("max_urls", MAX_URLS)
     31 
     32        # see Bug 1608604 & see Bug 1619107 - we have stability issues @ bitbar
     33        if self.mobile:
     34            self.max_urls = min(self.max_urls, 20)
     35 
     36        logger.info("platform: %s" % self.platform)
     37        logger.info("max_urls: %s" % self.max_urls)
     38        self.bookmark_frequency = options.get("bookmark_frequency", BOOKMARK_FREQUENCY)
     39 
     40        # we're syncing only on desktop for now
     41        self.syncing = not self.mobile
     42        if self.syncing:
     43            self.username, self.password = get_credentials()
     44            if self.username is None:
     45                raise ValueError("Sync operations need an FxA username and password")
     46        else:
     47            self.username, self.password = None, None
     48 
     49    def _load_script(self, name):
     50        return "\n".join(self._read_lines("%s.js" % name))
     51 
     52    def _read_lines(self, filename):
     53        path = os.path.join(os.path.dirname(__file__), filename)
     54        with open(path) as f:
     55            return f.readlines()
     56 
     57    def _build_url_list(self, urls):
     58        url_list = []
     59        for url in urls:
     60            url = url.strip()
     61            if url.startswith("#"):
     62                continue
     63            for word in self.words:
     64                word = word.strip()
     65                if word.startswith("#"):
     66                    continue
     67                url_list.append((url.format(word), word))
     68        random.shuffle(url_list)
     69        return url_list
     70 
     71    async def add_bookmark(self, session, url, title):
     72        logger.info("Adding bookmark to %s" % url)
     73        return await execute_async_script(
     74            session, self.bookmark_js, url, title, self.max_bookmarks
     75        )
     76 
     77    async def sync(self, session, metadata):
     78        if not self.syncing:
     79            return
     80        # now that we've visited all pages, we want to upload to FXSync
     81        logger.info("Syncing profile to FxSync")
     82        logger.info("Username is %s, password is %s" % (self.username, self.password))
     83        script_res = await execute_async_script(
     84            session,
     85            self.sync_js,
     86            self.username,
     87            self.password,
     88            "https://accounts.stage.mozaws.net",
     89        )
     90        if script_res is None:
     91            script_res = {}
     92        metadata["logs"] = script_res.get("logs", {})
     93        metadata["result"] = script_res.get("result", 0)
     94        metadata["result_message"] = script_res.get("result_message", "SUCCESS")
     95        return metadata
     96 
     97    async def _visit_url(self, current, session, url, word):
     98        await asyncio.wait_for(session.get(url), 5)
     99        if current % self.bookmark_frequency == 0 and not self.mobile:
    100            await asyncio.wait_for(self.add_bookmark(session, url, word), 5)
    101 
    102    async def __call__(self, session):
    103        metadata = {}
    104 
    105        tabs = TabSwitcher(session, self.options)
    106        await tabs.create_windows()
    107        visited = 0
    108 
    109        for current, (url, word) in enumerate(self.urls):
    110            logger.info("%d/%d %s" % (current + 1, self.max_urls, url))
    111            retries = 0
    112            while retries < 3:
    113                try:
    114                    await self._visit_url(current, session, url, word)
    115                    visited += 1
    116                    break
    117                except CallErrors:
    118                    await asyncio.sleep(1.0)
    119                    retries += 1
    120 
    121            if current == self.max_urls - 1:
    122                break
    123 
    124            # switch to the next tab
    125            try:
    126                await asyncio.wait_for(tabs.switch(), 5)
    127            except CallErrors:
    128                # if we can't switch, it's ok
    129                pass
    130 
    131        metadata["visited_url"] = visited
    132        await self.sync(session, metadata)
    133        return metadata
    134 
    135 
    136 async def full(session, options):
    137    builder = Builder(options)
    138    return await builder(session)