gen_last_modified.py (4145B)
1 #!/usr/bin/env python3 2 # This Source Code Form is subject to the terms of the Mozilla Public 3 # License, v. 2.0. If a copy of the MPL was not distributed with this 4 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 6 import glob 7 import json 8 import os 9 10 import buildconfig 11 import mozpack.path as mozpath 12 13 14 def get_last_modified(full_path_to_remote_settings_dump_file): 15 """ 16 Get the last_modified for the given file name. 17 - File must exist 18 - Must be a JSON dictionary with a data list and a timestamp, 19 e.g. `{"data": [], "timestamp": 42}` 20 - Every element in `data` should contain a "last_modified" key. 21 - The first element must have the highest "last_modified" value. 22 """ 23 with open(full_path_to_remote_settings_dump_file, "r", encoding="utf-8") as f: 24 changeset = json.load(f) 25 26 records = changeset["data"] 27 assert isinstance(records, list) 28 last_modified = changeset.get("timestamp") 29 assert isinstance( 30 last_modified, int 31 ), f"{full_path_to_remote_settings_dump_file} is missing the timestamp. See Bug 1725660" 32 33 return last_modified 34 35 36 def main(output): 37 """ 38 Generates a JSON file that maps "bucket/collection" to the last_modified 39 value within. 40 41 Returns a set of the file locations of the recorded RemoteSettings dumps, 42 so that the build backend can invoke this script again when needed. 43 44 The validity of the JSON file is verified through unit tests at 45 services/settings/test/unit/test_remote_settings_dump_lastmodified.js 46 """ 47 # The following build targets currently use RemoteSettings dumps: 48 # Firefox https://searchfox.org/mozilla-central/rev/94d6086481754e154b6f042820afab6bc9900a30/browser/installer/package-manifest.in#281-285 # NOQA: E501 49 # Firefox for Android https://searchfox.org/mozilla-central/rev/94d6086481754e154b6f042820afab6bc9900a30/mobile/android/installer/package-manifest.in#88-91 # NOQA: E501 50 # Thunderbird https://searchfox.org/comm-central/rev/89f957706bbda77e5f34e64e117e7ce121bb5d83/mail/installer/package-manifest.in#280-285 # NOQA: E501 51 # SeaMonkey https://searchfox.org/comm-central/rev/89f957706bbda77e5f34e64e117e7ce121bb5d83/suite/installer/package-manifest.in#307-309 # NOQA: E501 52 assert buildconfig.substs["MOZ_BUILD_APP"] in ( 53 "browser", 54 "mobile/android", 55 "mobile/ios", 56 "comm/mail", 57 "comm/suite", 58 ), ( 59 "Cannot determine location of Remote Settings " 60 f"dumps for platform {buildconfig.substs['MOZ_BUILD_APP']}" 61 ) 62 63 dumps_locations = [] 64 if buildconfig.substs["MOZ_BUILD_APP"] == "browser": 65 dumps_locations += ["services/settings/dumps/"] 66 dumps_locations += ["services/settings/static-dumps/"] 67 elif buildconfig.substs["MOZ_BUILD_APP"] == "mobile/android": 68 dumps_locations += ["services/settings/dumps/"] 69 dumps_locations += ["services/settings/static-dumps/"] 70 elif buildconfig.substs["MOZ_BUILD_APP"] == "mobile/ios": 71 dumps_locations += ["services/settings/dumps/"] 72 elif buildconfig.substs["MOZ_BUILD_APP"] == "comm/mail": 73 dumps_locations += ["services/settings/dumps/"] 74 dumps_locations += ["comm/mail/app/settings/dumps/"] 75 elif buildconfig.substs["MOZ_BUILD_APP"] == "comm/suite": 76 dumps_locations += ["services/settings/dumps/"] 77 78 remotesettings_dumps = {} 79 for dumps_location in dumps_locations: 80 dumps_root_dir = mozpath.join(buildconfig.topsrcdir, *dumps_location.split("/")) 81 for path in glob.iglob(mozpath.join(dumps_root_dir, "*", "*.json")): 82 folder, filename = os.path.split(path) 83 bucket = os.path.basename(folder) 84 collection, _ = os.path.splitext(filename) 85 remotesettings_dumps[f"{bucket}/{collection}"] = path 86 87 output_dict = {} 88 input_files = set() 89 90 for key, input_file in remotesettings_dumps.items(): 91 input_files.add(input_file) 92 output_dict[key] = get_last_modified(input_file) 93 94 json.dump(output_dict, output, sort_keys=True) 95 return input_files