remove-duplicates.py (599B)
1 #!/usr/bin/env python3 2 3 # This Source Code Form is subject to the terms of the Mozilla Public 4 # License, v. 2.0. If a copy of the MPL was not distributed with this 5 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 7 import glob 8 import hashlib 9 import os 10 11 have_hashes = set() 12 13 for file in glob.glob("IC-*"): 14 with open(file, "rb") as f: 15 content = f.read() 16 h = hashlib.new("sha256") 17 h.update(content) 18 digest = h.hexdigest() 19 if digest in have_hashes: 20 print("Removing: %s" % file) 21 os.unlink(file) 22 have_hashes.add(digest)