swarmling.sh (3731B)
1 #!/bin/bash 2 3 # resources: 4 # https://stackoverflow.com/questions/8789729/how-to-zero-pad-a-sequence-of-integers-in-bash-so-that-all-have-the-same-width 5 # https://tldp.org/LDP/Bash-Beginners-Guide/html/sect_07_01.html 6 # https://linuxize.com/post/bash-for-loop/ 7 # https://unix.stackexchange.com/questions/107048/pass-multiple-commands-to-flock 8 # https://superuser.com/questions/515313/force-wget-to-timeout#515315 9 10 # usage syntax 11 help(){ 12 echo "Download a large file quickly through Tor with concurrent circuits." 13 echo "Usage: (argument positions are hardcoded.)" 14 echo "swarmling.sh -t <concurrent_circuits> -n <largest_fragment> -u <url_without_part> -o <filename>" 15 echo "Note:" 16 echo "Number of fragments is last number on server +1 as index starts from 0." 17 } 18 19 # function to be forked in background to download 20 function dlThread(){ 21 # enter tempdir 22 cd .swarmling_buffer 23 24 # do while loop 25 success=1 26 while [ ${success} = 1 ]; do 27 # download through a different circuit each time, loop on timeout 28 # uses CURL to kill off a tor circuit if it is under 50KB/s for 5 seconds (hardcoded for now) 29 #timeout --preserve-status -s KILL 120 torsocks -i wget --quiet -c --tries=1 --content-disposition "$1" 30 curl -Y 50000 -y 5 -O -C - -s -J -U "${RANDOM}":"${RANDOM}" -x socks5h://localhost:9050 "$1" 31 if [ $? = 0 ]; then 32 success=0 33 fi 34 done 35 36 # update progress 37 flock -x ./progress bash -c 'echo "$(($(cat progress)+1))" > progress'; 38 flock -x ./threads bash -c 'echo "$(($(cat threads)-1))" > threads'; 39 } 40 41 function testThread(){ 42 cd .swarmling_buffer/ 43 44 sleep $((${RANDOM}%5)) 45 echo "Thread done." 46 47 # update progress 48 flock -x ./progress bash -c 'echo "$(($(cat progress)+1))" > progress'; 49 } 50 51 function testThreadConcurrent(){ 52 cd .swarmling_buffer/ 53 54 sleep $((${RANDOM}%5)) 55 echo "Concurrent thread done." 56 57 # update progress and decrease thread counter 58 flock -x ./progress bash -c 'echo "$(($(cat progress)+1))" > progress'; 59 flock -x ./threads bash -c 'echo "$(($(cat threads)-1))" > threads'; 60 } 61 62 if [ "$1" = "" -o "$1" = "-h" -o "$1" = "--help" ]; then 63 help; 64 exit; 65 else 66 threadsMax="$2" 67 largest_fragment="$4" 68 url="$6" 69 output="$8" 70 71 # clean and make temp download dir 72 if [ -a .swarmling_buffer ]; then 73 rm -r .swarmling_buffer 74 fi 75 mkdir .swarmling_buffer 76 echo 0 > .swarmling_buffer/progress 77 echo 0 > .swarmling_buffer/threads 78 79 # ---- v 80 81 # keep running concurrent threads up to ${threadsMax} while until there are no more fragments to download 82 running=1 83 i=0 84 while [ ${running} = 1 ]; do 85 # run concurrent thread in background if there are less than specified threads running 86 if [ $(flock -x .swarmling_buffer/threads cat .swarmling_buffer/threads) -lt ${threadsMax} ]; then 87 # add one thread to thread counter 88 flock -x .swarmling_buffer/threads bash -c 'echo "$(($(cat .swarmling_buffer/threads)+1))" > .swarmling_buffer/threads'; 89 90 # run thread in background 91 dlThread "${url}.part$(printf '%05d' $i)" & 92 i=$(($i+1)) 93 #testThreadConcurrent & 94 else 95 #debug 96 sleep 1 97 fi 98 99 # exit loop if downloaded + in progress are equal to ${largest_thread} 100 if [ $(flock -x .swarmling_buffer/progress cat .swarmling_buffer/progress) -ge $(( ${largest_fragment} - $(flock -x .swarmling_buffer/threads cat .swarmling_buffer/threads) )) ]; then 101 running=0; 102 fi 103 echo $(cat .swarmling_buffer/progress)/${largest_fragment} 104 done 105 106 # wait for all threads to be done 107 while [ $(flock -x .swarmling_buffer/threads cat .swarmling_buffer/threads) -gt 0 ]; do 108 echo $(cat .swarmling_buffer/progress)/${largest_fragment} 109 sleep 1 110 done 111 112 113 # ---- ^ 114 115 # indicate completion 116 echo "DONE" 117 118 # merge downloaded files 119 cat .swarmling_buffer/*part* > "${output}" 120 121 #remove buffer dir 122 rm -r .swarmling_buffer/ 123 fi