2020-03-20 14:24:05 +00:00
|
|
|
from typing import List
|
|
|
|
|
|
|
|
|
|
|
|
def allocation_num(number_of_bytes: int, partitions: int) -> List[str]:
|
|
|
|
"""
|
|
|
|
Divide a number of bytes into x partitions.
|
|
|
|
|
|
|
|
In a multi-threaded download, this algorithm could be used to provide
|
|
|
|
each worker thread with a block of non-overlapping bytes to download.
|
|
|
|
For example:
|
|
|
|
for i in allocation_list:
|
|
|
|
requests.get(url,headers={'Range':f'bytes={i}'})
|
|
|
|
|
|
|
|
parameter
|
|
|
|
------------
|
|
|
|
: param number_of_bytes
|
|
|
|
: param partitions
|
|
|
|
|
|
|
|
return
|
|
|
|
------------
|
|
|
|
: return: list of bytes to be assigned to each worker thread
|
|
|
|
|
|
|
|
Examples:
|
|
|
|
------------
|
|
|
|
>>> allocation_num(16647, 4)
|
|
|
|
['0-4161', '4162-8322', '8323-12483', '12484-16647']
|
|
|
|
>>> allocation_num(888, 888)
|
|
|
|
Traceback (most recent call last):
|
|
|
|
...
|
|
|
|
ValueError: partitions can not >= number_of_bytes!
|
|
|
|
>>> allocation_num(888, 999)
|
|
|
|
Traceback (most recent call last):
|
|
|
|
...
|
|
|
|
ValueError: partitions can not >= number_of_bytes!
|
|
|
|
>>> allocation_num(888, -4)
|
|
|
|
Traceback (most recent call last):
|
|
|
|
...
|
|
|
|
ValueError: partitions must be a positive number!
|
|
|
|
"""
|
|
|
|
if partitions <= 0:
|
2020-03-28 06:24:59 +00:00
|
|
|
raise ValueError("partitions must be a positive number!")
|
2020-03-20 14:24:05 +00:00
|
|
|
if partitions >= number_of_bytes:
|
2020-03-28 06:24:59 +00:00
|
|
|
raise ValueError("partitions can not >= number_of_bytes!")
|
2020-03-20 14:24:05 +00:00
|
|
|
bytes_per_partition = number_of_bytes // partitions
|
2020-03-28 06:24:59 +00:00
|
|
|
allocation_list = [f"0-{bytes_per_partition}"]
|
2020-03-20 14:24:05 +00:00
|
|
|
for i in range(1, partitions - 1):
|
2020-03-28 06:24:59 +00:00
|
|
|
length = f"{bytes_per_partition * i + 1}-{bytes_per_partition * (i + 1)}"
|
2020-03-20 14:24:05 +00:00
|
|
|
allocation_list.append(length)
|
2020-03-28 06:24:59 +00:00
|
|
|
allocation_list.append(
|
|
|
|
f"{(bytes_per_partition * (partitions - 1)) + 1}-" f"{number_of_bytes}"
|
|
|
|
)
|
2020-03-20 14:24:05 +00:00
|
|
|
return allocation_list
|
|
|
|
|
|
|
|
|
2020-03-28 06:24:59 +00:00
|
|
|
if __name__ == "__main__":
|
2020-03-20 14:24:05 +00:00
|
|
|
import doctest
|
2020-03-28 06:24:59 +00:00
|
|
|
|
2020-03-20 14:24:05 +00:00
|
|
|
doctest.testmod()
|