• Home
• io.deephaven
• deephaven-benchmark
• 0.33.4
• source code
• parallel_download.py

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

io.deephaven.benchmark.run.profile.parallel_download.py Maven / Gradle / Ivy

import asyncio, re, os, traceback
from urllib.request import urlretrieve

def background(f):
    def wrapped(*args, **kwargs):
        return asyncio.get_event_loop().run_in_executor(None, f, *args, **kwargs)
    return wrapped

@background
def download(url):
    try:
        out_path = re.sub('^http.*/deephaven-benchmark/', 'data/deephaven-benchmark/', url)
        os.makedirs(os.path.dirname(out_path), mode=0o777, exist_ok=True)
    except Exception:
        print('Error downloading file:', download, ':', traceback.format_exc())
        return
    try:
        urlretrieve(url + '.gz', out_path + '.gz')
        print('Got', out_path + '.gz')
    except Exception:
        try:
            urlretrieve(url, out_path)
            print('Got', out_path)
        except Exception:
            print('Error downloading file:', out_path, ':', traceback.format_exc())

urls = [
    ${downloadUrls}
]

for url in urls:
    download(url)