diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fab8f69 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +venv +*.pyc \ No newline at end of file diff --git a/create_instances.py b/create_instances.py new file mode 100644 index 0000000..e0d8eec --- /dev/null +++ b/create_instances.py @@ -0,0 +1,92 @@ +# copyright: Canonical +"""Tools for setting up a rsync bench environment.""" +import json +import subprocess + + +def run_osc(cmdlist: list[str]): + """Launch a nicely formatted command on openstack and retrieve result in json. + + Args: + cmdlist (list[str]): Cmd line args + + Returns: + _type_: _description_ + """ + cmdlist.extend(["--format", "json"]) + res = subprocess.run(cmdlist, capture_output=True, text=True) + return json.loads(res.stdout or "[]") + + +def provision(): + """Provision servers on openstack.""" + run_osc( + [ + "openstack", + "flavor", + "create", + "--public", + "8c-64r-150s", + "--ram", + "65536", + "--disk", + "150", + "--vcpus 8", + ] + ) + + for x in range(2): + serv_resp = run_osc( + [ + "openstack", + "server", + "create", + "--image", + "ubuntu2204", + "--key-name", + "alexmicouleau", + "--network", + "bench-net", + "--flavor", + "8c-64r-150s", + f"alex_rsync_{x}", + ] + ) + serv_id = serv_resp["id"] + fip_res = run_osc( + [ + "openstack", + "floating", + "ip", + "create", + "--description", + f"alex_rsync_{x}", + "ext-net", + ] + ) + fip_ad = fip_res["floating_ip_address"] + run_osc( + [ + "openstack", + "server", + "add", + "floating", + "ip", + f"{serv_id}", + f"{fip_ad}", + ] + ) + + vol_res = run_osc( + [ + "openstack", + "volume", + "create", + "--size", + "150", + f"alex_rsync{x}", + ] + ) + vol_id = vol_res["id"] + + add_vol_res = run_osc(["openstack", "server", "add", "volume", serv_id, vol_id]) diff --git a/generate_samples.py b/generate_samples.py new file mode 100644 index 0000000..bc356b7 --- /dev/null +++ b/generate_samples.py @@ -0,0 +1,78 @@ +# copyright: Canonical +import os +import subprocess +from uuid import uuid4 + +from loguru import logger as log +from tqdm import tqdm + +files = { + "XXXS": {"size": 0.1, "number": 10000}, + "XXS": {"size": 10, "number": 10000}, + "XS": {"size": 100, "number": 10000}, + "S": {"size": 1000, "number": 1000}, + "M": {"size": 10000, "number": 500}, + "ML": {"size": 100000, "number": 100}, + "L": {"size": 1000000, "number": 20}, + "XL": {"size": 5000000, "number": 5}, +} + +rand = { + "XXS": {"size": 10, "number": 1000}, + "M": {"size": 10000, "number": 50}, + "L": {"size": 1000000, "number": 5}, +} +tests = { + "XXS": {"size": 10, "number": 100}, + "M": {"size": 10000, "number": 5}, +} +root = "/mnt/inststo/" + +if __name__ == "__main__": + for path, data in files.items(): + log.info(f"processing {path}") + os.makedirs(root + path, exist_ok=True) + for x in tqdm(range(data["number"])): + subprocess.run( + [ + "dd", + "if=/dev/urandom", + f"of={root+path}/{x}", + f"bs={int(data['size']*1000)}", + "count=1", + ], + capture_output=True, + # shell=True, + ) + + for path, data in rand.items(): + log.info(f"processing RANDOM {path}") + os.makedirs(root + "Rand", exist_ok=True) + for x in tqdm(range(data["number"])): + subprocess.run( + [ + "dd", + "if=/dev/urandom", + f"of={root}Rand/{uuid4()}", + f"bs={int(data['size']*1000)}", + "count=1", + ], + capture_output=True, + # shell=True, + ) + + for path, data in tests.items(): + log.info(f"processing tests {path}") + os.makedirs(root + "test", exist_ok=True) + for x in tqdm(range(data["number"])): + subprocess.run( + [ + "dd", + "if=/dev/urandom", + f"of={root}test/{uuid4()}", + f"bs={int(data['size']*1000)}", + "count=1", + ], + capture_output=True, + # shell=True, + ) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0aeeeb0 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,21 @@ +asttokens==2.4.0 +backcall==0.2.0 +decorator==5.1.1 +exceptiongroup==1.1.3 +executing==1.2.0 +ipython==8.15.0 +jedi==0.19.0 +loguru==0.7.1 +matplotlib-inline==0.1.6 +parso==0.8.3 +pexpect==4.8.0 +pickleshare==0.7.5 +prompt-toolkit==3.0.39 +ptyprocess==0.7.0 +pure-eval==0.2.2 +Pygments==2.16.1 +six==1.16.0 +stack-data==0.6.2 +tqdm==4.66.1 +traitlets==5.9.0 +wcwidth==0.2.6 diff --git a/run.py b/run.py new file mode 100644 index 0000000..44860d8 --- /dev/null +++ b/run.py @@ -0,0 +1,27 @@ +"Test connectivity on a small subset of files." +# copyright: Canonical +import datetime +import subprocess +from pathlib import Path + +from loguru import logger as log + +SERV_URL = "172.172.0.196::share_rsync" +TARGET = "/mnt/inststo/test" +if __name__ == "__main__": + for x in range(10): + start = datetime.datetime.now() + subprocess.run( + [ + "rsync", + "-a", + f"{SERV_URL}/inststo/test", + f"/mnt/inststo", + ], + capture_output=True, + # shell=True, + ) + time = datetime.datetime.now() - start + size = sum(f.stat().st_size for f in Path(TARGET).glob("**/*") if f.is_file()) + log.info(f"{size/time.total_seconds()/1000/1000/8} MB/s") + subprocess.run(["rm", "-rf", TARGET])