moved all files to subproject

ceph_bench/benchmark.py

@@ -0,0 +1,36 @@
+# copyright: Canonical
+"Run the bench cases (remote to local)."
+import datetime
+import subprocess
+from pathlib import Path
+
+from loguru import logger as log
+
+
+def run_single_bench(
+    runnum,
+    test_case,
+    files_type,
+    source,
+    target,
+    results,
+):
+    log.info(f"Syncing {runnum} {test_case} {files_type}")
+    start = datetime.datetime.now()
+    subprocess.run(
+        [
+            "rsync",
+            "-a",
+            f"{source}",
+            f"{target}",
+        ],
+        capture_output=True,
+        # shell=True,
+    )
+    time = datetime.datetime.now() - start
+    size = sum(f.stat().st_size for f in Path(target).glob("**/*") if f.is_file())
+    size_MB = size / 1000 / 1000 / 8
+    speed = size_MB / time.total_seconds()
+    log.info(f"{files_type} - {speed} MB/s")
+    subprocess.run(["rm", "-rf", target])
+    results.add([runnum, test_case, files_type, speed, time.total_seconds(), size_MB])

ceph_bench/create_instances.py

@@ -0,0 +1,92 @@
+# copyright: Canonical
+"""Tools for setting up a rsync bench environment."""
+import json
+import subprocess
+
+
+def run_osc(cmdlist: list[str]):
+    """Launch a nicely formatted command on openstack and retrieve result in json.
+
+    Args:
+        cmdlist (list[str]): Cmd line args
+
+    Returns:
+        _type_: _description_
+    """
+    cmdlist.extend(["--format", "json"])
+    res = subprocess.run(cmdlist, capture_output=True, text=True)
+    return json.loads(res.stdout or "[]")
+
+
+def provision():
+    """Provision servers on openstack."""
+    run_osc(
+        [
+            "openstack",
+            "flavor",
+            "create",
+            "--public",
+            "8c-64r-150s",
+            "--ram",
+            "65536",
+            "--disk",
+            "150",
+            "--vcpus 8",
+        ]
+    )
+
+    for x in range(2):
+        serv_resp = run_osc(
+            [
+                "openstack",
+                "server",
+                "create",
+                "--image",
+                "ubuntu2204",
+                "--key-name",
+                "alexmicouleau",
+                "--network",
+                "bench-net",
+                "--flavor",
+                "8c-64r-150s",
+                f"alex_rsync_{x}",
+            ]
+        )
+        serv_id = serv_resp["id"]
+        fip_res = run_osc(
+            [
+                "openstack",
+                "floating",
+                "ip",
+                "create",
+                "--description",
+                f"alex_rsync_{x}",
+                "ext-net",
+            ]
+        )
+        fip_ad = fip_res["floating_ip_address"]
+        run_osc(
+            [
+                "openstack",
+                "server",
+                "add",
+                "floating",
+                "ip",
+                f"{serv_id}",
+                f"{fip_ad}",
+            ]
+        )
+
+        vol_res = run_osc(
+            [
+                "openstack",
+                "volume",
+                "create",
+                "--size",
+                "150",
+                f"alex_rsync{x}",
+            ]
+        )
+        vol_id = vol_res["id"]
+
+        add_vol_res = run_osc(["openstack", "server", "add", "volume", serv_id, vol_id])

ceph_bench/definitions.py

@@ -0,0 +1,25 @@
+SAMPLES = {
+    "XXXS": {"size": 0.1, "number": 10000},
+    "XXS": {"size": 10, "number": 10000},
+    "XS": {"size": 100, "number": 10000},
+    "S": {"size": 1000, "number": 1000},
+    "M": {"size": 10000, "number": 500},
+    "ML": {"size": 100000, "number": 100},
+    "L": {"size": 1000000, "number": 20},
+    "XL": {"size": 5000000, "number": 5},
+}
+
+
+BENCH_DIR_NAMES = list(SAMPLES.keys()) + ["Rand"]
+
+TEST_FILES = {
+    "XXS": {"size": 10, "number": 100},
+    "M": {"size": 10000, "number": 5},
+}
+
+INST_STO_ROOT = "/mnt/inststo/"
+CEPH_ROOT = "/mnt/vol/"
+SERV_URL = "172.172.0.196::share_rsync"
+REMOTE_INST_STO_ROOT = "inststo/"
+REMOTE_CEPH_ROOT = "vol/"
+NUM_RUNS = 50

ceph_bench/generate_samples.py

@@ -0,0 +1,58 @@
+# copyright: Canonical
+import os
+import subprocess
+from uuid import uuid4
+
+from loguru import logger as log
+from tqdm import tqdm
+
+from definitions import INST_STO_ROOT, SAMPLES, TEST_FILES
+
+if __name__ == "__main__":
+    for path, data in SAMPLES.items():
+        log.info(f"processing {path}")
+        os.makedirs(INST_STO_ROOT + path, exist_ok=True)
+        for x in tqdm(range(data["number"])):
+            subprocess.run(
+                [
+                    "dd",
+                    "if=/dev/urandom",
+                    f"of={INST_STO_ROOT+path}/{x}",
+                    f"bs={int(data['size']*1000)}",
+                    "count=1",
+                ],
+                capture_output=True,
+                # shell=True,
+            )
+
+    for path, data in SAMPLES.items():
+        log.info(f"processing RANDOM {path}")
+        os.makedirs(INST_STO_ROOT + "Rand", exist_ok=True)
+        for x in tqdm(range(data["number"])):
+            subprocess.run(
+                [
+                    "dd",
+                    "if=/dev/urandom",
+                    f"of={INST_STO_ROOT}Rand/{uuid4()}",
+                    f"bs={int(data['size']*1000)}",
+                    "count=1",
+                ],
+                capture_output=True,
+                # shell=True,
+            )
+
+    for path, data in TEST_FILES.items():
+        log.info(f"processing tests {path}")
+        os.makedirs(INST_STO_ROOT + "test", exist_ok=True)
+        for x in tqdm(range(data["number"])):
+            subprocess.run(
+                [
+                    "dd",
+                    "if=/dev/urandom",
+                    f"of={INST_STO_ROOT}test/{uuid4()}",
+                    f"bs={int(data['size']*1000)}",
+                    "count=1",
+                ],
+                capture_output=True,
+                # shell=True,
+            )

ceph_bench/requirements.txt

@@ -0,0 +1,21 @@
+asttokens==2.4.0
+backcall==0.2.0
+decorator==5.1.1
+exceptiongroup==1.1.3
+executing==1.2.0
+ipython==8.15.0
+jedi==0.19.0
+loguru==0.7.1
+matplotlib-inline==0.1.6
+parso==0.8.3
+pexpect==4.8.0
+pickleshare==0.7.5
+prompt-toolkit==3.0.39
+ptyprocess==0.7.0
+pure-eval==0.2.2
+Pygments==2.16.1
+six==1.16.0
+stack-data==0.6.2
+tqdm==4.66.1
+traitlets==5.9.0
+wcwidth==0.2.6

ceph_bench/results_tools.py

@@ -0,0 +1,26 @@
+# copyright: Canonical
+"Tools for managing results."
+import csv
+from uuid import uuid4
+
+
+class Results:
+    def __init__(self, *args, **kwargs):
+        self.result_list = [["Run num", "test_case", "files", "MB/s", "time", "full size"]]
+        self.name = f"results{uuid4()}.csv"
+
+    def add(self, row: list, save: bool = True) -> None:
+        """Add a row to the results lists.
+
+        Args:
+            row (list): a row of results.
+        """
+        self.result_list.append(row)
+        if save:
+            self.save()
+
+    def save_to_csv(self) -> None:
+        """Save results to csv file in the current directory."""
+        with open(self.name, "w") as f:
+            cw = csv.writer(f)
+            cw.writerows(self.result_list)

ceph_bench/run.py

@@ -0,0 +1,76 @@
+# copyright: Canonical
+"Run the bench cases (remote to local)."
+import json
+
+import click
+
+from benchmark import run_single_bench
+from definitions import (
+    BENCH_DIR_NAMES,
+    CEPH_ROOT,
+    INST_STO_ROOT,
+    NUM_RUNS,
+    REMOTE_INST_STO_ROOT,
+    SERV_URL,
+)
+from results_tools import Results
+
+results = Results()
+
+
+@click.group()
+def cli():
+    pass
+
+
+@cli.command()
+def local():
+    click.echo("Running locally")
+    for runnum in range(NUM_RUNS):
+        for files_type in BENCH_DIR_NAMES:
+            test_case = "Local-InsSto to CEPH"
+            source = f"{INST_STO_ROOT}/{files_type}"
+            target = CEPH_ROOT + files_type
+            run_single_bench(
+                runnum,
+                test_case,
+                files_type,
+                source,
+                target,
+            )
+
+
+@cli.command()
+def distant():
+    click.echo("Running distant to local")
+    for runnum in range(NUM_RUNS):
+        for files_type in BENCH_DIR_NAMES:
+            test_case = "Distant-InsSto to InsSto"
+            source = f"{SERV_URL}/{REMOTE_INST_STO_ROOT}/{files_type}"
+            target = INST_STO_ROOT + files_type
+            run_single_bench(
+                runnum,
+                test_case,
+                files_type,
+                source,
+                target,
+            )
+
+            test_case = "Distant-InsSto to CEPH"
+            source = f"{SERV_URL}/{REMOTE_INST_STO_ROOT}/{files_type}"
+            target = CEPH_ROOT + files_type
+            run_single_bench(
+                runnum,
+                test_case,
+                files_type,
+                source,
+                target,
+                results,
+            )
+
+
+if __name__ == "__main__":
+    cli()
+
+    json.dump(results.result_list, open("results.json", "w"))
+    results.save_to_csv()