Markk116
24175dde36
AssetStoreServer runs on a single dedicated smarm actor thread, so a plain HashMap+VecDeque LRU in front of the SQLite lookup needs no locking. Capacity configurable via CCC_CACHE_CAPACITY (default 256). Bench (bench/RPS): +42-47% RPS under a cache-sized/hot-set workload, but a small net loss under adversarial uniform-random access with a cache smaller than the catalog. Real traffic is hot-set skewed, so net win in practice; capacity should be tuned to the expected hot set.
71 lines
2.2 KiB
Python
71 lines
2.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Seed cdn.db with random packages/versions/files for benchmarking."""
|
|
import gzip
|
|
import os
|
|
import random
|
|
import sqlite3
|
|
import string
|
|
import sys
|
|
|
|
DB_PATH = sys.argv[1] if len(sys.argv) > 1 else "cdn.db"
|
|
N_PACKAGES = int(os.environ.get("N_PACKAGES", 500))
|
|
VERSIONS_PER_PKG = int(os.environ.get("VERSIONS_PER_PKG", 5))
|
|
MIN_SIZE = int(os.environ.get("MIN_SIZE", 512))
|
|
MAX_SIZE = int(os.environ.get("MAX_SIZE", 8192))
|
|
|
|
random.seed(42)
|
|
|
|
if os.path.exists(DB_PATH):
|
|
os.remove(DB_PATH)
|
|
|
|
conn = sqlite3.connect(DB_PATH)
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS packages (
|
|
name TEXT PRIMARY KEY,
|
|
archived INTEGER NOT NULL DEFAULT 0
|
|
)""")
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS versions (
|
|
package TEXT NOT NULL REFERENCES packages(name),
|
|
version TEXT NOT NULL,
|
|
filename TEXT NOT NULL,
|
|
mime_type TEXT NOT NULL,
|
|
gzipped_bytes BLOB NOT NULL,
|
|
archived INTEGER NOT NULL DEFAULT 0,
|
|
PRIMARY KEY (package, version)
|
|
)""")
|
|
|
|
def rand_name(n=10):
|
|
return "".join(random.choices(string.ascii_lowercase, k=n))
|
|
|
|
def rand_body(size):
|
|
chars = string.ascii_letters + string.digits + " \n"
|
|
return "".join(random.choices(chars, k=size)).encode()
|
|
|
|
manifest = [] # (package, version, filename) for the load generator
|
|
|
|
pkg_names = [f"pkg-{rand_name(8)}-{i}" for i in range(N_PACKAGES)]
|
|
|
|
for name in pkg_names:
|
|
conn.execute("INSERT INTO packages (name, archived) VALUES (?, 0)", (name,))
|
|
for v in range(VERSIONS_PER_PKG):
|
|
version = f"{v+1}.0.0"
|
|
filename = f"{rand_name(6)}.js"
|
|
size = random.randint(MIN_SIZE, MAX_SIZE)
|
|
raw = rand_body(size)
|
|
gz = gzip.compress(raw, compresslevel=6)
|
|
conn.execute(
|
|
"INSERT INTO versions (package, version, filename, mime_type, gzipped_bytes, archived) "
|
|
"VALUES (?, ?, ?, 'application/javascript', ?, 0)",
|
|
(name, version, filename, gz),
|
|
)
|
|
manifest.append((name, version, filename))
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
with open("bench/urls.txt", "w") as f:
|
|
for pkg, ver, fn in manifest:
|
|
f.write(f"/assets/{pkg}/{ver}/{fn}\n")
|
|
|
|
print(f"seeded {len(pkg_names)} packages, {len(manifest)} versions -> {DB_PATH}")
|
|
print(f"wrote {len(manifest)} urls -> bench/urls.txt")
|