Initial public release
Luxe 0.1.0: a coding agent harness for the terminal. See README.md to get started and CHANGELOG.md for releases.
This commit is contained in:
109
scripts/update-models-snapshot.sh
Executable file
109
scripts/update-models-snapshot.sh
Executable file
@@ -0,0 +1,109 @@
|
||||
#!/usr/bin/env bash
|
||||
# Regenerate the embedded models.dev snapshot.
|
||||
#
|
||||
# Luxe resolves model metadata from four layers (defaults -> snapshot -> cache -> config). The
|
||||
# snapshot is the offline/first-run floor: it's what a fresh install with no network knows. The
|
||||
# runtime refresh keeps a live install current, so this only needs to run at release cadence —
|
||||
# CI does it weekly and opens a PR when the bytes change.
|
||||
#
|
||||
# The projection keeps only the fields `luxe_providers::catalog::schema` reads, then gzips, which
|
||||
# takes the 3.2 MB API response down to ~150 KB in the binary.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/update-models-snapshot.sh [--url URL]
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
URL="${MODELS_DEV_URL:-https://models.dev/api.json}"
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--url)
|
||||
URL="$2"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
echo "unknown argument: $1" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
OUT="$ROOT/crates/luxe-providers/data/models-dev-snapshot.json.gz"
|
||||
TMP="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
echo "fetching $URL"
|
||||
curl --fail --silent --show-error --location --max-time 60 --compressed "$URL" -o "$TMP/api.json"
|
||||
|
||||
python3 - "$TMP/api.json" "$TMP/snapshot.json" <<'PY'
|
||||
import json, sys
|
||||
|
||||
MODEL_FIELDS = (
|
||||
"name", "reasoning", "tool_call", "release_date", "status",
|
||||
"limit", "cost", "modalities", "reasoning_options",
|
||||
)
|
||||
COST_FIELDS = ("input", "output", "cache_read", "cache_write")
|
||||
|
||||
def clean(value):
|
||||
"""Drop nulls and empties so the snapshot matches what the schema re-serializes.
|
||||
|
||||
Upstream does ship the occasional null inside a string array (sarvam-*), which the strict
|
||||
Rust schema would reject; stripping here keeps the rest of that entry usable.
|
||||
"""
|
||||
if isinstance(value, dict):
|
||||
out = {k: clean(v) for k, v in value.items() if v is not None}
|
||||
return {k: v for k, v in out.items() if v not in ({}, [], None)}
|
||||
if isinstance(value, list):
|
||||
return [clean(v) for v in value if v is not None]
|
||||
return value
|
||||
|
||||
def project_model(model):
|
||||
out = {}
|
||||
for field in MODEL_FIELDS:
|
||||
value = model.get(field)
|
||||
if value in (None, {}, []):
|
||||
continue
|
||||
if field == "cost":
|
||||
value = {k: value[k] for k in COST_FIELDS if value.get(k) is not None}
|
||||
elif field == "modalities":
|
||||
# Only input modalities are read; output is always ["text"] for chat models.
|
||||
value = {k: value[k] for k in ("input",) if value.get(k)}
|
||||
elif field == "reasoning_options":
|
||||
value = [clean(option) for option in value]
|
||||
out[field] = clean(value)
|
||||
return {k: v for k, v in out.items() if v not in ({}, [], None)}
|
||||
|
||||
source = json.load(open(sys.argv[1]))
|
||||
snapshot = {}
|
||||
for provider_id, provider in sorted(source.items()):
|
||||
models = {
|
||||
model_id: project_model(model)
|
||||
for model_id, model in sorted(provider.get("models", {}).items())
|
||||
}
|
||||
if not models:
|
||||
continue
|
||||
block = {"models": models}
|
||||
if provider.get("name"):
|
||||
block["name"] = provider["name"]
|
||||
snapshot[provider_id] = block
|
||||
|
||||
json.dump(snapshot, open(sys.argv[2], "w"), separators=(",", ":"), sort_keys=True)
|
||||
print(
|
||||
f"projected {len(snapshot)} providers / "
|
||||
f"{sum(len(p['models']) for p in snapshot.values())} models",
|
||||
file=sys.stderr,
|
||||
)
|
||||
PY
|
||||
|
||||
# -n so the gzip header carries no name/mtime: identical input yields identical bytes, which
|
||||
# keeps the weekly CI job from opening a PR that changes nothing.
|
||||
gzip -9 -n -c "$TMP/snapshot.json" >"$TMP/snapshot.json.gz"
|
||||
|
||||
mkdir -p "$(dirname "$OUT")"
|
||||
if [[ -f "$OUT" ]] && cmp -s "$TMP/snapshot.json.gz" "$OUT"; then
|
||||
echo "snapshot unchanged ($(wc -c <"$OUT") bytes)"
|
||||
exit 0
|
||||
fi
|
||||
mv "$TMP/snapshot.json.gz" "$OUT"
|
||||
echo "wrote $OUT ($(wc -c <"$OUT") bytes)"
|
||||
Reference in New Issue
Block a user