Luxe 0.1.0: a coding agent harness for the terminal. See README.md to get started and CHANGELOG.md for releases.
448 lines
18 KiB
Python
Executable File
448 lines
18 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""glyph-survey — ask real font files which characters the fallback tiers may use.
|
|
|
|
`glyphs.rs` has two tiers that promise portability, and for a long time nothing checked the promise
|
|
against a font. The tests asserted no private-use codepoints and one cell wide by the Unicode width
|
|
property; neither is a question about what the reader's font contains. Measured, Liberation Mono was
|
|
missing twenty of the thirty-one marks the TUI draws at every tier — including the geometric shapes
|
|
chosen *because* they were supposed to be safe.
|
|
|
|
This is the measurement behind `crates/luxe-tui/src/glyphs/portable.txt`. It reads the installed
|
|
files with fontTools rather than trusting a vendor's claim, and it refuses to run rather than
|
|
quietly survey eight faces when it was asked for nine — a panel with a hole in it produces a list
|
|
that is wrong in the one direction that matters.
|
|
|
|
scripts/glyph-survey.py # report coverage for the current vocabulary
|
|
scripts/glyph-survey.py --candidates # which of a wider palette every face has
|
|
scripts/glyph-survey.py --check # portable.txt still holds (CI)
|
|
scripts/glyph-survey.py --face FILE... # does a face draw everything Luxe does, cleanly
|
|
|
|
U+2500-U+259F is not surveyed: terminals synthesize that range against the cell box, so the font is
|
|
not in the path. Everything else has to be in every face on the panel.
|
|
|
|
`--face` asks the opposite question, of a font you might *recommend* rather than one a reader
|
|
already has: does it carry every character Luxe can put on screen at the nerd tier, and does each
|
|
one stay inside its cell? The second half is the one a coverage count misses. The non-`Mono` cut of
|
|
any Nerd Font has every icon and draws most of them twice the cell wide, and plain Iosevka has every
|
|
character while `→ … —` print over their neighbor. The character set is read out of the TUI's
|
|
string literals (test code stripped) and out of what the site's assets and captures draw, so it
|
|
follows the code instead of a list somebody has to remember to update. It takes font *files*, not
|
|
family names, for the same reason the panel does.
|
|
|
|
Requires fontTools (`pip install fonttools`). Point PANEL entries at real files with FONT_DIRS or
|
|
the `LUXE_FONT_DIRS` environment variable (colon-separated).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import sys
|
|
import unicodedata
|
|
|
|
ROOT = pathlib.Path(__file__).resolve().parent.parent
|
|
PORTABLE = ROOT / "crates" / "luxe-tui" / "src" / "glyphs" / "portable.txt"
|
|
|
|
# Where `--face` reads what Luxe draws from.
|
|
DRAWING_SOURCES = (ROOT / "crates" / "luxe-tui" / "src", ROOT / "crates" / "luxe-cli" / "src")
|
|
DRAWING_ASSETS = (
|
|
ROOT / "web" / "src" / "assets" / "generated",
|
|
ROOT / "web" / "src" / "assets" / "captures",
|
|
)
|
|
|
|
# Sources whose literals reach the terminal without ever reaching a font.
|
|
NOT_GLYPHS = {
|
|
# kitty's Unicode placeholder and its row/column diacritics address an image; the terminal
|
|
# replaces them with pixels
|
|
ROOT / "crates" / "luxe-tui" / "src" / "graphics" / "kitty.rs",
|
|
}
|
|
|
|
# How far ink may pass the advance before it lands on the next cell. Outlines routinely overshoot
|
|
# by a few units for optical balance; the failures this exists for overshoot by 50 to 100%.
|
|
INK_TOLERANCE = 0.15
|
|
|
|
# The faces a reader plausibly already has. Filenames rather than family names: fontconfig
|
|
# substitutes silently, and a control run that quietly became the experimental run is exactly how
|
|
# the wordmark probe produced confident wrong answers the first time.
|
|
PANEL: dict[str, tuple[str, ...]] = {
|
|
"Liberation Mono": ("LiberationMono-Regular.ttf",),
|
|
"DejaVu Sans Mono": ("DejaVuSansMono.ttf",),
|
|
"Noto Sans Mono": ("NotoSansMono-Regular.ttf", "NotoSansMono[wdth,wght].ttf"),
|
|
"Cascadia Mono": ("CascadiaMono.ttf", "CascadiaMono-Regular.otf"),
|
|
"Adwaita Mono": ("AdwaitaMono-Regular.ttf",),
|
|
"Hack": ("Hack-Regular.ttf",),
|
|
"Fira Code": ("FiraCode-Regular.ttf", "FiraCode[wght].ttf"),
|
|
"JetBrains Mono": ("JetBrainsMono-Regular.ttf", "JetBrainsMono[wght].ttf"),
|
|
"Source Code Pro": ("SourceCodePro-Regular.ttf", "SourceCodePro-Regular.otf"),
|
|
}
|
|
|
|
FONT_DIRS = [
|
|
"/usr/share/fonts",
|
|
"/usr/local/share/fonts",
|
|
os.path.expanduser("~/.local/share/fonts"),
|
|
os.path.expanduser("~/Library/Fonts"),
|
|
"/Library/Fonts",
|
|
"/System/Library/Fonts",
|
|
"C:/Windows/Fonts",
|
|
]
|
|
|
|
# A wider palette to measure when looking for a substitute, so the choice is made from numbers.
|
|
CANDIDATES = (
|
|
"·•…‹›«»†‡¶§¤¬°±×÷√∞≈≡≤≥∙–—←↑→↓"
|
|
"□■▪▫▲▼◄►○●◆◇◉◌◦△▽"
|
|
"✓✔✗✖★☆↳↺⊘⋯ℹ⦸❯❮▸▾‣⁃∴∵⌐‾"
|
|
)
|
|
|
|
|
|
def synthesized(cp: int) -> bool:
|
|
"""Whether the terminal draws this itself, which takes the font out of the path."""
|
|
return 0x2500 <= cp <= 0x259F
|
|
|
|
|
|
def find_font(names: tuple[str, ...]) -> pathlib.Path | None:
|
|
roots = [pathlib.Path(d) for d in os.environ.get("LUXE_FONT_DIRS", "").split(":") if d]
|
|
roots += [pathlib.Path(d) for d in FONT_DIRS]
|
|
for root in roots:
|
|
if not root.is_dir():
|
|
continue
|
|
for name in names:
|
|
hit = next(root.rglob(name), None)
|
|
if hit:
|
|
return hit
|
|
return None
|
|
|
|
|
|
def coverage() -> dict[str, set[int]]:
|
|
try:
|
|
from fontTools.ttLib import TTFont
|
|
except ImportError:
|
|
sys.exit("glyph-survey needs fontTools: pip install fonttools")
|
|
|
|
found: dict[str, set[int]] = {}
|
|
missing: list[str] = []
|
|
for face, names in PANEL.items():
|
|
path = find_font(names)
|
|
if path is None:
|
|
missing.append(f"{face} ({' or '.join(names)})")
|
|
continue
|
|
font = TTFont(path, fontNumber=0, lazy=True)
|
|
chars: set[int] = set()
|
|
for table in font["cmap"].tables:
|
|
chars |= set(table.cmap.keys())
|
|
font.close()
|
|
found[face] = chars
|
|
if missing:
|
|
sys.exit(
|
|
"the panel is incomplete, so the survey would understate what is missing.\n"
|
|
" absent: " + "\n ".join(missing) + "\n"
|
|
" install them, or point LUXE_FONT_DIRS at a directory that has them."
|
|
)
|
|
return found
|
|
|
|
|
|
def parse_portable() -> list[int]:
|
|
listed = []
|
|
for line in PORTABLE.read_text(encoding="utf-8").splitlines():
|
|
match = re.match(r"^U\+([0-9A-F]{4,6})\t", line)
|
|
if match:
|
|
listed.append(int(match.group(1), 16))
|
|
return listed
|
|
|
|
|
|
# === What Luxe draws, for --face ===
|
|
|
|
|
|
def rust_literals(src: str) -> tuple[list[tuple[int, int]], str]:
|
|
"""Every string and char literal's span, and the source with literals and comments blanked.
|
|
|
|
The blanked copy is what braces are matched in, so a `{` inside a format string or a comment
|
|
cannot unbalance the item it sits in. A lexer and not a regex because Rust's lifetimes look
|
|
like the start of a char literal and raw strings nest their own quotes.
|
|
"""
|
|
spans: list[tuple[int, int]] = []
|
|
masked = list(src)
|
|
i, n = 0, len(src)
|
|
|
|
def blank(start: int, end: int) -> None:
|
|
for k in range(start, end):
|
|
if masked[k] != "\n":
|
|
masked[k] = " "
|
|
|
|
while i < n:
|
|
c = src[i]
|
|
if src.startswith("//", i):
|
|
end = src.find("\n", i)
|
|
end = n if end < 0 else end
|
|
blank(i, end)
|
|
i = end
|
|
elif src.startswith("/*", i):
|
|
depth, j = 1, i + 2
|
|
while j < n and depth:
|
|
if src.startswith("/*", j):
|
|
depth, j = depth + 1, j + 2
|
|
elif src.startswith("*/", j):
|
|
depth, j = depth - 1, j + 2
|
|
else:
|
|
j += 1
|
|
blank(i, j)
|
|
i = j
|
|
elif c == "r" and re.match(r'r#*"', src[i : i + 260]) and not identifier_tail(src, i):
|
|
hashes = re.match(r"r(#*)", src[i:]).group(1)
|
|
start = i + 2 + len(hashes)
|
|
end = src.find('"' + hashes, start)
|
|
end = n if end < 0 else end
|
|
spans.append((start, end))
|
|
blank(start, end)
|
|
i = end + 1 + len(hashes)
|
|
elif c == '"':
|
|
j = i + 1
|
|
while j < n and src[j] != '"':
|
|
j += 2 if src[j] == "\\" else 1
|
|
spans.append((i + 1, j))
|
|
blank(i + 1, j)
|
|
i = j + 1
|
|
elif c == "'" and i + 2 < n and (src[i + 1] == "\\" or src[i + 2] == "'"):
|
|
# an escape is two characters or more, so `'\''` cannot close on itself
|
|
end = src.find("'", i + 3 if src[i + 1] == "\\" else i + 2)
|
|
end = n if end < 0 else end
|
|
spans.append((i + 1, end))
|
|
blank(i + 1, end)
|
|
i = end + 1
|
|
else:
|
|
i += 1
|
|
return spans, "".join(masked)
|
|
|
|
|
|
def test_ranges(masked: str) -> tuple[list[tuple[int, int]], list[str]]:
|
|
"""The spans of `#[cfg(test)]` items, and the names of test modules kept in their own file."""
|
|
ranges: list[tuple[int, int]] = []
|
|
outlined: list[str] = []
|
|
for match in re.finditer(r"#\[cfg\(test\)\]", masked):
|
|
body = re.compile(r"[{;]").search(masked, match.end())
|
|
if body is None:
|
|
continue
|
|
if body.group() == ";":
|
|
declared = re.search(r"\bmod\s+(\w+)\s*$", masked[match.end() : body.start()])
|
|
if declared:
|
|
outlined.append(declared.group(1))
|
|
continue
|
|
depth, j = 0, body.start()
|
|
while j < len(masked):
|
|
if masked[j] == "{":
|
|
depth += 1
|
|
elif masked[j] == "}":
|
|
depth -= 1
|
|
if depth == 0:
|
|
break
|
|
j += 1
|
|
ranges.append((match.start(), j))
|
|
return ranges, outlined
|
|
|
|
|
|
def source_codepoints() -> set[int]:
|
|
"""Non-ASCII characters in the TUI's string literals, outside test code.
|
|
|
|
Errs toward including: a literal that never reaches the screen costs one extra row in the
|
|
report, while a missed one is a character the recommended face was never asked about.
|
|
"""
|
|
parsed = {}
|
|
skipped: set[pathlib.Path] = set(NOT_GLYPHS)
|
|
for path in (p for root in DRAWING_SOURCES for p in root.rglob("*.rs")):
|
|
src = path.read_text(encoding="utf-8")
|
|
spans, masked = rust_literals(src)
|
|
ranges, outlined = test_ranges(masked)
|
|
parsed[path] = (src, spans, ranges)
|
|
# `foo.rs` keeps its children in `foo/`; `lib.rs`, `main.rs` and `mod.rs` beside themselves
|
|
home = path.parent if path.stem in ("lib", "main", "mod") else path.parent / path.stem
|
|
for name in outlined:
|
|
skipped.update({home / f"{name}.rs", home / name})
|
|
|
|
# a second pass, because a directory's files can be walked before the file declaring them tests
|
|
used: set[int] = set()
|
|
for path, (src, spans, ranges) in parsed.items():
|
|
if any(path == s or s in path.parents for s in skipped):
|
|
continue
|
|
for start, end in spans:
|
|
if any(lo <= start <= hi for lo, hi in ranges):
|
|
continue
|
|
literal = src[start:end]
|
|
used.update(ord(c) for c in literal if ord(c) > 0x7F)
|
|
used.update(int(h, 16) for h in re.findall(r"\\u\{([0-9A-Fa-f]{1,6})\}", literal))
|
|
return {cp for cp in used if cp > 0x7F}
|
|
|
|
|
|
def asset_codepoints() -> set[int]:
|
|
"""Non-ASCII characters the site's assets and captures draw, as site-fonts.sh reads them."""
|
|
import json
|
|
|
|
used: set[str] = set()
|
|
for directory in DRAWING_ASSETS:
|
|
for path in directory.glob("*.svg"):
|
|
body = re.sub(r"<[^>]+>", "", path.read_text(encoding="utf-8"))
|
|
used.update(body.replace("<", "<").replace(">", ">").replace("&", "&"))
|
|
for path in directory.glob("*.cast"):
|
|
for line in path.read_text(encoding="utf-8").splitlines():
|
|
try:
|
|
event = json.loads(line)
|
|
except ValueError:
|
|
continue
|
|
if isinstance(event, list) and len(event) == 3 and isinstance(event[2], str):
|
|
used.update(event[2])
|
|
return {ord(c) for c in used if ord(c) > 0x7F}
|
|
|
|
|
|
def identifier_tail(src: str, i: int) -> bool:
|
|
"""Whether position `i` continues an identifier, which makes an `r` there not a raw string."""
|
|
return i > 0 and (src[i - 1].isalnum() or src[i - 1] == "_")
|
|
|
|
|
|
def private_use(cp: int) -> bool:
|
|
return 0xE000 <= cp <= 0xF8FF or 0xF0000 <= cp <= 0xFFFFD
|
|
|
|
|
|
def braille(cp: int) -> bool:
|
|
return 0x2800 <= cp <= 0x28FF
|
|
|
|
|
|
def survey_faces(paths: list[str]) -> int:
|
|
"""Report each face against everything Luxe draws. Non-zero if any misses or spills anything."""
|
|
try:
|
|
from fontTools.pens.boundsPen import BoundsPen
|
|
from fontTools.ttLib import TTFont
|
|
except ImportError:
|
|
sys.exit("glyph-survey needs fontTools: pip install fonttools")
|
|
|
|
drawn = source_codepoints() | asset_codepoints()
|
|
classes = {
|
|
"icons": [cp for cp in drawn if private_use(cp)],
|
|
"box and block": [cp for cp in drawn if synthesized(cp)],
|
|
"braille": [cp for cp in drawn if braille(cp)],
|
|
}
|
|
classes["marks"] = [cp for cp in drawn if not any(cp in group for group in classes.values())]
|
|
print(
|
|
f"Luxe draws {len(drawn)} non-ASCII characters: "
|
|
+ ", ".join(f"{len(group)} {name}" for name, group in classes.items())
|
|
+ "\n",
|
|
file=sys.stderr,
|
|
)
|
|
|
|
failed = False
|
|
for raw in paths:
|
|
path = pathlib.Path(raw)
|
|
if not path.is_file():
|
|
print(f"{raw}: no such file")
|
|
failed = True
|
|
continue
|
|
font = TTFont(path, fontNumber=0)
|
|
cmap = font.getBestCmap()
|
|
glyphs = font.getGlyphSet()
|
|
# a symbols-only face has no letters to take the cell from; its space is the next best thing
|
|
cell_glyph = cmap.get(ord("M")) or cmap.get(ord(" "))
|
|
if cell_glyph is None:
|
|
print(f"{path.name}: no letter or space to measure a cell from")
|
|
failed = True
|
|
continue
|
|
advance = font["hmtx"][cell_glyph][0]
|
|
em = font["head"].unitsPerEm
|
|
|
|
def spills(cp: int) -> bool:
|
|
pen = BoundsPen(glyphs)
|
|
glyphs[cmap[cp]].draw(pen)
|
|
if pen.bounds is None:
|
|
return False
|
|
left, _, right, _ = pen.bounds
|
|
return right > advance * (1 + INK_TOLERANCE) or left < -advance * INK_TOLERANCE
|
|
|
|
missing = {name: [cp for cp in group if cp not in cmap] for name, group in classes.items()}
|
|
spilled = sorted(cp for cp in drawn if cp in cmap and not synthesized(cp) and spills(cp))
|
|
absent = sum(len(group) for group in missing.values())
|
|
print(f"{path.name}")
|
|
print(f" has {len(drawn) - absent}/{len(drawn)}, cell {advance / em:.2f} em")
|
|
for name, group in missing.items():
|
|
if group:
|
|
shown = "" if name == "icons" else " " + " ".join(chr(cp) for cp in sorted(group))
|
|
print(f" missing {name}: {len(group)}{shown}")
|
|
if spilled:
|
|
icons = [cp for cp in spilled if private_use(cp)]
|
|
text = [cp for cp in spilled if not private_use(cp)]
|
|
if icons:
|
|
print(f" {len(icons)} icons spill into the next cell (the non-Mono cut?)")
|
|
if text:
|
|
print(f" spills into the next cell: {' '.join(chr(cp) for cp in text)}")
|
|
# braille is the one class allowed to come from another font, so it cannot fail a face
|
|
if absent - len(missing["braille"]) or spilled:
|
|
failed = True
|
|
else:
|
|
print(" draws everything Luxe does, each inside its cell")
|
|
font.close()
|
|
return 1 if failed else 0
|
|
|
|
|
|
def emit(cps: list[int]) -> None:
|
|
for cp in cps:
|
|
try:
|
|
name = unicodedata.name(chr(cp))
|
|
except ValueError:
|
|
name = "?"
|
|
print(f"U+{cp:04X}\t{chr(cp)}\t{name}")
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--check", action="store_true", help="verify portable.txt against the panel")
|
|
parser.add_argument("--candidates", action="store_true", help="survey the wider palette")
|
|
parser.add_argument(
|
|
"--face", nargs="+", metavar="FILE", help="measure font files against everything Luxe draws"
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
if args.face:
|
|
return survey_faces(args.face)
|
|
|
|
cov = coverage()
|
|
panel = list(cov)
|
|
print(f"panel: {len(panel)} faces — {', '.join(panel)}\n", file=sys.stderr)
|
|
|
|
if args.candidates:
|
|
rows = []
|
|
for ch in dict.fromkeys(CANDIDATES):
|
|
if synthesized(ord(ch)):
|
|
continue
|
|
absent = [f for f in panel if ord(ch) not in cov[f]]
|
|
rows.append((len(absent), ch, absent))
|
|
for count, ch, absent in sorted(rows):
|
|
verdict = "PORTABLE" if not count else f"missing {count}/{len(panel)}: {', '.join(absent)}"
|
|
print(f" {ch} U+{ord(ch):04X} {verdict}")
|
|
return 0
|
|
|
|
listed = parse_portable()
|
|
broken = {}
|
|
for cp in listed:
|
|
absent = [f for f in panel if cp not in cov[f]]
|
|
if absent:
|
|
broken[cp] = absent
|
|
|
|
if args.check:
|
|
if broken:
|
|
print(f"{PORTABLE.relative_to(ROOT)} claims coverage the panel does not have:\n")
|
|
for cp, absent in broken.items():
|
|
print(f" U+{cp:04X} {chr(cp)} — absent from {', '.join(absent)}")
|
|
return 1
|
|
print(f"portable.txt — {len(listed)} characters, present in all {len(panel)} faces")
|
|
return 0
|
|
|
|
print(f"# {len(listed)} surveyed characters, present in all {len(panel)} faces")
|
|
emit(listed)
|
|
if broken:
|
|
print("\n# NOT portable, listed anyway:", file=sys.stderr)
|
|
for cp, absent in broken.items():
|
|
print(f"# U+{cp:04X} {chr(cp)} — {', '.join(absent)}", file=sys.stderr)
|
|
return 1
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|