- luxe starts on the model you last chose in this project, then the last one anywhere. Signing in opens the model picker instead of choosing for you. - Every list in a modal takes the mouse wheel and a draggable scrollbar. - The phone app shows reasoning in full, stays put while you read back through a long session, and shows a question's answer on its card. - On Windows, luxe update replaces the running luxe, and a signed-in install no longer sees setup again. - The file tree no longer shows "engine busy" in a large project, loop output is easier to follow, and the name is written lowercase. The CHANGELOG has the full list.
448 lines
18 KiB
Python
Executable File
448 lines
18 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""glyph-survey — ask real font files which characters the fallback tiers may use.
|
|
|
|
`glyphs.rs` has two tiers that promise portability, and for a long time nothing checked the promise
|
|
against a font. The tests asserted no private-use codepoints and one cell wide by the Unicode width
|
|
property; neither is a question about what the reader's font contains. Measured, Liberation Mono was
|
|
missing twenty of the thirty-one marks the TUI draws at every tier — including the geometric shapes
|
|
chosen *because* they were supposed to be safe.
|
|
|
|
This is the measurement behind `crates/luxe-tui/src/glyphs/portable.txt`. It reads the installed
|
|
files with fontTools rather than trusting a vendor's claim, and it refuses to run rather than
|
|
quietly survey eight faces when it was asked for nine — a panel with a hole in it produces a list
|
|
that is wrong in the one direction that matters.
|
|
|
|
scripts/glyph-survey.py # report coverage for the current vocabulary
|
|
scripts/glyph-survey.py --candidates # which of a wider palette every face has
|
|
scripts/glyph-survey.py --check # portable.txt still holds (CI)
|
|
scripts/glyph-survey.py --face FILE... # does a face draw everything luxe does, cleanly
|
|
|
|
U+2500-U+259F is not surveyed: terminals synthesize that range against the cell box, so the font is
|
|
not in the path. Everything else has to be in every face on the panel.
|
|
|
|
`--face` asks the opposite question, of a font you might *recommend* rather than one a reader
|
|
already has: does it carry every character luxe can put on screen at the nerd tier, and does each
|
|
one stay inside its cell? The second half is the one a coverage count misses. The non-`Mono` cut of
|
|
any Nerd Font has every icon and draws most of them twice the cell wide, and plain Iosevka has every
|
|
character while `→ … —` print over their neighbor. The character set is read out of the TUI's
|
|
string literals (test code stripped) and out of what the site's assets and captures draw, so it
|
|
follows the code instead of a list somebody has to remember to update. It takes font *files*, not
|
|
family names, for the same reason the panel does.
|
|
|
|
Requires fontTools (`pip install fonttools`). Point PANEL entries at real files with FONT_DIRS or
|
|
the `LUXE_FONT_DIRS` environment variable (colon-separated).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import sys
|
|
import unicodedata
|
|
|
|
ROOT = pathlib.Path(__file__).resolve().parent.parent
|
|
PORTABLE = ROOT / "crates" / "luxe-tui" / "src" / "glyphs" / "portable.txt"
|
|
|
|
# Where `--face` reads what luxe draws from.
|
|
DRAWING_SOURCES = (ROOT / "crates" / "luxe-tui" / "src", ROOT / "crates" / "luxe-cli" / "src")
|
|
DRAWING_ASSETS = (
|
|
ROOT / "web" / "src" / "assets" / "generated",
|
|
ROOT / "web" / "src" / "assets" / "captures",
|
|
)
|
|
|
|
# Sources whose literals reach the terminal without ever reaching a font.
|
|
NOT_GLYPHS = {
|
|
# kitty's Unicode placeholder and its row/column diacritics address an image; the terminal
|
|
# replaces them with pixels
|
|
ROOT / "crates" / "luxe-tui" / "src" / "graphics" / "kitty.rs",
|
|
}
|
|
|
|
# How far ink may pass the advance before it lands on the next cell. Outlines routinely overshoot
|
|
# by a few units for optical balance; the failures this exists for overshoot by 50 to 100%.
|
|
INK_TOLERANCE = 0.15
|
|
|
|
# The faces a reader plausibly already has. Filenames rather than family names: fontconfig
|
|
# substitutes silently, and a control run that quietly became the experimental run is exactly how
|
|
# the wordmark probe produced confident wrong answers the first time.
|
|
PANEL: dict[str, tuple[str, ...]] = {
|
|
"Liberation Mono": ("LiberationMono-Regular.ttf",),
|
|
"DejaVu Sans Mono": ("DejaVuSansMono.ttf",),
|
|
"Noto Sans Mono": ("NotoSansMono-Regular.ttf", "NotoSansMono[wdth,wght].ttf"),
|
|
"Cascadia Mono": ("CascadiaMono.ttf", "CascadiaMono-Regular.otf"),
|
|
"Adwaita Mono": ("AdwaitaMono-Regular.ttf",),
|
|
"Hack": ("Hack-Regular.ttf",),
|
|
"Fira Code": ("FiraCode-Regular.ttf", "FiraCode[wght].ttf"),
|
|
"JetBrains Mono": ("JetBrainsMono-Regular.ttf", "JetBrainsMono[wght].ttf"),
|
|
"Source Code Pro": ("SourceCodePro-Regular.ttf", "SourceCodePro-Regular.otf"),
|
|
}
|
|
|
|
FONT_DIRS = [
|
|
"/usr/share/fonts",
|
|
"/usr/local/share/fonts",
|
|
os.path.expanduser("~/.local/share/fonts"),
|
|
os.path.expanduser("~/Library/Fonts"),
|
|
"/Library/Fonts",
|
|
"/System/Library/Fonts",
|
|
"C:/Windows/Fonts",
|
|
]
|
|
|
|
# A wider palette to measure when looking for a substitute, so the choice is made from numbers.
|
|
CANDIDATES = (
|
|
"·•…‹›«»†‡¶§¤¬°±×÷√∞≈≡≤≥∙–—←↑→↓"
|
|
"□■▪▫▲▼◄►○●◆◇◉◌◦△▽"
|
|
"✓✔✗✖★☆↳↺⊘⋯ℹ⦸❯❮▸▾‣⁃∴∵⌐‾"
|
|
)
|
|
|
|
|
|
def synthesized(cp: int) -> bool:
|
|
"""Whether the terminal draws this itself, which takes the font out of the path."""
|
|
return 0x2500 <= cp <= 0x259F
|
|
|
|
|
|
def find_font(names: tuple[str, ...]) -> pathlib.Path | None:
|
|
roots = [pathlib.Path(d) for d in os.environ.get("LUXE_FONT_DIRS", "").split(":") if d]
|
|
roots += [pathlib.Path(d) for d in FONT_DIRS]
|
|
for root in roots:
|
|
if not root.is_dir():
|
|
continue
|
|
for name in names:
|
|
hit = next(root.rglob(name), None)
|
|
if hit:
|
|
return hit
|
|
return None
|
|
|
|
|
|
def coverage() -> dict[str, set[int]]:
|
|
try:
|
|
from fontTools.ttLib import TTFont
|
|
except ImportError:
|
|
sys.exit("glyph-survey needs fontTools: pip install fonttools")
|
|
|
|
found: dict[str, set[int]] = {}
|
|
missing: list[str] = []
|
|
for face, names in PANEL.items():
|
|
path = find_font(names)
|
|
if path is None:
|
|
missing.append(f"{face} ({' or '.join(names)})")
|
|
continue
|
|
font = TTFont(path, fontNumber=0, lazy=True)
|
|
chars: set[int] = set()
|
|
for table in font["cmap"].tables:
|
|
chars |= set(table.cmap.keys())
|
|
font.close()
|
|
found[face] = chars
|
|
if missing:
|
|
sys.exit(
|
|
"the panel is incomplete, so the survey would understate what is missing.\n"
|
|
" absent: " + "\n ".join(missing) + "\n"
|
|
" install them, or point LUXE_FONT_DIRS at a directory that has them."
|
|
)
|
|
return found
|
|
|
|
|
|
def parse_portable() -> list[int]:
|
|
listed = []
|
|
for line in PORTABLE.read_text(encoding="utf-8").splitlines():
|
|
match = re.match(r"^U\+([0-9A-F]{4,6})\t", line)
|
|
if match:
|
|
listed.append(int(match.group(1), 16))
|
|
return listed
|
|
|
|
|
|
# === What luxe draws, for --face ===
|
|
|
|
|
|
def rust_literals(src: str) -> tuple[list[tuple[int, int]], str]:
|
|
"""Every string and char literal's span, and the source with literals and comments blanked.
|
|
|
|
The blanked copy is what braces are matched in, so a `{` inside a format string or a comment
|
|
cannot unbalance the item it sits in. A lexer and not a regex because Rust's lifetimes look
|
|
like the start of a char literal and raw strings nest their own quotes.
|
|
"""
|
|
spans: list[tuple[int, int]] = []
|
|
masked = list(src)
|
|
i, n = 0, len(src)
|
|
|
|
def blank(start: int, end: int) -> None:
|
|
for k in range(start, end):
|
|
if masked[k] != "\n":
|
|
masked[k] = " "
|
|
|
|
while i < n:
|
|
c = src[i]
|
|
if src.startswith("//", i):
|
|
end = src.find("\n", i)
|
|
end = n if end < 0 else end
|
|
blank(i, end)
|
|
i = end
|
|
elif src.startswith("/*", i):
|
|
depth, j = 1, i + 2
|
|
while j < n and depth:
|
|
if src.startswith("/*", j):
|
|
depth, j = depth + 1, j + 2
|
|
elif src.startswith("*/", j):
|
|
depth, j = depth - 1, j + 2
|
|
else:
|
|
j += 1
|
|
blank(i, j)
|
|
i = j
|
|
elif c == "r" and re.match(r'r#*"', src[i : i + 260]) and not identifier_tail(src, i):
|
|
hashes = re.match(r"r(#*)", src[i:]).group(1)
|
|
start = i + 2 + len(hashes)
|
|
end = src.find('"' + hashes, start)
|
|
end = n if end < 0 else end
|
|
spans.append((start, end))
|
|
blank(start, end)
|
|
i = end + 1 + len(hashes)
|
|
elif c == '"':
|
|
j = i + 1
|
|
while j < n and src[j] != '"':
|
|
j += 2 if src[j] == "\\" else 1
|
|
spans.append((i + 1, j))
|
|
blank(i + 1, j)
|
|
i = j + 1
|
|
elif c == "'" and i + 2 < n and (src[i + 1] == "\\" or src[i + 2] == "'"):
|
|
# an escape is two characters or more, so `'\''` cannot close on itself
|
|
end = src.find("'", i + 3 if src[i + 1] == "\\" else i + 2)
|
|
end = n if end < 0 else end
|
|
spans.append((i + 1, end))
|
|
blank(i + 1, end)
|
|
i = end + 1
|
|
else:
|
|
i += 1
|
|
return spans, "".join(masked)
|
|
|
|
|
|
def test_ranges(masked: str) -> tuple[list[tuple[int, int]], list[str]]:
|
|
"""The spans of `#[cfg(test)]` items, and the names of test modules kept in their own file."""
|
|
ranges: list[tuple[int, int]] = []
|
|
outlined: list[str] = []
|
|
for match in re.finditer(r"#\[cfg\(test\)\]", masked):
|
|
body = re.compile(r"[{;]").search(masked, match.end())
|
|
if body is None:
|
|
continue
|
|
if body.group() == ";":
|
|
declared = re.search(r"\bmod\s+(\w+)\s*$", masked[match.end() : body.start()])
|
|
if declared:
|
|
outlined.append(declared.group(1))
|
|
continue
|
|
depth, j = 0, body.start()
|
|
while j < len(masked):
|
|
if masked[j] == "{":
|
|
depth += 1
|
|
elif masked[j] == "}":
|
|
depth -= 1
|
|
if depth == 0:
|
|
break
|
|
j += 1
|
|
ranges.append((match.start(), j))
|
|
return ranges, outlined
|
|
|
|
|
|
def source_codepoints() -> set[int]:
|
|
"""Non-ASCII characters in the TUI's string literals, outside test code.
|
|
|
|
Errs toward including: a literal that never reaches the screen costs one extra row in the
|
|
report, while a missed one is a character the recommended face was never asked about.
|
|
"""
|
|
parsed = {}
|
|
skipped: set[pathlib.Path] = set(NOT_GLYPHS)
|
|
for path in (p for root in DRAWING_SOURCES for p in root.rglob("*.rs")):
|
|
src = path.read_text(encoding="utf-8")
|
|
spans, masked = rust_literals(src)
|
|
ranges, outlined = test_ranges(masked)
|
|
parsed[path] = (src, spans, ranges)
|
|
# `foo.rs` keeps its children in `foo/`; `lib.rs`, `main.rs` and `mod.rs` beside themselves
|
|
home = path.parent if path.stem in ("lib", "main", "mod") else path.parent / path.stem
|
|
for name in outlined:
|
|
skipped.update({home / f"{name}.rs", home / name})
|
|
|
|
# a second pass, because a directory's files can be walked before the file declaring them tests
|
|
used: set[int] = set()
|
|
for path, (src, spans, ranges) in parsed.items():
|
|
if any(path == s or s in path.parents for s in skipped):
|
|
continue
|
|
for start, end in spans:
|
|
if any(lo <= start <= hi for lo, hi in ranges):
|
|
continue
|
|
literal = src[start:end]
|
|
used.update(ord(c) for c in literal if ord(c) > 0x7F)
|
|
used.update(int(h, 16) for h in re.findall(r"\\u\{([0-9A-Fa-f]{1,6})\}", literal))
|
|
return {cp for cp in used if cp > 0x7F}
|
|
|
|
|
|
def asset_codepoints() -> set[int]:
|
|
"""Non-ASCII characters the site's assets and captures draw, as site-fonts.sh reads them."""
|
|
import json
|
|
|
|
used: set[str] = set()
|
|
for directory in DRAWING_ASSETS:
|
|
for path in directory.glob("*.svg"):
|
|
body = re.sub(r"<[^>]+>", "", path.read_text(encoding="utf-8"))
|
|
used.update(body.replace("<", "<").replace(">", ">").replace("&", "&"))
|
|
for path in directory.glob("*.cast"):
|
|
for line in path.read_text(encoding="utf-8").splitlines():
|
|
try:
|
|
event = json.loads(line)
|
|
except ValueError:
|
|
continue
|
|
if isinstance(event, list) and len(event) == 3 and isinstance(event[2], str):
|
|
used.update(event[2])
|
|
return {ord(c) for c in used if ord(c) > 0x7F}
|
|
|
|
|
|
def identifier_tail(src: str, i: int) -> bool:
|
|
"""Whether position `i` continues an identifier, which makes an `r` there not a raw string."""
|
|
return i > 0 and (src[i - 1].isalnum() or src[i - 1] == "_")
|
|
|
|
|
|
def private_use(cp: int) -> bool:
|
|
return 0xE000 <= cp <= 0xF8FF or 0xF0000 <= cp <= 0xFFFFD
|
|
|
|
|
|
def braille(cp: int) -> bool:
|
|
return 0x2800 <= cp <= 0x28FF
|
|
|
|
|
|
def survey_faces(paths: list[str]) -> int:
|
|
"""Report each face against everything luxe draws. Non-zero if any misses or spills anything."""
|
|
try:
|
|
from fontTools.pens.boundsPen import BoundsPen
|
|
from fontTools.ttLib import TTFont
|
|
except ImportError:
|
|
sys.exit("glyph-survey needs fontTools: pip install fonttools")
|
|
|
|
drawn = source_codepoints() | asset_codepoints()
|
|
classes = {
|
|
"icons": [cp for cp in drawn if private_use(cp)],
|
|
"box and block": [cp for cp in drawn if synthesized(cp)],
|
|
"braille": [cp for cp in drawn if braille(cp)],
|
|
}
|
|
classes["marks"] = [cp for cp in drawn if not any(cp in group for group in classes.values())]
|
|
print(
|
|
f"luxe draws {len(drawn)} non-ASCII characters: "
|
|
+ ", ".join(f"{len(group)} {name}" for name, group in classes.items())
|
|
+ "\n",
|
|
file=sys.stderr,
|
|
)
|
|
|
|
failed = False
|
|
for raw in paths:
|
|
path = pathlib.Path(raw)
|
|
if not path.is_file():
|
|
print(f"{raw}: no such file")
|
|
failed = True
|
|
continue
|
|
font = TTFont(path, fontNumber=0)
|
|
cmap = font.getBestCmap()
|
|
glyphs = font.getGlyphSet()
|
|
# a symbols-only face has no letters to take the cell from; its space is the next best thing
|
|
cell_glyph = cmap.get(ord("M")) or cmap.get(ord(" "))
|
|
if cell_glyph is None:
|
|
print(f"{path.name}: no letter or space to measure a cell from")
|
|
failed = True
|
|
continue
|
|
advance = font["hmtx"][cell_glyph][0]
|
|
em = font["head"].unitsPerEm
|
|
|
|
def spills(cp: int) -> bool:
|
|
pen = BoundsPen(glyphs)
|
|
glyphs[cmap[cp]].draw(pen)
|
|
if pen.bounds is None:
|
|
return False
|
|
left, _, right, _ = pen.bounds
|
|
return right > advance * (1 + INK_TOLERANCE) or left < -advance * INK_TOLERANCE
|
|
|
|
missing = {name: [cp for cp in group if cp not in cmap] for name, group in classes.items()}
|
|
spilled = sorted(cp for cp in drawn if cp in cmap and not synthesized(cp) and spills(cp))
|
|
absent = sum(len(group) for group in missing.values())
|
|
print(f"{path.name}")
|
|
print(f" has {len(drawn) - absent}/{len(drawn)}, cell {advance / em:.2f} em")
|
|
for name, group in missing.items():
|
|
if group:
|
|
shown = "" if name == "icons" else " " + " ".join(chr(cp) for cp in sorted(group))
|
|
print(f" missing {name}: {len(group)}{shown}")
|
|
if spilled:
|
|
icons = [cp for cp in spilled if private_use(cp)]
|
|
text = [cp for cp in spilled if not private_use(cp)]
|
|
if icons:
|
|
print(f" {len(icons)} icons spill into the next cell (the non-Mono cut?)")
|
|
if text:
|
|
print(f" spills into the next cell: {' '.join(chr(cp) for cp in text)}")
|
|
# braille is the one class allowed to come from another font, so it cannot fail a face
|
|
if absent - len(missing["braille"]) or spilled:
|
|
failed = True
|
|
else:
|
|
print(" draws everything luxe does, each inside its cell")
|
|
font.close()
|
|
return 1 if failed else 0
|
|
|
|
|
|
def emit(cps: list[int]) -> None:
|
|
for cp in cps:
|
|
try:
|
|
name = unicodedata.name(chr(cp))
|
|
except ValueError:
|
|
name = "?"
|
|
print(f"U+{cp:04X}\t{chr(cp)}\t{name}")
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--check", action="store_true", help="verify portable.txt against the panel")
|
|
parser.add_argument("--candidates", action="store_true", help="survey the wider palette")
|
|
parser.add_argument(
|
|
"--face", nargs="+", metavar="FILE", help="measure font files against everything luxe draws"
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
if args.face:
|
|
return survey_faces(args.face)
|
|
|
|
cov = coverage()
|
|
panel = list(cov)
|
|
print(f"panel: {len(panel)} faces — {', '.join(panel)}\n", file=sys.stderr)
|
|
|
|
if args.candidates:
|
|
rows = []
|
|
for ch in dict.fromkeys(CANDIDATES):
|
|
if synthesized(ord(ch)):
|
|
continue
|
|
absent = [f for f in panel if ord(ch) not in cov[f]]
|
|
rows.append((len(absent), ch, absent))
|
|
for count, ch, absent in sorted(rows):
|
|
verdict = "PORTABLE" if not count else f"missing {count}/{len(panel)}: {', '.join(absent)}"
|
|
print(f" {ch} U+{ord(ch):04X} {verdict}")
|
|
return 0
|
|
|
|
listed = parse_portable()
|
|
broken = {}
|
|
for cp in listed:
|
|
absent = [f for f in panel if cp not in cov[f]]
|
|
if absent:
|
|
broken[cp] = absent
|
|
|
|
if args.check:
|
|
if broken:
|
|
print(f"{PORTABLE.relative_to(ROOT)} claims coverage the panel does not have:\n")
|
|
for cp, absent in broken.items():
|
|
print(f" U+{cp:04X} {chr(cp)} — absent from {', '.join(absent)}")
|
|
return 1
|
|
print(f"portable.txt — {len(listed)} characters, present in all {len(panel)} faces")
|
|
return 0
|
|
|
|
print(f"# {len(listed)} surveyed characters, present in all {len(panel)} faces")
|
|
emit(listed)
|
|
if broken:
|
|
print("\n# NOT portable, listed anyway:", file=sys.stderr)
|
|
for cp, absent in broken.items():
|
|
print(f"# U+{cp:04X} {chr(cp)} — {', '.join(absent)}", file=sys.stderr)
|
|
return 1
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|