cottongin/immich-frame
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: fit-contain letterbox + smart queue refill

Browse Source 
- Replace center-crop with fit-contain (no photo cropping)
- Letterbox bars filled with adjacent edge average color
- Queue fetches in batches until 50 usable assets
- Auto-refill queue on exhaustion, dedup via shown history
- Filter RAW files from asset selection

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
cottongin
2026-08-03 14:39:55 -04:00
parent 244d589f10
commit e4fbbdb95f
10 changed files with 985 additions and 411 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
data/app.js
View File

@@ -162,14 +162,53 @@ async function renderDisplay(el) {
}
async function renderDevice(el) {
const settings = await api('/api/settings');
el.innerHTML = `
<div class="card">
<h2>Device</h2>
<h2>Immich Connection</h2>
<div class="field">
<label>Immich URL</label>
<input type="url" id="immichUrl" value="${settings.immich_url || ''}" placeholder="https://photos.example.com">
</div>
<div class="field">
<label>API Key</label>
<input type="password" id="immichKey" value="${settings.immich_key || ''}" placeholder="Your Immich API key">
</div>
<div class="btn-group">
<button class="btn btn-primary" id="saveImmich">Save</button>
<button class="btn" id="testConn">Test Connection</button>
</div>
<div id="connStatus" style="margin-top:12px"></div>
</div>
<div class="card">
<h2>Device Actions</h2>
<div class="btn-group">
<button class="btn btn-danger" onclick="if(confirm('Enter deep sleep?'))api('/api/action/sleep',{method:'POST'})">Deep Sleep</button>
<button class="btn btn-danger" onclick="if(confirm('Reboot?'))fetch('/api/action/reboot',{method:'POST'})">Reboot</button>
<button class="btn btn-danger" onclick="if(confirm('Reboot device?'))fetch('/api/action/reboot',{method:'POST'})">Reboot</button>
</div>
</div>`;
document.getElementById('saveImmich').onclick = async () => {
const url = document.getElementById('immichUrl').value.trim();
const key = document.getElementById('immichKey').value.trim();
if (!url || !key) return toast('URL and API key are required');
await api('/api/settings', { method: 'POST', body: JSON.stringify({ immich_url: url, immich_key: key }) });
toast('Immich settings saved. Reboot to apply.');
};
document.getElementById('testConn').onclick = async () => {
const status = document.getElementById('connStatus');
status.textContent = 'Testing...';
status.style.color = '';
try {
const result = await api('/api/test-connection', { method: 'POST' });
status.textContent = result.msg;
status.style.color = result.ok ? '#2e7d32' : '#c62828';
} catch (e) {
status.textContent = 'Request failed: ' + e.message;
status.style.color = '#c62828';
}
};
}
async function renderFirmware(el) {

1
platformio.ini
View File

@@ -23,6 +23,7 @@ lib_deps =
mathieucarbou/ESP Async WebServer @ ^3.0.6
bblanchon/ArduinoJson @ ^7.0.0
ricmoo/QRCode @ ^0.0.1
bitbank2/JPEGDEC @ ^1.8.4
lib_ignore =
WebServer

871
src/image_pipeline.cpp
View File

@@ -1,248 +1,185 @@
#include "image_pipeline.h"
#include <M5GFX.h>
#include <lgfx/utility/lgfx_tjpgd.h>
#include <esp_heap_caps.h>
#include <cstring>
#include <cmath>
// Spectra 6 palette
static const uint8_t PALETTE_RGB[6][3] = {
{PALETTE_BLACK_R, PALETTE_BLACK_G, PALETTE_BLACK_B},
{PALETTE_WHITE_R, PALETTE_WHITE_G, PALETTE_WHITE_B},
{PALETTE_RED_R, PALETTE_RED_G, PALETTE_RED_B},
{PALETTE_GREEN_R, PALETTE_GREEN_G, PALETTE_GREEN_B},
{PALETTE_BLUE_R, PALETTE_BLUE_G, PALETTE_BLUE_B},
{PALETTE_YELLOW_R, PALETTE_YELLOW_G, PALETTE_YELLOW_B}
// Spectra 6 calibrated palette (measured display appearance from epdoptimize)
// These represent what the display ACTUALLY shows, used for dithering decisions
static const uint8_t PALETTE_CALIBRATED[6][3] = {
{0x1F, 0x22, 0x26}, // Black -> appears as dark gray
{0xB9, 0xC7, 0xC9}, // White -> appears as light gray-blue
{0x62, 0x20, 0x1E}, // Red -> appears as dark red/brown
{0x35, 0x56, 0x3A}, // Green -> appears as dark forest green
{0x23, 0x3F, 0x8E}, // Blue -> appears as dark navy
{0xC1, 0xBB, 0x1E} // Yellow -> appears as olive/mustard
};
struct JpegMemReader {
const uint8_t* data;
size_t size;
size_t pos;
};
// JPEGDEC draw callback: receives decoded MCU blocks and writes RGB888 to buffer
static int jpegDrawCallback(JPEGDRAW* pDraw) {
auto* ctx = static_cast<DecodeContext*>(pDraw->pUser);
if (ctx == nullptr || ctx->rgbBuffer == nullptr) return 0;
static uint32_t jpegMemInput(void* device, uint8_t* buf, uint32_t len) {
auto* ctx = static_cast<JpegMemReader*>(device);
if (ctx->pos >= ctx->size) {
return 0;
for (int y = 0; y < pDraw->iHeight; y++) {
int dstRow = pDraw->y + y;
if (dstRow < 0 || dstRow >= ctx->bufferHeight) continue;
for (int x = 0; x < pDraw->iWidthUsed; x++) {
int dstCol = pDraw->x + x;
if (dstCol < 0 || dstCol >= ctx->bufferWidth) continue;
// Convert RGB565 to RGB888
uint16_t pixel = pDraw->pPixels[y * pDraw->iWidth + x];
uint8_t r = (pixel >> 11) << 3;
uint8_t g = ((pixel >> 5) & 0x3F) << 2;
uint8_t b = (pixel & 0x1F) << 3;
size_t dstIdx = ((size_t)dstRow * ctx->bufferWidth + dstCol) * 3;
ctx->rgbBuffer[dstIdx] = r;
ctx->rgbBuffer[dstIdx + 1] = g;
ctx->rgbBuffer[dstIdx + 2] = b;
}
}
uint32_t remain = static_cast<uint32_t>(ctx->size - ctx->pos);
uint32_t n = len < remain ? len : remain;
memcpy(buf, ctx->data + ctx->pos, n);
ctx->pos += n;
return n;
return 1;
}
ProcessedImage ImagePipeline::process(uint8_t* jpegData, size_t jpegSize) {
ProcessedImage result = {nullptr, DISPLAY_WIDTH, DISPLAY_HEIGHT, false};
// Contrast/saturation enhancement applied before dithering
static void enhanceContrast(uint8_t* rgb, size_t pixelCount) {
static constexpr float CONTRAST = 1.25f;
static constexpr float SATURATION = 1.15f;
static constexpr float MID = 128.0f;
// Decode JPEG
uint16_t srcW, srcH;
uint8_t* rgb = decodeJpeg(jpegData, jpegSize, &srcW, &srcH);
if (rgb == nullptr) {
Serial.println("[pipeline] JPEG decode failed");
return result;
for (size_t i = 0; i < pixelCount; i++) {
size_t idx = i * 3;
float r = rgb[idx];
float g = rgb[idx + 1];
float b = rgb[idx + 2];
r = (r - MID) * CONTRAST + MID;
g = (g - MID) * CONTRAST + MID;
b = (b - MID) * CONTRAST + MID;
float lum = 0.299f * r + 0.587f * g + 0.114f * b;
r = lum + (r - lum) * SATURATION;
g = lum + (g - lum) * SATURATION;
b = lum + (b - lum) * SATURATION;
rgb[idx] = (uint8_t)fminf(fmaxf(r, 0.0f), 255.0f);
rgb[idx + 1] = (uint8_t)fminf(fmaxf(g, 0.0f), 255.0f);
rgb[idx + 2] = (uint8_t)fminf(fmaxf(b, 0.0f), 255.0f);
}
Serial.printf("[pipeline] Decoded: %dx%d\n", srcW, srcH);
// Calculate crop region (fill-crop to display aspect ratio)
uint16_t cropX, cropY, cropW, cropH;
centerCrop(rgb, srcW, srcH, DISPLAY_WIDTH, DISPLAY_HEIGHT,
&cropX, &cropY, &cropW, &cropH);
// Resize cropped region to display dimensions
uint8_t* cropped = (uint8_t*)ps_malloc(cropW * cropH * 3);
if (cropped == nullptr) {
free(rgb);
return result;
}
// Extract crop region
for (uint16_t y = 0; y < cropH; y++) {
memcpy(cropped + y * cropW * 3,
rgb + ((cropY + y) * srcW + cropX) * 3,
cropW * 3);
}
free(rgb);
// Resize to display dimensions
uint8_t* resized = resize(cropped, cropW, cropH, DISPLAY_WIDTH, DISPLAY_HEIGHT);
free(cropped);
if (resized == nullptr) {
return result;
}
// Dither to 6-color palette
uint8_t* dithered = dither(resized, DISPLAY_WIDTH, DISPLAY_HEIGHT);
free(resized);
if (dithered == nullptr) {
return result;
}
result.framebuffer = dithered;
result.valid = true;
Serial.println("[pipeline] Processing complete");
return result;
}
ProcessedImage ImagePipeline::processPortraitPair(uint8_t* jpeg1Data, size_t jpeg1Size,
uint8_t* jpeg2Data, size_t jpeg2Size) {
ProcessedImage result = {nullptr, DISPLAY_WIDTH, DISPLAY_HEIGHT, false};
// Average a single edge of the fitted image (4 rows or columns deep)
static constexpr int EDGE_DEPTH = 4;
// Each portrait gets half the width minus gap
uint16_t portraitW = (DISPLAY_WIDTH - PORTRAIT_GAP_PX) / 2;
uint16_t portraitH = DISPLAY_HEIGHT;
struct EdgeColor { uint8_t r, g, b; };
// Allocate combined RGB buffer
uint8_t* combined = (uint8_t*)ps_calloc(DISPLAY_WIDTH * DISPLAY_HEIGHT * 3, 1);
if (combined == nullptr) return result;
static EdgeColor averageEdge(const uint8_t* rgb, uint16_t w, uint16_t h,
int side) {
// side: 0=top, 1=bottom, 2=left, 3=right
uint32_t sumR = 0, sumG = 0, sumB = 0, count = 0;
// Process first portrait
uint16_t src1W, src1H;
uint8_t* rgb1 = decodeJpeg(jpeg1Data, jpeg1Size, &src1W, &src1H);
if (rgb1 != nullptr) {
uint16_t cropX, cropY, cropW, cropH;
centerCrop(rgb1, src1W, src1H, portraitW, portraitH,
&cropX, &cropY, &cropW, &cropH);
uint8_t* cropped1 = (uint8_t*)ps_malloc(cropW * cropH * 3);
if (cropped1) {
for (uint16_t y = 0; y < cropH; y++) {
memcpy(cropped1 + y * cropW * 3,
rgb1 + ((cropY + y) * src1W + cropX) * 3, cropW * 3);
}
uint8_t* resized1 = resize(cropped1, cropW, cropH, portraitW, portraitH);
free(cropped1);
if (resized1) {
// Copy into left side of combined buffer
for (uint16_t y = 0; y < portraitH; y++) {
memcpy(combined + y * DISPLAY_WIDTH * 3,
resized1 + y * portraitW * 3, portraitW * 3);
switch (side) {
case 0: { // top rows
int rows = min((int)h, EDGE_DEPTH);
for (int y = 0; y < rows; y++)
for (int x = 0; x < w; x++) {
size_t idx = ((size_t)y * w + x) * 3;
sumR += rgb[idx]; sumG += rgb[idx+1]; sumB += rgb[idx+2]; count++;
}
free(resized1);
}
break;
}
free(rgb1);
}
// Process second portrait
uint16_t src2W, src2H;
uint8_t* rgb2 = decodeJpeg(jpeg2Data, jpeg2Size, &src2W, &src2H);
if (rgb2 != nullptr) {
uint16_t cropX, cropY, cropW, cropH;
centerCrop(rgb2, src2W, src2H, portraitW, portraitH,
&cropX, &cropY, &cropW, &cropH);
uint8_t* cropped2 = (uint8_t*)ps_malloc(cropW * cropH * 3);
if (cropped2) {
for (uint16_t y = 0; y < cropH; y++) {
memcpy(cropped2 + y * cropW * 3,
rgb2 + ((cropY + y) * src2W + cropX) * 3, cropW * 3);
}
uint8_t* resized2 = resize(cropped2, cropW, cropH, portraitW, portraitH);
free(cropped2);
if (resized2) {
// Copy into right side of combined buffer
uint16_t offsetX = portraitW + PORTRAIT_GAP_PX;
for (uint16_t y = 0; y < portraitH; y++) {
memcpy(combined + (y * DISPLAY_WIDTH + offsetX) * 3,
resized2 + y * portraitW * 3, portraitW * 3);
case 1: { // bottom rows
int rows = min((int)h, EDGE_DEPTH);
for (int y = h - rows; y < h; y++)
for (int x = 0; x < w; x++) {
size_t idx = ((size_t)y * w + x) * 3;
sumR += rgb[idx]; sumG += rgb[idx+1]; sumB += rgb[idx+2]; count++;
}
free(resized2);
break;
}
case 2: { // left columns
int cols = min((int)w, EDGE_DEPTH);
for (int y = 0; y < h; y++)
for (int x = 0; x < cols; x++) {
size_t idx = ((size_t)y * w + x) * 3;
sumR += rgb[idx]; sumG += rgb[idx+1]; sumB += rgb[idx+2]; count++;
}
break;
}
case 3: { // right columns
int cols = min((int)w, EDGE_DEPTH);
for (int y = 0; y < h; y++)
for (int x = w - cols; x < w; x++) {
size_t idx = ((size_t)y * w + x) * 3;
sumR += rgb[idx]; sumG += rgb[idx+1]; sumB += rgb[idx+2]; count++;
}
break;
}
}
if (count == 0) return {0, 0, 0};
return {(uint8_t)(sumR / count), (uint8_t)(sumG / count), (uint8_t)(sumB / count)};
}
// Fill letterbox bars in the final buffer, matching each bar to its adjacent photo edge.
// fitted image is placed at (offsetX, offsetY) with size (fitW x fitH) inside (targetW x targetH).
static void fillLetterbox(uint8_t* final_buf, uint16_t targetW, uint16_t targetH,
const uint8_t* fitted, uint16_t fitW, uint16_t fitH,
uint16_t offsetX, uint16_t offsetY) {
if (offsetY > 0) {
// Horizontal letterbox (top and bottom bars)
EdgeColor top = averageEdge(fitted, fitW, fitH, 0);
EdgeColor bot = averageEdge(fitted, fitW, fitH, 1);
// Fill top bar
for (uint16_t y = 0; y < offsetY; y++)
for (uint16_t x = 0; x < targetW; x++) {
size_t idx = ((size_t)y * targetW + x) * 3;
final_buf[idx] = top.r; final_buf[idx+1] = top.g; final_buf[idx+2] = top.b;
}
}
free(rgb2);
// Fill bottom bar
uint16_t botStart = offsetY + fitH;
for (uint16_t y = botStart; y < targetH; y++)
for (uint16_t x = 0; x < targetW; x++) {
size_t idx = ((size_t)y * targetW + x) * 3;
final_buf[idx] = bot.r; final_buf[idx+1] = bot.g; final_buf[idx+2] = bot.b;
}
Serial.printf("[pipeline] Letterbox TB: top=#%02X%02X%02X bot=#%02X%02X%02X\n",
top.r, top.g, top.b, bot.r, bot.g, bot.b);
}
// Dither combined buffer
uint8_t* dithered = dither(combined, DISPLAY_WIDTH, DISPLAY_HEIGHT);
free(combined);
if (offsetX > 0) {
// Vertical letterbox (left and right bars)
EdgeColor left = averageEdge(fitted, fitW, fitH, 2);
EdgeColor right = averageEdge(fitted, fitW, fitH, 3);
if (dithered == nullptr) return result;
// Fill left bar
for (uint16_t y = 0; y < targetH; y++)
for (uint16_t x = 0; x < offsetX; x++) {
size_t idx = ((size_t)y * targetW + x) * 3;
final_buf[idx] = left.r; final_buf[idx+1] = left.g; final_buf[idx+2] = left.b;
}
// Fill right bar
uint16_t rightStart = offsetX + fitW;
for (uint16_t y = 0; y < targetH; y++)
for (uint16_t x = rightStart; x < targetW; x++) {
size_t idx = ((size_t)y * targetW + x) * 3;
final_buf[idx] = right.r; final_buf[idx+1] = right.g; final_buf[idx+2] = right.b;
}
result.framebuffer = dithered;
result.valid = true;
return result;
Serial.printf("[pipeline] Letterbox LR: left=#%02X%02X%02X right=#%02X%02X%02X\n",
left.r, left.g, left.b, right.r, right.g, right.b);
}
}
void ImagePipeline::freeImage(ProcessedImage& img) {
if (img.framebuffer) {
free(img.framebuffer);
img.framebuffer = nullptr;
}
img.valid = false;
}
// Bilinear resize from src (srcW x srcH) to dst (dstW x dstH)
static void bilinearResize(const uint8_t* src, uint16_t srcW, uint16_t srcH,
uint8_t* dst, uint16_t dstW, uint16_t dstH) {
if (dstW <= 1 || dstH <= 1 || srcW <= 1 || srcH <= 1) return;
uint8_t* ImagePipeline::decodeJpeg(uint8_t* data, size_t size,
uint16_t* outWidth, uint16_t* outHeight) {
JpegMemReader reader = {data, size, 0};
lgfxJdec jdec;
static constexpr uint16_t poolSize = 3900;
uint8_t* pool = (uint8_t*)malloc(poolSize);
if (pool == nullptr) {
return nullptr;
}
JRESULT prep = lgfx_jd_prepare(&jdec, jpegMemInput, pool, poolSize, &reader);
if (prep != JDR_OK) {
free(pool);
return nullptr;
}
uint16_t w = jdec.width;
uint16_t h = jdec.height;
free(pool);
lgfx::LGFX_Sprite sprite;
sprite.setPsram(true);
sprite.setColorDepth(lgfx::color_depth_t::rgb888_3Byte);
if (!sprite.createSprite(w, h)) {
return nullptr;
}
if (!sprite.drawJpg(data, size, 0, 0)) {
sprite.deleteSprite();
return nullptr;
}
size_t bufSize = static_cast<size_t>(w) * h * 3;
uint8_t* rgb = (uint8_t*)ps_malloc(bufSize);
if (rgb == nullptr) {
sprite.deleteSprite();
return nullptr;
}
for (uint16_t y = 0; y < h; y++) {
for (uint16_t x = 0; x < w; x++) {
lgfx::bgr888_t color = sprite.readPixelRGB(x, y);
size_t idx = (static_cast<size_t>(y) * w + x) * 3;
rgb[idx] = color.r;
rgb[idx + 1] = color.g;
rgb[idx + 2] = color.b;
}
}
sprite.deleteSprite();
*outWidth = w;
*outHeight = h;
return rgb;
}
uint8_t* ImagePipeline::resize(uint8_t* rgb, uint16_t srcW, uint16_t srcH,
uint16_t dstW, uint16_t dstH) {
size_t bufSize = dstW * dstH * 3;
uint8_t* dst = (uint8_t*)ps_malloc(bufSize);
if (dst == nullptr) return nullptr;
if (dstW <= 1 || dstH <= 1 || srcW == 0 || srcH == 0) {
free(dst);
return nullptr;
}
// Bilinear interpolation
float xRatio = (float)(srcW - 1) / (float)(dstW - 1);
float yRatio = (float)(srcH - 1) / (float)(dstH - 1);
@@ -259,105 +196,465 @@ uint8_t* ImagePipeline::resize(uint8_t* rgb, uint16_t srcW, uint16_t srcH,
float xFrac = srcX - x0;
for (int c = 0; c < 3; c++) {
float top = rgb[(y0 * srcW + x0) * 3 + c] * (1 - xFrac) +
rgb[(y0 * srcW + x1) * 3 + c] * xFrac;
float bot = rgb[(y1 * srcW + x0) * 3 + c] * (1 - xFrac) +
rgb[(y1 * srcW + x1) * 3 + c] * xFrac;
float top = src[(y0 * srcW + x0) * 3 + c] * (1 - xFrac) +
src[(y0 * srcW + x1) * 3 + c] * xFrac;
float bot = src[(y1 * srcW + x0) * 3 + c] * (1 - xFrac) +
src[(y1 * srcW + x1) * 3 + c] * xFrac;
float val = top * (1 - yFrac) + bot * yFrac;
dst[(y * dstW + x) * 3 + c] = (uint8_t)(val + 0.5f);
}
}
}
return dst;
}
void ImagePipeline::centerCrop(uint8_t* rgb, uint16_t srcW, uint16_t srcH,
uint16_t targetW, uint16_t targetH,
uint16_t* cropX, uint16_t* cropY,
uint16_t* cropW, uint16_t* cropH) {
(void)rgb;
float targetAspect = (float)targetW / (float)targetH;
float srcAspect = (float)srcW / (float)srcH;
// Parse JPEG markers properly to find SOF and extract dimensions.
static bool parseJpegHeader(const uint8_t* data, size_t size,
uint16_t* width, uint16_t* height, bool* isProgressive) {
*width = 0;
*height = 0;
*isProgressive = false;
if (srcAspect > targetAspect) {
// Source is wider — crop sides
*cropH = srcH;
*cropW = (uint16_t)(srcH * targetAspect);
*cropX = (srcW - *cropW) / 2;
*cropY = 0;
} else {
// Source is taller — crop top/bottom
*cropW = srcW;
*cropH = (uint16_t)(srcW / targetAspect);
*cropX = 0;
*cropY = (srcH - *cropH) / 2;
if (size < 2 || data[0] != 0xFF || data[1] != 0xD8) return false;
size_t pos = 2;
while (pos + 4 < size) {
if (data[pos] != 0xFF) { pos++; continue; }
while (pos < size && data[pos] == 0xFF) pos++;
if (pos >= size) break;
uint8_t marker = data[pos++];
if (marker >= 0xC0 && marker <= 0xCF && marker != 0xC4 && marker != 0xCC) {
if (pos + 7 > size) break;
*height = (data[pos + 3] << 8) | data[pos + 4];
*width = (data[pos + 5] << 8) | data[pos + 6];
*isProgressive = (marker == 0xC2);
return true;
}
if (marker == 0xDA) break;
if (pos + 1 >= size) break;
uint16_t segLen = (data[pos] << 8) | data[pos + 1];
pos += segLen;
}
return false;
}
uint8_t* ImagePipeline::dither(uint8_t* rgb, uint16_t width, uint16_t height) {
size_t pixelCount = width * height;
uint8_t* output = (uint8_t*)ps_malloc(pixelCount);
if (output == nullptr) return nullptr;
// Fallback decoder using LGFX's built-in TJPGD
static uint8_t* decodeFallbackLGFX(uint8_t* data, size_t size,
uint16_t targetW, uint16_t targetH,
uint16_t imgW, uint16_t imgH,
uint16_t* outW, uint16_t* outH) {
// Calculate decode dimensions (cap at 960px longest side)
static constexpr uint16_t MAX_DECODE = 960;
uint16_t decodeW = imgW;
uint16_t decodeH = imgH;
uint16_t maxSide = max(imgW, imgH);
// Work buffer with int16 to handle error overflow
int16_t* work = (int16_t*)ps_malloc(pixelCount * 3 * sizeof(int16_t));
if (work == nullptr) {
free(output);
if (maxSide > MAX_DECODE) {
float scale = (float)MAX_DECODE / (float)maxSide;
decodeW = (uint16_t)(imgW * scale);
decodeH = (uint16_t)(imgH * scale);
}
Serial.printf("[pipeline/lgfx] Decoding %dx%d -> %dx%d sprite\n",
imgW, imgH, decodeW, decodeH);
lgfx::LGFX_Sprite sprite;
sprite.setPsram(true);
sprite.setColorDepth(lgfx::color_depth_t::rgb888_3Byte);
if (!sprite.createSprite(decodeW, decodeH)) {
Serial.println("[pipeline/lgfx] Sprite alloc failed");
return nullptr;
}
// Copy to work buffer
for (size_t i = 0; i < pixelCount * 3; i++) {
work[i] = rgb[i];
sprite.fillSprite(0);
float scaleX = (float)decodeW / (float)imgW;
float scaleY = (float)decodeH / (float)imgH;
if (!sprite.drawJpg(data, size, 0, 0, decodeW, decodeH, 0, 0, scaleX, scaleY)) {
Serial.println("[pipeline/lgfx] drawJpg failed (possibly progressive)");
sprite.deleteSprite();
return nullptr;
}
// Fit-contain: scale to fit entirely within target
float fitScaleW = (float)targetW / (float)decodeW;
float fitScaleH = (float)targetH / (float)decodeH;
float fitScale = fminf(fitScaleW, fitScaleH);
uint16_t fitW = (uint16_t)(decodeW * fitScale);
uint16_t fitH = (uint16_t)(decodeH * fitScale);
// Extract full decoded image from sprite
size_t decodeBufSize = (size_t)decodeW * decodeH * 3;
uint8_t* decoded = (uint8_t*)ps_malloc(decodeBufSize);
if (decoded == nullptr) {
sprite.deleteSprite();
return nullptr;
}
for (uint16_t y = 0; y < decodeH; y++) {
for (uint16_t x = 0; x < decodeW; x++) {
lgfx::bgr888_t color = sprite.readPixelRGB(x, y);
size_t idx = ((size_t)y * decodeW + x) * 3;
decoded[idx] = color.r;
decoded[idx + 1] = color.g;
decoded[idx + 2] = color.b;
}
}
sprite.deleteSprite();
// Resize to fit dimensions
uint8_t* fitted = (uint8_t*)ps_malloc((size_t)fitW * fitH * 3);
if (fitted == nullptr) {
free(decoded);
return nullptr;
}
bilinearResize(decoded, decodeW, decodeH, fitted, fitW, fitH);
free(decoded);
// Allocate final target buffer (zeroed)
size_t finalSize = (size_t)targetW * targetH * 3;
uint8_t* final_buf = (uint8_t*)ps_calloc(targetW * targetH, 3);
if (final_buf == nullptr) {
free(fitted);
return nullptr;
}
// Center the fitted image in the final buffer
uint16_t offsetX = (targetW - fitW) / 2;
uint16_t offsetY = (targetH - fitH) / 2;
for (uint16_t y = 0; y < fitH; y++) {
memcpy(final_buf + ((size_t)(offsetY + y) * targetW + offsetX) * 3,
fitted + (size_t)y * fitW * 3,
fitW * 3);
}
// Fill letterbox bars with per-edge average colors
fillLetterbox(final_buf, targetW, targetH, fitted, fitW, fitH, offsetX, offsetY);
free(fitted);
Serial.printf("[pipeline/lgfx] Fit %dx%d into %dx%d (offset %d,%d)\n",
fitW, fitH, targetW, targetH, offsetX, offsetY);
*outW = targetW;
*outH = targetH;
return final_buf;
}
uint8_t* ImagePipeline::decodeAndFit(uint8_t* data, size_t size,
uint16_t targetW, uint16_t targetH,
uint16_t* outW, uint16_t* outH) {
// Parse JPEG header to get dimensions and detect progressive
uint16_t headerW = 0, headerH = 0;
bool progressive = false;
if (parseJpegHeader(data, size, &headerW, &headerH, &progressive)) {
Serial.printf("[pipeline] Header: %dx%d %s\n", headerW, headerH,
progressive ? "progressive" : "baseline");
} else {
Serial.println("[pipeline] Could not parse JPEG header");
}
if (progressive) {
Serial.println("[pipeline] Progressive JPEG — skipping (server still reprocessing?)");
return nullptr;
}
// Primary path: JPEGDEC
JPEGDEC jpeg;
if (!jpeg.openRAM(data, (int)size, jpegDrawCallback)) {
int err = jpeg.getLastError();
Serial.printf("[pipeline] JPEGDEC openRAM failed (err %d), trying LGFX fallback\n", err);
if (headerW > 0 && headerH > 0) {
return decodeFallbackLGFX(data, size, targetW, targetH,
headerW, headerH, outW, outH);
}
return nullptr;
}
int imgW = jpeg.getWidth();
int imgH = jpeg.getHeight();
Serial.printf("[pipeline] JPEG: %dx%d, type=baseline\n", imgW, imgH);
// Determine decode scale (reduce large images before fit)
int decodeOptions = 0;
int decodeW = imgW;
int decodeH = imgH;
if (imgW > targetW * 4 && imgH > targetH * 4) {
decodeOptions = JPEG_SCALE_QUARTER;
decodeW = imgW / 4;
decodeH = imgH / 4;
Serial.printf("[pipeline] Using 1/4 scale: %dx%d\n", decodeW, decodeH);
} else if (imgW > targetW * 2 && imgH > targetH * 2) {
decodeOptions = JPEG_SCALE_HALF;
decodeW = imgW / 2;
decodeH = imgH / 2;
Serial.printf("[pipeline] Using 1/2 scale: %dx%d\n", decodeW, decodeH);
}
// Decode full image (no cropping — we want the whole photo)
size_t decodeBufSize = (size_t)decodeW * decodeH * 3;
uint8_t* decodedRgb = (uint8_t*)ps_malloc(decodeBufSize);
if (decodedRgb == nullptr) {
Serial.printf("[pipeline] Failed to allocate %u KB for decode buffer\n",
(unsigned)(decodeBufSize / 1024));
jpeg.close();
return nullptr;
}
memset(decodedRgb, 0, decodeBufSize);
DecodeContext ctx;
ctx.rgbBuffer = decodedRgb;
ctx.bufferWidth = (uint16_t)decodeW;
ctx.bufferHeight = (uint16_t)decodeH;
jpeg.setUserPointer(&ctx);
jpeg.setPixelType(RGB565_LITTLE_ENDIAN);
if (!jpeg.decode(0, 0, decodeOptions)) {
Serial.printf("[pipeline] JPEGDEC decode failed (err %d), trying LGFX fallback\n",
jpeg.getLastError());
free(decodedRgb);
jpeg.close();
if (headerW > 0 && headerH > 0) {
return decodeFallbackLGFX(data, size, targetW, targetH,
headerW, headerH, outW, outH);
}
return nullptr;
}
jpeg.close();
// Fit-contain: scale to fit entirely within target dimensions
float fitScaleW = (float)targetW / (float)decodeW;
float fitScaleH = (float)targetH / (float)decodeH;
float fitScale = fminf(fitScaleW, fitScaleH);
uint16_t fitW = (uint16_t)(decodeW * fitScale);
uint16_t fitH = (uint16_t)(decodeH * fitScale);
Serial.printf("[pipeline] Fit: %dx%d -> %dx%d (scale %.2f)\n",
decodeW, decodeH, fitW, fitH, fitScale);
// Resize decoded image to fit dimensions
uint8_t* fitted = (uint8_t*)ps_malloc((size_t)fitW * fitH * 3);
if (fitted == nullptr) {
Serial.println("[pipeline] Fit alloc failed");
free(decodedRgb);
return nullptr;
}
bilinearResize(decodedRgb, (uint16_t)decodeW, (uint16_t)decodeH, fitted, fitW, fitH);
free(decodedRgb);
// Allocate final target-sized buffer (zeroed)
size_t finalSize = (size_t)targetW * targetH * 3;
uint8_t* final_buf = (uint8_t*)ps_calloc(targetW * targetH, 3);
if (final_buf == nullptr) {
Serial.println("[pipeline] Final buffer alloc failed");
free(fitted);
return nullptr;
}
// Center the fitted image into the final buffer
uint16_t offsetX = (targetW - fitW) / 2;
uint16_t offsetY = (targetH - fitH) / 2;
for (uint16_t y = 0; y < fitH; y++) {
memcpy(final_buf + ((size_t)(offsetY + y) * targetW + offsetX) * 3,
fitted + (size_t)y * fitW * 3,
fitW * 3);
}
// Fill letterbox bars — each bar matches the adjacent photo edge
fillLetterbox(final_buf, targetW, targetH, fitted, fitW, fitH, offsetX, offsetY);
free(fitted);
*outW = targetW;
*outH = targetH;
return final_buf;
}
ProcessedImage ImagePipeline::process(uint8_t* jpegData, size_t jpegSize) {
ProcessedImage result = {nullptr, DISPLAY_WIDTH, DISPLAY_HEIGHT, false};
uint16_t outW, outH;
uint8_t* rgb = decodeAndFit(jpegData, jpegSize, DISPLAY_WIDTH, DISPLAY_HEIGHT, &outW, &outH);
if (rgb == nullptr) {
Serial.println("[pipeline] Decode/fit failed");
return result;
}
Serial.printf("[pipeline] Got %dx%d fitted RGB\n", outW, outH);
// Enhance contrast/saturation for e-ink readability
enhanceContrast(rgb, (size_t)outW * outH);
// Row-by-row Floyd-Steinberg dithering to 6-color palette
uint8_t* dithered = ditherRowByRow(rgb, outW, outH);
free(rgb);
if (dithered == nullptr) {
return result;
}
result.framebuffer = dithered;
result.width = outW;
result.height = outH;
result.valid = true;
Serial.printf("[pipeline] Processing complete (%dx%d)\n", outW, outH);
return result;
}
ProcessedImage ImagePipeline::processPortraitPair(uint8_t* jpeg1Data, size_t jpeg1Size,
uint8_t* jpeg2Data, size_t jpeg2Size) {
ProcessedImage result = {nullptr, DISPLAY_WIDTH, DISPLAY_HEIGHT, false};
uint16_t portraitW = (DISPLAY_WIDTH - PORTRAIT_GAP_PX) / 2;
uint16_t portraitH = DISPLAY_HEIGHT;
uint16_t out1W, out1H;
uint8_t* rgb1 = decodeAndFit(jpeg1Data, jpeg1Size, portraitW, portraitH, &out1W, &out1H);
uint16_t out2W, out2H;
uint8_t* rgb2 = decodeAndFit(jpeg2Data, jpeg2Size, portraitW, portraitH, &out2W, &out2H);
// Allocate combined RGB buffer
uint8_t* combined = (uint8_t*)ps_calloc(DISPLAY_WIDTH * DISPLAY_HEIGHT, 3);
if (combined == nullptr) {
if (rgb1) free(rgb1);
if (rgb2) free(rgb2);
return result;
}
if (rgb1 != nullptr) {
for (uint16_t y = 0; y < out1H; y++) {
memcpy(combined + y * DISPLAY_WIDTH * 3,
rgb1 + y * out1W * 3, out1W * 3);
}
free(rgb1);
}
if (rgb2 != nullptr) {
uint16_t offsetX = portraitW + PORTRAIT_GAP_PX;
for (uint16_t y = 0; y < out2H; y++) {
memcpy(combined + (y * DISPLAY_WIDTH + offsetX) * 3,
rgb2 + y * out2W * 3, out2W * 3);
}
free(rgb2);
}
enhanceContrast(combined, (size_t)DISPLAY_WIDTH * DISPLAY_HEIGHT);
uint8_t* dithered = ditherRowByRow(combined, DISPLAY_WIDTH, DISPLAY_HEIGHT);
free(combined);
if (dithered == nullptr) return result;
result.framebuffer = dithered;
result.width = DISPLAY_WIDTH;
result.height = DISPLAY_HEIGHT;
result.valid = true;
return result;
}
void ImagePipeline::freeImage(ProcessedImage& img) {
if (img.framebuffer) {
free(img.framebuffer);
img.framebuffer = nullptr;
}
img.valid = false;
}
uint8_t* ImagePipeline::ditherRowByRow(uint8_t* rgb, uint16_t width, uint16_t height) {
size_t pixelCount = (size_t)width * height;
uint8_t* output = (uint8_t*)ps_malloc(pixelCount);
if (output == nullptr) {
Serial.println("[pipeline] Dither output alloc failed");
return nullptr;
}
size_t rowBytes = (size_t)width * 3 * sizeof(int16_t);
int16_t* errCurrent = (int16_t*)ps_malloc(rowBytes);
int16_t* errNext = (int16_t*)ps_malloc(rowBytes);
if (errCurrent == nullptr || errNext == nullptr) {
Serial.println("[pipeline] Dither error buffer alloc failed");
free(output);
if (errCurrent) free(errCurrent);
if (errNext) free(errNext);
return nullptr;
}
// Initialize first row
for (uint16_t x = 0; x < width; x++) {
size_t srcIdx = x * 3;
size_t errIdx = x * 3;
errCurrent[errIdx] = rgb[srcIdx];
errCurrent[errIdx + 1] = rgb[srcIdx + 1];
errCurrent[errIdx + 2] = rgb[srcIdx + 2];
}
// Floyd-Steinberg dithering
for (uint16_t y = 0; y < height; y++) {
// Prepare next row from source
if (y + 1 < height) {
size_t nextRowOffset = (size_t)(y + 1) * width * 3;
for (uint16_t x = 0; x < width; x++) {
size_t srcIdx = nextRowOffset + x * 3;
size_t errIdx = x * 3;
errNext[errIdx] = rgb[srcIdx];
errNext[errIdx + 1] = rgb[srcIdx + 1];
errNext[errIdx + 2] = rgb[srcIdx + 2];
}
} else {
memset(errNext, 0, rowBytes);
}
for (uint16_t x = 0; x < width; x++) {
size_t idx = (y * width + x) * 3;
int r = constrain(work[idx], 0, 255);
int g = constrain(work[idx + 1], 0, 255);
int b = constrain(work[idx + 2], 0, 255);
size_t errIdx = x * 3;
int r = constrain(errCurrent[errIdx], 0, 255);
int g = constrain(errCurrent[errIdx + 1], 0, 255);
int b = constrain(errCurrent[errIdx + 2], 0, 255);
uint8_t nearest = findNearest(r, g, b);
output[y * width + x] = nearest;
int errR = r - PALETTE_RGB[nearest][0];
int errG = g - PALETTE_RGB[nearest][1];
int errB = b - PALETTE_RGB[nearest][2];
int errR = r - PALETTE_CALIBRATED[nearest][0];
int errG = g - PALETTE_CALIBRATED[nearest][1];
int errB = b - PALETTE_CALIBRATED[nearest][2];
// Distribute error
if (x + 1 < width) {
size_t ni = (y * width + (x + 1)) * 3;
work[ni] += errR * 7 / 16;
work[ni + 1] += errG * 7 / 16;
work[ni + 2] += errB * 7 / 16;
size_t ni = (x + 1) * 3;
errCurrent[ni] += errR * 7 / 16;
errCurrent[ni + 1] += errG * 7 / 16;
errCurrent[ni + 2] += errB * 7 / 16;
}
if (y + 1 < height && x > 0) {
size_t ni = (x - 1) * 3;
errNext[ni] += errR * 3 / 16;
errNext[ni + 1] += errG * 3 / 16;
errNext[ni + 2] += errB * 3 / 16;
}
if (y + 1 < height) {
if (x > 0) {
size_t ni = ((y + 1) * width + (x - 1)) * 3;
work[ni] += errR * 3 / 16;
work[ni + 1] += errG * 3 / 16;
work[ni + 2] += errB * 3 / 16;
}
{
size_t ni = ((y + 1) * width + x) * 3;
work[ni] += errR * 5 / 16;
work[ni + 1] += errG * 5 / 16;
work[ni + 2] += errB * 5 / 16;
}
if (x + 1 < width) {
size_t ni = ((y + 1) * width + (x + 1)) * 3;
work[ni] += errR * 1 / 16;
work[ni + 1] += errG * 1 / 16;
work[ni + 2] += errB * 1 / 16;
}
size_t ni = x * 3;
errNext[ni] += errR * 5 / 16;
errNext[ni + 1] += errG * 5 / 16;
errNext[ni + 2] += errB * 5 / 16;
}
if (y + 1 < height && x + 1 < width) {
size_t ni = (x + 1) * 3;
errNext[ni] += errR * 1 / 16;
errNext[ni + 1] += errG * 1 / 16;
errNext[ni + 2] += errB * 1 / 16;
}
}
int16_t* tmp = errCurrent;
errCurrent = errNext;
errNext = tmp;
}
free(work);
free(errCurrent);
free(errNext);
return output;
}
@@ -365,9 +662,9 @@ uint8_t ImagePipeline::findNearest(int r, int g, int b) {
uint8_t best = 0;
int bestDist = INT32_MAX;
for (int i = 0; i < DISPLAY_COLORS; i++) {
int dr = r - PALETTE_RGB[i][0];
int dg = g - PALETTE_RGB[i][1];
int db = b - PALETTE_RGB[i][2];
int dr = r - PALETTE_CALIBRATED[i][0];
int dg = g - PALETTE_CALIBRATED[i][1];
int db = b - PALETTE_CALIBRATED[i][2];
int dist = dr * dr + dg * dg + db * db;
if (dist < bestDist) {
bestDist = dist;

32
src/image_pipeline.h
View File

@@ -1,6 +1,7 @@
#pragma once
#include <Arduino.h>
#include <JPEGDEC.h>
#include "config.h"
struct ProcessedImage {
@@ -10,9 +11,16 @@ struct ProcessedImage {
bool valid;
};
// Context passed through JPEGDEC's user pointer for the draw callback
struct DecodeContext {
uint8_t* rgbBuffer; // Destination RGB888 buffer in PSRAM
uint16_t bufferWidth; // Width of the output buffer
uint16_t bufferHeight; // Height of the output buffer
};
class ImagePipeline {
public:
// Process single landscape photo
// Process single landscape photo (fit-contain with edge-color letterbox)
ProcessedImage process(uint8_t* jpegData, size_t jpegSize);
// Process portrait pair (two photos side by side)
@@ -22,22 +30,14 @@ public:
void freeImage(ProcessedImage& img);
private:
// Decode JPEG into RGB888 buffer in PSRAM
uint8_t* decodeJpeg(uint8_t* data, size_t size, uint16_t* outWidth, uint16_t* outHeight);
// Decode JPEG and fit into target dimensions with letterboxing
uint8_t* decodeAndFit(uint8_t* data, size_t size,
uint16_t targetW, uint16_t targetH,
uint16_t* outW, uint16_t* outH);
// Resize RGB buffer to target dimensions (bilinear)
uint8_t* resize(uint8_t* rgb, uint16_t srcW, uint16_t srcH,
uint16_t dstW, uint16_t dstH);
// Row-by-row Floyd-Steinberg dither RGB888 to 6-color palette indices
uint8_t* ditherRowByRow(uint8_t* rgb, uint16_t width, uint16_t height);
// Center-crop to target aspect ratio
void centerCrop(uint8_t* rgb, uint16_t srcW, uint16_t srcH,
uint16_t targetW, uint16_t targetH,
uint16_t* cropX, uint16_t* cropY,
uint16_t* cropW, uint16_t* cropH);
// Floyd-Steinberg dither RGB888 to 6-color palette indices
uint8_t* dither(uint8_t* rgb, uint16_t width, uint16_t height);
// Find nearest palette color
// Find nearest palette color (Euclidean distance in RGB space)
uint8_t findNearest(int r, int g, int b);
};

204
src/immich_client.cpp
View File

@@ -11,7 +11,13 @@ void ImmichClient::begin(const String& baseUrl, const String& apiKey) {
_baseUrl.remove(_baseUrl.length() - 1);
}
_apiKey = apiKey;
Serial.printf("[immich] Configured: %s\n", _baseUrl.c_str());
// Debug: show masked key so we can verify it's non-empty and correct prefix
String maskedKey = _apiKey.length() > 8
? _apiKey.substring(0, 8) + "..."
: (_apiKey.length() > 0 ? "***" : "(empty)");
Serial.printf("[immich] Configured: %s | key: %s (len=%d)\n",
_baseUrl.c_str(), maskedKey.c_str(), _apiKey.length());
}
std::vector<AlbumInfo> ImmichClient::fetchAlbums() {
@@ -47,12 +53,23 @@ std::vector<AlbumInfo> ImmichClient::fetchAlbums() {
std::vector<String> ImmichClient::fetchAlbumAssetIds(const String& albumId) {
std::vector<String> ids;
String url = buildUrl("/api/albums/" + albumId);
String response = httpGet(url);
if (response.isEmpty()) return ids;
// Album responses can be very large (full asset metadata per photo).
// Use PSRAM-backed fetch to avoid exhausting regular heap.
size_t responseLen = 0;
char* response = httpGetPsram(url, &responseLen);
if (response == nullptr || responseLen == 0) {
Serial.printf("[immich] Album %s: empty response\n", albumId.c_str());
if (response) free(response);
return ids;
}
Serial.printf("[immich] Album %s: response %d bytes\n", albumId.c_str(), responseLen);
JsonDocument doc;
DeserializationError err = deserializeJson(doc, response);
DeserializationError err = deserializeJson(doc, response, responseLen);
free(response); // Free PSRAM buffer immediately after parsing
if (err) {
Serial.printf("[immich] fetchAlbumAssets JSON error: %s\n", err.c_str());
return ids;
@@ -60,13 +77,67 @@ std::vector<String> ImmichClient::fetchAlbumAssetIds(const String& albumId) {
JsonArray assets = doc["assets"].as<JsonArray>();
for (JsonObject asset : assets) {
ids.push_back(asset["id"].as<String>());
String id = asset["id"].as<String>();
if (id.length() > 0) {
ids.push_back(id);
}
}
Serial.printf("[immich] Album %s: %d assets\n", albumId.c_str(), ids.size());
return ids;
}
std::vector<String> ImmichClient::fetchRandomAssetIds(int count) {
std::vector<String> ids;
String url = buildUrl("/api/search/random");
// Build POST body — filter to images only (excludes videos, audio)
JsonDocument reqDoc;
reqDoc["size"] = count;
reqDoc["type"] = "IMAGE";
String body;
serializeJson(reqDoc, body);
String response = httpPost(url, body);
if (response.isEmpty()) {
Serial.println("[immich] fetchRandom: empty response");
return ids;
}
JsonDocument doc;
DeserializationError err = deserializeJson(doc, response);
if (err) {
Serial.printf("[immich] fetchRandom JSON error: %s\n", err.c_str());
return ids;
}
JsonArray arr = doc.as<JsonArray>();
int skippedRaw = 0;
for (JsonObject asset : arr) {
String id = asset["id"].as<String>();
if (id.length() == 0) continue;
// Filter out RAW/DNG files — their previews work but we prefer actual photos
String filename = asset["originalFileName"] | "";
filename.toLowerCase();
if (filename.endsWith(".dng") || filename.endsWith(".raw") ||
filename.endsWith(".cr2") || filename.endsWith(".cr3") ||
filename.endsWith(".nef") || filename.endsWith(".arw") ||
filename.endsWith(".orf") || filename.endsWith(".rw2") ||
filename.endsWith(".raf") || filename.endsWith(".srw")) {
skippedRaw++;
continue;
}
ids.push_back(id);
}
Serial.printf("[immich] Fetched %d random assets (skipped %d RAW)\n",
ids.size(), skippedRaw);
return ids;
}
std::vector<String> ImmichClient::fetchFavoriteAssetIds() {
std::vector<String> ids;
String url = buildUrl("/api/assets?isFavorite=true");
@@ -147,14 +218,29 @@ AssetInfo ImmichClient::fetchAssetInfo(const String& assetId) {
bool ImmichClient::downloadAsset(const String& assetId, ImageQuality quality,
uint8_t** outBuffer, size_t* outSize) {
String url;
if (quality == ImageQuality::Original) {
url = buildUrl("/api/assets/" + assetId + "/original");
} else {
url = buildUrl("/api/assets/" + assetId + "/thumbnail?size=preview");
// Always request preview size with JPEG format.
// Server controls actual pixel dimensions via its image config (720px baseline).
// The "quality" setting is kept for future use but we always use preview for now
// to stay within PSRAM budget.
(void)quality;
String url = buildUrl("/api/assets/" + assetId + "/thumbnail?size=preview&format=jpeg");
if (!httpGetBinary(url, outBuffer, outSize)) {
return false;
}
return httpGetBinary(url, outBuffer, outSize);
// Verify JPEG magic bytes (FF D8 FF)
if (*outSize < 3 || (*outBuffer)[0] != 0xFF ||
(*outBuffer)[1] != 0xD8 || (*outBuffer)[2] != 0xFF) {
Serial.printf("[immich] Not JPEG data (magic: %02X %02X %02X)\n",
(*outBuffer)[0], (*outBuffer)[1], (*outBuffer)[2]);
free(*outBuffer);
*outBuffer = nullptr;
*outSize = 0;
return false;
}
return true;
}
String ImmichClient::buildUrl(const String& path) {
@@ -183,6 +269,102 @@ String ImmichClient::httpGet(const String& url) {
return result;
}
String ImmichClient::httpPost(const String& url, const String& body) {
WiFiClientSecure client;
client.setInsecure();
HTTPClient http;
http.begin(client, url);
http.addHeader("x-api-key", _apiKey);
http.addHeader("Content-Type", "application/json");
http.setTimeout(30000);
int code = http.POST(body);
String result = "";
if (code == HTTP_CODE_OK) {
result = http.getString();
} else {
Serial.printf("[immich] HTTP POST %s failed: %d\n", url.c_str(), code);
}
http.end();
return result;
}
char* ImmichClient::httpGetPsram(const String& url, size_t* outLen) {
*outLen = 0;
WiFiClientSecure client;
client.setInsecure();
HTTPClient http;
http.begin(client, url);
http.addHeader("x-api-key", _apiKey);
http.setTimeout(60000);
int code = http.GET();
if (code != HTTP_CODE_OK) {
Serial.printf("[immich] PSRAM GET %s failed: %d\n", url.c_str(), code);
http.end();
return nullptr;
}
int contentLength = http.getSize();
WiFiClient* stream = http.getStreamPtr();
// For chunked responses without Content-Length, read in chunks
if (contentLength <= 0) {
// Read incrementally into PSRAM, up to 4MB max
const size_t maxSize = 4 * 1024 * 1024;
size_t capacity = 64 * 1024;
char* buf = (char*)ps_malloc(capacity);
if (!buf) {
Serial.println("[immich] PSRAM alloc failed");
http.end();
return nullptr;
}
size_t total = 0;
while (stream->connected() || stream->available()) {
int avail = stream->available();
if (avail <= 0) { delay(1); continue; }
if (total + avail >= capacity) {
capacity = min(capacity * 2, maxSize);
char* newBuf = (char*)ps_realloc(buf, capacity);
if (!newBuf) { free(buf); http.end(); return nullptr; }
buf = newBuf;
}
int read = stream->readBytes(buf + total, avail);
total += read;
if (total >= maxSize) break;
}
buf[total] = '\0';
*outLen = total;
http.end();
return buf;
}
// Known content length — single PSRAM allocation
char* buf = (char*)ps_malloc(contentLength + 1);
if (!buf) {
Serial.printf("[immich] PSRAM alloc failed for %d bytes\n", contentLength);
http.end();
return nullptr;
}
size_t bytesRead = 0;
while (bytesRead < (size_t)contentLength && (stream->connected() || stream->available())) {
int avail = stream->available();
if (avail <= 0) { delay(1); continue; }
int read = stream->readBytes(buf + bytesRead, min(avail, (int)(contentLength - bytesRead)));
bytesRead += read;
}
buf[bytesRead] = '\0';
*outLen = bytesRead;
http.end();
return buf;
}
bool ImmichClient::httpGetBinary(const String& url, uint8_t** outBuffer, size_t* outSize) {
WiFiClientSecure client;
client.setInsecure();

3
src/immich_client.h
View File

@@ -26,6 +26,7 @@ public:
void begin(const String& baseUrl, const String& apiKey);
std::vector<AlbumInfo> fetchAlbums();
std::vector<String> fetchAlbumAssetIds(const String& albumId);
std::vector<String> fetchRandomAssetIds(int count = 50);
std::vector<String> fetchFavoriteAssetIds();
AssetInfo fetchAssetInfo(const String& assetId);
bool downloadAsset(const String& assetId, ImageQuality quality,
@@ -37,5 +38,7 @@ private:
String buildUrl(const String& path);
String httpGet(const String& url);
String httpPost(const String& url, const String& body);
char* httpGetPsram(const String& url, size_t* outLen);
bool httpGetBinary(const String& url, uint8_t** outBuffer, size_t* outSize);
};

107
src/main.cpp
View File

@@ -1,5 +1,6 @@
#include <Arduino.h>
#include <M5Unified.h>
#include <esp_heap_caps.h>
#include <freertos/FreeRTOS.h>
#include <freertos/task.h>
#include <freertos/semphr.h>
@@ -47,6 +48,11 @@ void displayTask(void* param) {
if (photoQueue.sync()) {
Serial.printf("[display_task] Queue ready: %d photos\n", photoQueue.size());
// Trigger first photo immediately
if (xSemaphoreTake(stateMutex, pdMS_TO_TICKS(100)) == pdTRUE) {
refreshRequested = true;
xSemaphoreGive(stateMutex);
}
} else {
displayManager.showMessage("No Photos", "Select albums in web UI");
}
@@ -80,16 +86,22 @@ void displayTask(void* param) {
}
if (shouldRefresh && photoQueue.size() > 0 && wifiManager.isConnected()) {
// Get next photo ID
String assetId;
if (wantsRandom) {
assetId = photoQueue.random();
} else {
assetId = photoQueue.next();
}
// Try up to 3 assets in case some fail to decode
static constexpr int MAX_RETRIES = 10;
bool displayed = false;
if (assetId.length() > 0) {
Serial.printf("[display_task] Loading asset: %s\n", assetId.c_str());
for (int attempt = 0; attempt < MAX_RETRIES && !displayed; attempt++) {
String assetId;
if (wantsRandom) {
assetId = photoQueue.random();
} else {
assetId = photoQueue.next();
}
if (assetId.length() == 0) break;
Serial.printf("[display_task] Loading asset: %s (attempt %d)\n",
assetId.c_str(), attempt + 1);
// Fetch asset info for portrait detection and metadata
AssetInfo info = immichClient.fetchAssetInfo(assetId);
@@ -100,55 +112,37 @@ void displayTask(void* param) {
bool downloaded = immichClient.downloadAsset(
assetId, s.img_quality, &jpegBuf, &jpegSize);
if (downloaded && jpegBuf != nullptr) {
ProcessedImage img;
if (info.isPortrait) {
// Try to find a portrait pair
String pairId = photoQueue.findPortraitPair(
settingsManager.get().queue_cursor);
if (pairId.length() > 0) {
AssetInfo pairInfo = immichClient.fetchAssetInfo(pairId);
if (pairInfo.isPortrait) {
uint8_t* jpeg2Buf = nullptr;
size_t jpeg2Size = 0;
if (immichClient.downloadAsset(pairId, s.img_quality,
&jpeg2Buf, &jpeg2Size)) {
img = imagePipeline.processPortraitPair(
jpegBuf, jpegSize, jpeg2Buf, jpeg2Size);
free(jpeg2Buf);
} else {
img = imagePipeline.process(jpegBuf, jpegSize);
}
} else {
img = imagePipeline.process(jpegBuf, jpegSize);
}
} else {
img = imagePipeline.process(jpegBuf, jpegSize);
}
} else {
img = imagePipeline.process(jpegBuf, jpegSize);
}
free(jpegBuf);
if (img.valid) {
displayManager.showImage(img);
// Show metadata overlay if enabled
if (s.meta_flags != 0) {
displayManager.showMetadata(info, s.meta_flags, s.meta_pos);
}
imagePipeline.freeImage(img);
} else {
Serial.println("[display_task] Image processing failed");
}
if (!downloaded || jpegBuf == nullptr) {
Serial.println("[display_task] Download failed, trying next");
continue;
}
if (xSemaphoreTake(stateMutex, pdMS_TO_TICKS(100)) == pdTRUE) {
lastRefreshTime = millis();
xSemaphoreGive(stateMutex);
Serial.printf("[display_task] JPEG: %u KB, PSRAM free: %u KB\n",
(unsigned)(jpegSize / 1024),
(unsigned)(heap_caps_get_free_size(MALLOC_CAP_SPIRAM) / 1024));
ProcessedImage img = imagePipeline.process(jpegBuf, jpegSize);
free(jpegBuf);
Serial.printf("[display_task] Post-process PSRAM free: %u KB\n",
(unsigned)(heap_caps_get_free_size(MALLOC_CAP_SPIRAM) / 1024));
if (img.valid) {
displayManager.showImage(img);
if (s.meta_flags != 0) {
displayManager.showMetadata(info, s.meta_flags, s.meta_pos);
}
imagePipeline.freeImage(img);
displayed = true;
} else {
Serial.println("[display_task] Processing failed, trying next");
}
}
if (xSemaphoreTake(stateMutex, pdMS_TO_TICKS(100)) == pdTRUE) {
lastRefreshTime = millis();
xSemaphoreGive(stateMutex);
}
}
// Yield — check every second
@@ -181,6 +175,11 @@ void setup() {
Serial.begin(115200);
Serial.println("[main] Immich Frame v1.0 booting...");
Serial.printf("[main] PSRAM: %u KB free / %u KB total\n",
(unsigned)(heap_caps_get_free_size(MALLOC_CAP_SPIRAM) / 1024),
(unsigned)(heap_caps_get_total_size(MALLOC_CAP_SPIRAM) / 1024));
Serial.printf("[main] Internal RAM: %u KB free\n",
(unsigned)(heap_caps_get_free_size(MALLOC_CAP_INTERNAL) / 1024));
// Create state mutex
stateMutex = xSemaphoreCreateMutex();

100
src/photo_queue.cpp
View File

@@ -1,6 +1,7 @@
#include "photo_queue.h"
#include "immich_client.h"
#include <ArduinoJson.h>
#include <algorithm>
#include "config.h"
void PhotoQueue::begin(ImmichClient& client, SettingsManager& settings) {
@@ -14,37 +15,45 @@ void PhotoQueue::begin(ImmichClient& client, SettingsManager& settings) {
bool PhotoQueue::sync() {
if (_client == nullptr || _settings == nullptr) return false;
_lastSyncAttempt = millis();
Settings s = _settings->get();
// Fetch in batches until we have TARGET_QUEUE_SIZE usable, non-duplicate IDs
std::vector<String> newIds;
int rounds = 0;
// Get selected album IDs
std::vector<String> albumIds = getSelectedAlbumIds();
Serial.printf("[queue] Fetching until %d usable assets (excluding %d shown)\n",
TARGET_QUEUE_SIZE, _shown.size());
if (albumIds.empty()) {
// "All photos" mode — fetch from all albums
auto albums = _client->fetchAlbums();
for (auto& album : albums) {
auto ids = _client->fetchAlbumAssetIds(album.id);
for (auto& id : ids) {
newIds.push_back(id);
}
while (newIds.size() < TARGET_QUEUE_SIZE && rounds < MAX_FETCH_ROUNDS) {
auto batch = _client->fetchRandomAssetIds(50);
if (batch.empty()) {
Serial.printf("[queue] Fetch round %d returned empty\n", rounds + 1);
break;
}
} else {
// Fetch from selected albums only
for (auto& albumId : albumIds) {
auto ids = _client->fetchAlbumAssetIds(albumId);
for (auto& id : ids) {
newIds.push_back(id);
}
for (auto& id : batch) {
if (newIds.size() >= TARGET_QUEUE_SIZE) break;
// Skip if already in this batch
if (std::find(newIds.begin(), newIds.end(), id) != newIds.end()) continue;
// Skip if recently shown
if (std::find(_shown.begin(), _shown.end(), id) != _shown.end()) continue;
newIds.push_back(id);
}
rounds++;
Serial.printf("[queue] Round %d: have %d/%d usable assets\n",
rounds, newIds.size(), TARGET_QUEUE_SIZE);
}
// Deduplicate
std::sort(newIds.begin(), newIds.end());
newIds.erase(std::unique(newIds.begin(), newIds.end()), newIds.end());
if (newIds.empty()) {
Serial.println("[queue] No assets found");
_syncRetryDelay = min(_syncRetryDelay * 2, SYNC_RETRY_MAX);
Serial.printf("[queue] Sync failed after %d rounds, retry in %lus\n",
rounds, _syncRetryDelay / 1000);
return false;
}
@@ -68,24 +77,34 @@ bool PhotoQueue::sync() {
break;
}
// Clamp cursor
if (_cursor >= _queue.size()) {
_cursor = 0;
}
// Reset cursor for fresh queue
_cursor = 0;
_lastSyncTime = millis();
Serial.printf("[queue] Synced: %d assets, cursor at %d\n", _queue.size(), _cursor);
_syncRetryDelay = SYNC_RETRY_MIN;
Serial.printf("[queue] Synced: %d assets in %d rounds, cursor at %d\n",
_queue.size(), rounds, _cursor);
return true;
}
String PhotoQueue::next() {
if (_queue.empty()) return "";
if (_cursor >= _queue.size()) {
// Auto-refill when queue is exhausted
if (_queue.empty() || _cursor >= _queue.size()) {
Serial.println("[queue] Queue exhausted, fetching new batch");
sync();
_cursor = 0;
shuffle(); // Reshuffle on wrap for random mode
}
if (_queue.empty()) return "";
String id = _queue[_cursor++];
// Track shown IDs for dedup on next refill
_shown.push_back(id);
if (_shown.size() > MAX_SHOWN_HISTORY) {
_shown.erase(_shown.begin());
}
// Persist cursor
_settings->saveField("queue_cursor", static_cast<uint32_t>(_cursor));
@@ -109,17 +128,21 @@ size_t PhotoQueue::size() {
}
bool PhotoQueue::needsResync() {
if (_lastSyncTime == 0) return true;
unsigned long elapsed = millis() - _lastSyncTime;
unsigned long now = millis();
// If never synced successfully, use retry backoff
if (_lastSyncTime == 0) {
if (_lastSyncAttempt == 0) return true;
return (now - _lastSyncAttempt) >= _syncRetryDelay;
}
// Normal resync interval
unsigned long elapsed = now - _lastSyncTime;
return elapsed >= (QUEUE_RESYNC_HOURS * 3600000UL);
}
String PhotoQueue::findPortraitPair(size_t startIndex) {
// Look ahead up to PORTRAIT_LOOKAHEAD items for another portrait
for (size_t i = 1; i <= PORTRAIT_LOOKAHEAD && (startIndex + i) < _queue.size(); i++) {
// We'd need asset info to determine portrait status
// This will be called by the display task which fetches AssetInfo
// Return the ID — caller checks isPortrait from AssetInfo
return _queue[startIndex + i];
}
return "";
@@ -133,9 +156,6 @@ void PhotoQueue::shuffle() {
}
void PhotoQueue::sortChronological(bool reverse) {
// For chronological sort, we'd need timestamps which we don't store in the queue
// The IDs from Immich are UUIDs, not sortable by time
// For now, keep the order returned by Immich (which is chronological within albums)
if (reverse) {
std::reverse(_queue.begin(), _queue.end());
}
@@ -148,9 +168,7 @@ void PhotoQueue::applyFavoritesWeighting() {
if (s.cycle_mode != CycleMode::FavoritesWeighted) return;
auto favorites = _client->fetchFavoriteAssetIds();
// Add favorites 2 more times (total 3× appearance)
for (auto& fav : favorites) {
// Only add if already in queue
bool inQueue = false;
for (auto& id : _queue) {
if (id == fav) { inQueue = true; break; }

9
src/photo_queue.h
View File

@@ -22,8 +22,17 @@ private:
SettingsManager* _settings = nullptr;
std::vector<String> _queue;
std::vector<String> _shown; // Recently shown IDs for dedup on refill
size_t _cursor = 0;
unsigned long _lastSyncTime = 0;
unsigned long _lastSyncAttempt = 0;
unsigned long _syncRetryDelay = 10000;
static constexpr unsigned long SYNC_RETRY_MIN = 10000; // 10 seconds
static constexpr unsigned long SYNC_RETRY_MAX = 300000; // 5 minutes
static constexpr size_t TARGET_QUEUE_SIZE = 50;
static constexpr int MAX_FETCH_ROUNDS = 5;
static constexpr size_t MAX_SHOWN_HISTORY = 200;
void shuffle();
void sortChronological(bool reverse);

26
src/web_server.cpp
View File

@@ -157,6 +157,31 @@ void AppWebServer::setupAPIRoutes() {
_power->enterDeepSleep();
});
// Test Immich connectivity
_server.on("/api/test-connection", HTTP_POST,
[this](AsyncWebServerRequest* req) {
Settings s = _settings->get();
Serial.printf("[web] Testing connection to: %s (key: %s...)\n",
s.immich_url.c_str(),
s.immich_key.substring(0, 8).c_str());
// Reinitialize immich client with current settings
_immich->begin(s.immich_url, s.immich_key);
auto albums = _immich->fetchAlbums();
JsonDocument doc;
if (!albums.empty()) {
doc["ok"] = true;
doc["msg"] = String("Connected! Found ") + String(albums.size()) + " albums.";
} else {
doc["ok"] = false;
doc["msg"] = "Failed to connect. Check URL and API key.";
}
String response;
serializeJson(doc, response);
req->send(200, "application/json", response);
});
_server.on("/api/action/reboot", HTTP_POST,
[](AsyncWebServerRequest* req) {
req->send(200, "application/json", "{\"ok\":true,\"msg\":\"Rebooting...\"}");
@@ -259,6 +284,7 @@ void AppWebServer::handleGetSettings(AsyncWebServerRequest* request) {
doc["meta_pos"] = static_cast<uint8_t>(s.meta_pos);
doc["led_brightness"] = s.led_brightness;
doc["immich_url"] = s.immich_url;
doc["immich_key"] = s.immich_key;
doc["albums_json"] = s.albums_json;
String response;