From 4ae0e61610ddae559fc1373e0598853724e0846c Mon Sep 17 00:00:00 2001 From: cottongin Date: Mon, 3 Aug 2026 14:39:55 -0400 Subject: [PATCH] feat: fit-contain letterbox + smart queue refill - Replace center-crop with fit-contain (no photo cropping) - Letterbox bars filled with adjacent edge average color - Queue fetches in batches until 50 usable assets - Auto-refill queue on exhaustion, dedup via shown history - Filter RAW files from asset selection Co-authored-by: Cursor --- data/app.js | 43 +- platformio.ini | 1 + src/image_pipeline.cpp | 871 +++++++++++++++++++++++++++-------------- src/image_pipeline.h | 32 +- src/immich_client.cpp | 204 +++++++++- src/immich_client.h | 3 + src/main.cpp | 107 +++-- src/photo_queue.cpp | 100 +++-- src/photo_queue.h | 9 + src/web_server.cpp | 26 ++ 10 files changed, 985 insertions(+), 411 deletions(-) diff --git a/data/app.js b/data/app.js index d045619..db26a45 100644 --- a/data/app.js +++ b/data/app.js @@ -162,14 +162,53 @@ async function renderDisplay(el) { } async function renderDevice(el) { + const settings = await api('/api/settings'); el.innerHTML = `
-

Device

+

Immich Connection

+
+ + +
+
+ + +
+
+ + +
+
+
+
+

Device Actions

- +
`; + + document.getElementById('saveImmich').onclick = async () => { + const url = document.getElementById('immichUrl').value.trim(); + const key = document.getElementById('immichKey').value.trim(); + if (!url || !key) return toast('URL and API key are required'); + await api('/api/settings', { method: 'POST', body: JSON.stringify({ immich_url: url, immich_key: key }) }); + toast('Immich settings saved. Reboot to apply.'); + }; + + document.getElementById('testConn').onclick = async () => { + const status = document.getElementById('connStatus'); + status.textContent = 'Testing...'; + status.style.color = ''; + try { + const result = await api('/api/test-connection', { method: 'POST' }); + status.textContent = result.msg; + status.style.color = result.ok ? '#2e7d32' : '#c62828'; + } catch (e) { + status.textContent = 'Request failed: ' + e.message; + status.style.color = '#c62828'; + } + }; } async function renderFirmware(el) { diff --git a/platformio.ini b/platformio.ini index 1667bce..104a7c8 100644 --- a/platformio.ini +++ b/platformio.ini @@ -23,6 +23,7 @@ lib_deps = mathieucarbou/ESP Async WebServer @ ^3.0.6 bblanchon/ArduinoJson @ ^7.0.0 ricmoo/QRCode @ ^0.0.1 + bitbank2/JPEGDEC @ ^1.8.4 lib_ignore = WebServer diff --git a/src/image_pipeline.cpp b/src/image_pipeline.cpp index 1d88439..6a75f5e 100644 --- a/src/image_pipeline.cpp +++ b/src/image_pipeline.cpp @@ -1,248 +1,185 @@ #include "image_pipeline.h" #include -#include #include #include +#include -// Spectra 6 palette -static const uint8_t PALETTE_RGB[6][3] = { - {PALETTE_BLACK_R, PALETTE_BLACK_G, PALETTE_BLACK_B}, - {PALETTE_WHITE_R, PALETTE_WHITE_G, PALETTE_WHITE_B}, - {PALETTE_RED_R, PALETTE_RED_G, PALETTE_RED_B}, - {PALETTE_GREEN_R, PALETTE_GREEN_G, PALETTE_GREEN_B}, - {PALETTE_BLUE_R, PALETTE_BLUE_G, PALETTE_BLUE_B}, - {PALETTE_YELLOW_R, PALETTE_YELLOW_G, PALETTE_YELLOW_B} +// Spectra 6 calibrated palette (measured display appearance from epdoptimize) +// These represent what the display ACTUALLY shows, used for dithering decisions +static const uint8_t PALETTE_CALIBRATED[6][3] = { + {0x1F, 0x22, 0x26}, // Black -> appears as dark gray + {0xB9, 0xC7, 0xC9}, // White -> appears as light gray-blue + {0x62, 0x20, 0x1E}, // Red -> appears as dark red/brown + {0x35, 0x56, 0x3A}, // Green -> appears as dark forest green + {0x23, 0x3F, 0x8E}, // Blue -> appears as dark navy + {0xC1, 0xBB, 0x1E} // Yellow -> appears as olive/mustard }; -struct JpegMemReader { - const uint8_t* data; - size_t size; - size_t pos; -}; +// JPEGDEC draw callback: receives decoded MCU blocks and writes RGB888 to buffer +static int jpegDrawCallback(JPEGDRAW* pDraw) { + auto* ctx = static_cast(pDraw->pUser); + if (ctx == nullptr || ctx->rgbBuffer == nullptr) return 0; -static uint32_t jpegMemInput(void* device, uint8_t* buf, uint32_t len) { - auto* ctx = static_cast(device); - if (ctx->pos >= ctx->size) { - return 0; + for (int y = 0; y < pDraw->iHeight; y++) { + int dstRow = pDraw->y + y; + if (dstRow < 0 || dstRow >= ctx->bufferHeight) continue; + + for (int x = 0; x < pDraw->iWidthUsed; x++) { + int dstCol = pDraw->x + x; + if (dstCol < 0 || dstCol >= ctx->bufferWidth) continue; + + // Convert RGB565 to RGB888 + uint16_t pixel = pDraw->pPixels[y * pDraw->iWidth + x]; + uint8_t r = (pixel >> 11) << 3; + uint8_t g = ((pixel >> 5) & 0x3F) << 2; + uint8_t b = (pixel & 0x1F) << 3; + + size_t dstIdx = ((size_t)dstRow * ctx->bufferWidth + dstCol) * 3; + ctx->rgbBuffer[dstIdx] = r; + ctx->rgbBuffer[dstIdx + 1] = g; + ctx->rgbBuffer[dstIdx + 2] = b; + } } - uint32_t remain = static_cast(ctx->size - ctx->pos); - uint32_t n = len < remain ? len : remain; - memcpy(buf, ctx->data + ctx->pos, n); - ctx->pos += n; - return n; + return 1; } -ProcessedImage ImagePipeline::process(uint8_t* jpegData, size_t jpegSize) { - ProcessedImage result = {nullptr, DISPLAY_WIDTH, DISPLAY_HEIGHT, false}; +// Contrast/saturation enhancement applied before dithering +static void enhanceContrast(uint8_t* rgb, size_t pixelCount) { + static constexpr float CONTRAST = 1.25f; + static constexpr float SATURATION = 1.15f; + static constexpr float MID = 128.0f; - // Decode JPEG - uint16_t srcW, srcH; - uint8_t* rgb = decodeJpeg(jpegData, jpegSize, &srcW, &srcH); - if (rgb == nullptr) { - Serial.println("[pipeline] JPEG decode failed"); - return result; + for (size_t i = 0; i < pixelCount; i++) { + size_t idx = i * 3; + float r = rgb[idx]; + float g = rgb[idx + 1]; + float b = rgb[idx + 2]; + + r = (r - MID) * CONTRAST + MID; + g = (g - MID) * CONTRAST + MID; + b = (b - MID) * CONTRAST + MID; + + float lum = 0.299f * r + 0.587f * g + 0.114f * b; + r = lum + (r - lum) * SATURATION; + g = lum + (g - lum) * SATURATION; + b = lum + (b - lum) * SATURATION; + + rgb[idx] = (uint8_t)fminf(fmaxf(r, 0.0f), 255.0f); + rgb[idx + 1] = (uint8_t)fminf(fmaxf(g, 0.0f), 255.0f); + rgb[idx + 2] = (uint8_t)fminf(fmaxf(b, 0.0f), 255.0f); } - - Serial.printf("[pipeline] Decoded: %dx%d\n", srcW, srcH); - - // Calculate crop region (fill-crop to display aspect ratio) - uint16_t cropX, cropY, cropW, cropH; - centerCrop(rgb, srcW, srcH, DISPLAY_WIDTH, DISPLAY_HEIGHT, - &cropX, &cropY, &cropW, &cropH); - - // Resize cropped region to display dimensions - uint8_t* cropped = (uint8_t*)ps_malloc(cropW * cropH * 3); - if (cropped == nullptr) { - free(rgb); - return result; - } - - // Extract crop region - for (uint16_t y = 0; y < cropH; y++) { - memcpy(cropped + y * cropW * 3, - rgb + ((cropY + y) * srcW + cropX) * 3, - cropW * 3); - } - free(rgb); - - // Resize to display dimensions - uint8_t* resized = resize(cropped, cropW, cropH, DISPLAY_WIDTH, DISPLAY_HEIGHT); - free(cropped); - - if (resized == nullptr) { - return result; - } - - // Dither to 6-color palette - uint8_t* dithered = dither(resized, DISPLAY_WIDTH, DISPLAY_HEIGHT); - free(resized); - - if (dithered == nullptr) { - return result; - } - - result.framebuffer = dithered; - result.valid = true; - Serial.println("[pipeline] Processing complete"); - return result; } -ProcessedImage ImagePipeline::processPortraitPair(uint8_t* jpeg1Data, size_t jpeg1Size, - uint8_t* jpeg2Data, size_t jpeg2Size) { - ProcessedImage result = {nullptr, DISPLAY_WIDTH, DISPLAY_HEIGHT, false}; +// Average a single edge of the fitted image (4 rows or columns deep) +static constexpr int EDGE_DEPTH = 4; - // Each portrait gets half the width minus gap - uint16_t portraitW = (DISPLAY_WIDTH - PORTRAIT_GAP_PX) / 2; - uint16_t portraitH = DISPLAY_HEIGHT; +struct EdgeColor { uint8_t r, g, b; }; - // Allocate combined RGB buffer - uint8_t* combined = (uint8_t*)ps_calloc(DISPLAY_WIDTH * DISPLAY_HEIGHT * 3, 1); - if (combined == nullptr) return result; +static EdgeColor averageEdge(const uint8_t* rgb, uint16_t w, uint16_t h, + int side) { + // side: 0=top, 1=bottom, 2=left, 3=right + uint32_t sumR = 0, sumG = 0, sumB = 0, count = 0; - // Process first portrait - uint16_t src1W, src1H; - uint8_t* rgb1 = decodeJpeg(jpeg1Data, jpeg1Size, &src1W, &src1H); - if (rgb1 != nullptr) { - uint16_t cropX, cropY, cropW, cropH; - centerCrop(rgb1, src1W, src1H, portraitW, portraitH, - &cropX, &cropY, &cropW, &cropH); - - uint8_t* cropped1 = (uint8_t*)ps_malloc(cropW * cropH * 3); - if (cropped1) { - for (uint16_t y = 0; y < cropH; y++) { - memcpy(cropped1 + y * cropW * 3, - rgb1 + ((cropY + y) * src1W + cropX) * 3, cropW * 3); - } - uint8_t* resized1 = resize(cropped1, cropW, cropH, portraitW, portraitH); - free(cropped1); - if (resized1) { - // Copy into left side of combined buffer - for (uint16_t y = 0; y < portraitH; y++) { - memcpy(combined + y * DISPLAY_WIDTH * 3, - resized1 + y * portraitW * 3, portraitW * 3); + switch (side) { + case 0: { // top rows + int rows = min((int)h, EDGE_DEPTH); + for (int y = 0; y < rows; y++) + for (int x = 0; x < w; x++) { + size_t idx = ((size_t)y * w + x) * 3; + sumR += rgb[idx]; sumG += rgb[idx+1]; sumB += rgb[idx+2]; count++; } - free(resized1); - } + break; } - free(rgb1); - } - - // Process second portrait - uint16_t src2W, src2H; - uint8_t* rgb2 = decodeJpeg(jpeg2Data, jpeg2Size, &src2W, &src2H); - if (rgb2 != nullptr) { - uint16_t cropX, cropY, cropW, cropH; - centerCrop(rgb2, src2W, src2H, portraitW, portraitH, - &cropX, &cropY, &cropW, &cropH); - - uint8_t* cropped2 = (uint8_t*)ps_malloc(cropW * cropH * 3); - if (cropped2) { - for (uint16_t y = 0; y < cropH; y++) { - memcpy(cropped2 + y * cropW * 3, - rgb2 + ((cropY + y) * src2W + cropX) * 3, cropW * 3); - } - uint8_t* resized2 = resize(cropped2, cropW, cropH, portraitW, portraitH); - free(cropped2); - if (resized2) { - // Copy into right side of combined buffer - uint16_t offsetX = portraitW + PORTRAIT_GAP_PX; - for (uint16_t y = 0; y < portraitH; y++) { - memcpy(combined + (y * DISPLAY_WIDTH + offsetX) * 3, - resized2 + y * portraitW * 3, portraitW * 3); + case 1: { // bottom rows + int rows = min((int)h, EDGE_DEPTH); + for (int y = h - rows; y < h; y++) + for (int x = 0; x < w; x++) { + size_t idx = ((size_t)y * w + x) * 3; + sumR += rgb[idx]; sumG += rgb[idx+1]; sumB += rgb[idx+2]; count++; } - free(resized2); + break; + } + case 2: { // left columns + int cols = min((int)w, EDGE_DEPTH); + for (int y = 0; y < h; y++) + for (int x = 0; x < cols; x++) { + size_t idx = ((size_t)y * w + x) * 3; + sumR += rgb[idx]; sumG += rgb[idx+1]; sumB += rgb[idx+2]; count++; + } + break; + } + case 3: { // right columns + int cols = min((int)w, EDGE_DEPTH); + for (int y = 0; y < h; y++) + for (int x = w - cols; x < w; x++) { + size_t idx = ((size_t)y * w + x) * 3; + sumR += rgb[idx]; sumG += rgb[idx+1]; sumB += rgb[idx+2]; count++; + } + break; + } + } + + if (count == 0) return {0, 0, 0}; + return {(uint8_t)(sumR / count), (uint8_t)(sumG / count), (uint8_t)(sumB / count)}; +} + +// Fill letterbox bars in the final buffer, matching each bar to its adjacent photo edge. +// fitted image is placed at (offsetX, offsetY) with size (fitW x fitH) inside (targetW x targetH). +static void fillLetterbox(uint8_t* final_buf, uint16_t targetW, uint16_t targetH, + const uint8_t* fitted, uint16_t fitW, uint16_t fitH, + uint16_t offsetX, uint16_t offsetY) { + if (offsetY > 0) { + // Horizontal letterbox (top and bottom bars) + EdgeColor top = averageEdge(fitted, fitW, fitH, 0); + EdgeColor bot = averageEdge(fitted, fitW, fitH, 1); + + // Fill top bar + for (uint16_t y = 0; y < offsetY; y++) + for (uint16_t x = 0; x < targetW; x++) { + size_t idx = ((size_t)y * targetW + x) * 3; + final_buf[idx] = top.r; final_buf[idx+1] = top.g; final_buf[idx+2] = top.b; } - } - free(rgb2); + // Fill bottom bar + uint16_t botStart = offsetY + fitH; + for (uint16_t y = botStart; y < targetH; y++) + for (uint16_t x = 0; x < targetW; x++) { + size_t idx = ((size_t)y * targetW + x) * 3; + final_buf[idx] = bot.r; final_buf[idx+1] = bot.g; final_buf[idx+2] = bot.b; + } + + Serial.printf("[pipeline] Letterbox TB: top=#%02X%02X%02X bot=#%02X%02X%02X\n", + top.r, top.g, top.b, bot.r, bot.g, bot.b); } - // Dither combined buffer - uint8_t* dithered = dither(combined, DISPLAY_WIDTH, DISPLAY_HEIGHT); - free(combined); + if (offsetX > 0) { + // Vertical letterbox (left and right bars) + EdgeColor left = averageEdge(fitted, fitW, fitH, 2); + EdgeColor right = averageEdge(fitted, fitW, fitH, 3); - if (dithered == nullptr) return result; + // Fill left bar + for (uint16_t y = 0; y < targetH; y++) + for (uint16_t x = 0; x < offsetX; x++) { + size_t idx = ((size_t)y * targetW + x) * 3; + final_buf[idx] = left.r; final_buf[idx+1] = left.g; final_buf[idx+2] = left.b; + } + // Fill right bar + uint16_t rightStart = offsetX + fitW; + for (uint16_t y = 0; y < targetH; y++) + for (uint16_t x = rightStart; x < targetW; x++) { + size_t idx = ((size_t)y * targetW + x) * 3; + final_buf[idx] = right.r; final_buf[idx+1] = right.g; final_buf[idx+2] = right.b; + } - result.framebuffer = dithered; - result.valid = true; - return result; + Serial.printf("[pipeline] Letterbox LR: left=#%02X%02X%02X right=#%02X%02X%02X\n", + left.r, left.g, left.b, right.r, right.g, right.b); + } } -void ImagePipeline::freeImage(ProcessedImage& img) { - if (img.framebuffer) { - free(img.framebuffer); - img.framebuffer = nullptr; - } - img.valid = false; -} +// Bilinear resize from src (srcW x srcH) to dst (dstW x dstH) +static void bilinearResize(const uint8_t* src, uint16_t srcW, uint16_t srcH, + uint8_t* dst, uint16_t dstW, uint16_t dstH) { + if (dstW <= 1 || dstH <= 1 || srcW <= 1 || srcH <= 1) return; -uint8_t* ImagePipeline::decodeJpeg(uint8_t* data, size_t size, - uint16_t* outWidth, uint16_t* outHeight) { - JpegMemReader reader = {data, size, 0}; - lgfxJdec jdec; - static constexpr uint16_t poolSize = 3900; - uint8_t* pool = (uint8_t*)malloc(poolSize); - if (pool == nullptr) { - return nullptr; - } - - JRESULT prep = lgfx_jd_prepare(&jdec, jpegMemInput, pool, poolSize, &reader); - if (prep != JDR_OK) { - free(pool); - return nullptr; - } - - uint16_t w = jdec.width; - uint16_t h = jdec.height; - free(pool); - - lgfx::LGFX_Sprite sprite; - sprite.setPsram(true); - sprite.setColorDepth(lgfx::color_depth_t::rgb888_3Byte); - - if (!sprite.createSprite(w, h)) { - return nullptr; - } - - if (!sprite.drawJpg(data, size, 0, 0)) { - sprite.deleteSprite(); - return nullptr; - } - - size_t bufSize = static_cast(w) * h * 3; - uint8_t* rgb = (uint8_t*)ps_malloc(bufSize); - if (rgb == nullptr) { - sprite.deleteSprite(); - return nullptr; - } - - for (uint16_t y = 0; y < h; y++) { - for (uint16_t x = 0; x < w; x++) { - lgfx::bgr888_t color = sprite.readPixelRGB(x, y); - size_t idx = (static_cast(y) * w + x) * 3; - rgb[idx] = color.r; - rgb[idx + 1] = color.g; - rgb[idx + 2] = color.b; - } - } - - sprite.deleteSprite(); - *outWidth = w; - *outHeight = h; - return rgb; -} - -uint8_t* ImagePipeline::resize(uint8_t* rgb, uint16_t srcW, uint16_t srcH, - uint16_t dstW, uint16_t dstH) { - size_t bufSize = dstW * dstH * 3; - uint8_t* dst = (uint8_t*)ps_malloc(bufSize); - if (dst == nullptr) return nullptr; - - if (dstW <= 1 || dstH <= 1 || srcW == 0 || srcH == 0) { - free(dst); - return nullptr; - } - - // Bilinear interpolation float xRatio = (float)(srcW - 1) / (float)(dstW - 1); float yRatio = (float)(srcH - 1) / (float)(dstH - 1); @@ -259,105 +196,465 @@ uint8_t* ImagePipeline::resize(uint8_t* rgb, uint16_t srcW, uint16_t srcH, float xFrac = srcX - x0; for (int c = 0; c < 3; c++) { - float top = rgb[(y0 * srcW + x0) * 3 + c] * (1 - xFrac) + - rgb[(y0 * srcW + x1) * 3 + c] * xFrac; - float bot = rgb[(y1 * srcW + x0) * 3 + c] * (1 - xFrac) + - rgb[(y1 * srcW + x1) * 3 + c] * xFrac; + float top = src[(y0 * srcW + x0) * 3 + c] * (1 - xFrac) + + src[(y0 * srcW + x1) * 3 + c] * xFrac; + float bot = src[(y1 * srcW + x0) * 3 + c] * (1 - xFrac) + + src[(y1 * srcW + x1) * 3 + c] * xFrac; float val = top * (1 - yFrac) + bot * yFrac; dst[(y * dstW + x) * 3 + c] = (uint8_t)(val + 0.5f); } } } - - return dst; } -void ImagePipeline::centerCrop(uint8_t* rgb, uint16_t srcW, uint16_t srcH, - uint16_t targetW, uint16_t targetH, - uint16_t* cropX, uint16_t* cropY, - uint16_t* cropW, uint16_t* cropH) { - (void)rgb; - float targetAspect = (float)targetW / (float)targetH; - float srcAspect = (float)srcW / (float)srcH; +// Parse JPEG markers properly to find SOF and extract dimensions. +static bool parseJpegHeader(const uint8_t* data, size_t size, + uint16_t* width, uint16_t* height, bool* isProgressive) { + *width = 0; + *height = 0; + *isProgressive = false; - if (srcAspect > targetAspect) { - // Source is wider — crop sides - *cropH = srcH; - *cropW = (uint16_t)(srcH * targetAspect); - *cropX = (srcW - *cropW) / 2; - *cropY = 0; - } else { - // Source is taller — crop top/bottom - *cropW = srcW; - *cropH = (uint16_t)(srcW / targetAspect); - *cropX = 0; - *cropY = (srcH - *cropH) / 2; + if (size < 2 || data[0] != 0xFF || data[1] != 0xD8) return false; + + size_t pos = 2; + while (pos + 4 < size) { + if (data[pos] != 0xFF) { pos++; continue; } + while (pos < size && data[pos] == 0xFF) pos++; + if (pos >= size) break; + + uint8_t marker = data[pos++]; + + if (marker >= 0xC0 && marker <= 0xCF && marker != 0xC4 && marker != 0xCC) { + if (pos + 7 > size) break; + *height = (data[pos + 3] << 8) | data[pos + 4]; + *width = (data[pos + 5] << 8) | data[pos + 6]; + *isProgressive = (marker == 0xC2); + return true; + } + + if (marker == 0xDA) break; + + if (pos + 1 >= size) break; + uint16_t segLen = (data[pos] << 8) | data[pos + 1]; + pos += segLen; } + return false; } -uint8_t* ImagePipeline::dither(uint8_t* rgb, uint16_t width, uint16_t height) { - size_t pixelCount = width * height; - uint8_t* output = (uint8_t*)ps_malloc(pixelCount); - if (output == nullptr) return nullptr; +// Fallback decoder using LGFX's built-in TJPGD +static uint8_t* decodeFallbackLGFX(uint8_t* data, size_t size, + uint16_t targetW, uint16_t targetH, + uint16_t imgW, uint16_t imgH, + uint16_t* outW, uint16_t* outH) { + // Calculate decode dimensions (cap at 960px longest side) + static constexpr uint16_t MAX_DECODE = 960; + uint16_t decodeW = imgW; + uint16_t decodeH = imgH; + uint16_t maxSide = max(imgW, imgH); - // Work buffer with int16 to handle error overflow - int16_t* work = (int16_t*)ps_malloc(pixelCount * 3 * sizeof(int16_t)); - if (work == nullptr) { - free(output); + if (maxSide > MAX_DECODE) { + float scale = (float)MAX_DECODE / (float)maxSide; + decodeW = (uint16_t)(imgW * scale); + decodeH = (uint16_t)(imgH * scale); + } + + Serial.printf("[pipeline/lgfx] Decoding %dx%d -> %dx%d sprite\n", + imgW, imgH, decodeW, decodeH); + + lgfx::LGFX_Sprite sprite; + sprite.setPsram(true); + sprite.setColorDepth(lgfx::color_depth_t::rgb888_3Byte); + + if (!sprite.createSprite(decodeW, decodeH)) { + Serial.println("[pipeline/lgfx] Sprite alloc failed"); return nullptr; } - // Copy to work buffer - for (size_t i = 0; i < pixelCount * 3; i++) { - work[i] = rgb[i]; + sprite.fillSprite(0); + float scaleX = (float)decodeW / (float)imgW; + float scaleY = (float)decodeH / (float)imgH; + + if (!sprite.drawJpg(data, size, 0, 0, decodeW, decodeH, 0, 0, scaleX, scaleY)) { + Serial.println("[pipeline/lgfx] drawJpg failed (possibly progressive)"); + sprite.deleteSprite(); + return nullptr; + } + + // Fit-contain: scale to fit entirely within target + float fitScaleW = (float)targetW / (float)decodeW; + float fitScaleH = (float)targetH / (float)decodeH; + float fitScale = fminf(fitScaleW, fitScaleH); + uint16_t fitW = (uint16_t)(decodeW * fitScale); + uint16_t fitH = (uint16_t)(decodeH * fitScale); + + // Extract full decoded image from sprite + size_t decodeBufSize = (size_t)decodeW * decodeH * 3; + uint8_t* decoded = (uint8_t*)ps_malloc(decodeBufSize); + if (decoded == nullptr) { + sprite.deleteSprite(); + return nullptr; + } + for (uint16_t y = 0; y < decodeH; y++) { + for (uint16_t x = 0; x < decodeW; x++) { + lgfx::bgr888_t color = sprite.readPixelRGB(x, y); + size_t idx = ((size_t)y * decodeW + x) * 3; + decoded[idx] = color.r; + decoded[idx + 1] = color.g; + decoded[idx + 2] = color.b; + } + } + sprite.deleteSprite(); + + // Resize to fit dimensions + uint8_t* fitted = (uint8_t*)ps_malloc((size_t)fitW * fitH * 3); + if (fitted == nullptr) { + free(decoded); + return nullptr; + } + bilinearResize(decoded, decodeW, decodeH, fitted, fitW, fitH); + free(decoded); + + // Allocate final target buffer (zeroed) + size_t finalSize = (size_t)targetW * targetH * 3; + uint8_t* final_buf = (uint8_t*)ps_calloc(targetW * targetH, 3); + if (final_buf == nullptr) { + free(fitted); + return nullptr; + } + + // Center the fitted image in the final buffer + uint16_t offsetX = (targetW - fitW) / 2; + uint16_t offsetY = (targetH - fitH) / 2; + for (uint16_t y = 0; y < fitH; y++) { + memcpy(final_buf + ((size_t)(offsetY + y) * targetW + offsetX) * 3, + fitted + (size_t)y * fitW * 3, + fitW * 3); + } + + // Fill letterbox bars with per-edge average colors + fillLetterbox(final_buf, targetW, targetH, fitted, fitW, fitH, offsetX, offsetY); + free(fitted); + + Serial.printf("[pipeline/lgfx] Fit %dx%d into %dx%d (offset %d,%d)\n", + fitW, fitH, targetW, targetH, offsetX, offsetY); + + *outW = targetW; + *outH = targetH; + return final_buf; +} + +uint8_t* ImagePipeline::decodeAndFit(uint8_t* data, size_t size, + uint16_t targetW, uint16_t targetH, + uint16_t* outW, uint16_t* outH) { + // Parse JPEG header to get dimensions and detect progressive + uint16_t headerW = 0, headerH = 0; + bool progressive = false; + if (parseJpegHeader(data, size, &headerW, &headerH, &progressive)) { + Serial.printf("[pipeline] Header: %dx%d %s\n", headerW, headerH, + progressive ? "progressive" : "baseline"); + } else { + Serial.println("[pipeline] Could not parse JPEG header"); + } + + if (progressive) { + Serial.println("[pipeline] Progressive JPEG — skipping (server still reprocessing?)"); + return nullptr; + } + + // Primary path: JPEGDEC + JPEGDEC jpeg; + + if (!jpeg.openRAM(data, (int)size, jpegDrawCallback)) { + int err = jpeg.getLastError(); + Serial.printf("[pipeline] JPEGDEC openRAM failed (err %d), trying LGFX fallback\n", err); + if (headerW > 0 && headerH > 0) { + return decodeFallbackLGFX(data, size, targetW, targetH, + headerW, headerH, outW, outH); + } + return nullptr; + } + + int imgW = jpeg.getWidth(); + int imgH = jpeg.getHeight(); + Serial.printf("[pipeline] JPEG: %dx%d, type=baseline\n", imgW, imgH); + + // Determine decode scale (reduce large images before fit) + int decodeOptions = 0; + int decodeW = imgW; + int decodeH = imgH; + + if (imgW > targetW * 4 && imgH > targetH * 4) { + decodeOptions = JPEG_SCALE_QUARTER; + decodeW = imgW / 4; + decodeH = imgH / 4; + Serial.printf("[pipeline] Using 1/4 scale: %dx%d\n", decodeW, decodeH); + } else if (imgW > targetW * 2 && imgH > targetH * 2) { + decodeOptions = JPEG_SCALE_HALF; + decodeW = imgW / 2; + decodeH = imgH / 2; + Serial.printf("[pipeline] Using 1/2 scale: %dx%d\n", decodeW, decodeH); + } + + // Decode full image (no cropping — we want the whole photo) + size_t decodeBufSize = (size_t)decodeW * decodeH * 3; + uint8_t* decodedRgb = (uint8_t*)ps_malloc(decodeBufSize); + if (decodedRgb == nullptr) { + Serial.printf("[pipeline] Failed to allocate %u KB for decode buffer\n", + (unsigned)(decodeBufSize / 1024)); + jpeg.close(); + return nullptr; + } + memset(decodedRgb, 0, decodeBufSize); + + DecodeContext ctx; + ctx.rgbBuffer = decodedRgb; + ctx.bufferWidth = (uint16_t)decodeW; + ctx.bufferHeight = (uint16_t)decodeH; + + jpeg.setUserPointer(&ctx); + jpeg.setPixelType(RGB565_LITTLE_ENDIAN); + + if (!jpeg.decode(0, 0, decodeOptions)) { + Serial.printf("[pipeline] JPEGDEC decode failed (err %d), trying LGFX fallback\n", + jpeg.getLastError()); + free(decodedRgb); + jpeg.close(); + if (headerW > 0 && headerH > 0) { + return decodeFallbackLGFX(data, size, targetW, targetH, + headerW, headerH, outW, outH); + } + return nullptr; + } + jpeg.close(); + + // Fit-contain: scale to fit entirely within target dimensions + float fitScaleW = (float)targetW / (float)decodeW; + float fitScaleH = (float)targetH / (float)decodeH; + float fitScale = fminf(fitScaleW, fitScaleH); + uint16_t fitW = (uint16_t)(decodeW * fitScale); + uint16_t fitH = (uint16_t)(decodeH * fitScale); + + Serial.printf("[pipeline] Fit: %dx%d -> %dx%d (scale %.2f)\n", + decodeW, decodeH, fitW, fitH, fitScale); + + // Resize decoded image to fit dimensions + uint8_t* fitted = (uint8_t*)ps_malloc((size_t)fitW * fitH * 3); + if (fitted == nullptr) { + Serial.println("[pipeline] Fit alloc failed"); + free(decodedRgb); + return nullptr; + } + bilinearResize(decodedRgb, (uint16_t)decodeW, (uint16_t)decodeH, fitted, fitW, fitH); + free(decodedRgb); + + // Allocate final target-sized buffer (zeroed) + size_t finalSize = (size_t)targetW * targetH * 3; + uint8_t* final_buf = (uint8_t*)ps_calloc(targetW * targetH, 3); + if (final_buf == nullptr) { + Serial.println("[pipeline] Final buffer alloc failed"); + free(fitted); + return nullptr; + } + + // Center the fitted image into the final buffer + uint16_t offsetX = (targetW - fitW) / 2; + uint16_t offsetY = (targetH - fitH) / 2; + for (uint16_t y = 0; y < fitH; y++) { + memcpy(final_buf + ((size_t)(offsetY + y) * targetW + offsetX) * 3, + fitted + (size_t)y * fitW * 3, + fitW * 3); + } + + // Fill letterbox bars — each bar matches the adjacent photo edge + fillLetterbox(final_buf, targetW, targetH, fitted, fitW, fitH, offsetX, offsetY); + free(fitted); + + *outW = targetW; + *outH = targetH; + return final_buf; +} + +ProcessedImage ImagePipeline::process(uint8_t* jpegData, size_t jpegSize) { + ProcessedImage result = {nullptr, DISPLAY_WIDTH, DISPLAY_HEIGHT, false}; + + uint16_t outW, outH; + uint8_t* rgb = decodeAndFit(jpegData, jpegSize, DISPLAY_WIDTH, DISPLAY_HEIGHT, &outW, &outH); + if (rgb == nullptr) { + Serial.println("[pipeline] Decode/fit failed"); + return result; + } + + Serial.printf("[pipeline] Got %dx%d fitted RGB\n", outW, outH); + + // Enhance contrast/saturation for e-ink readability + enhanceContrast(rgb, (size_t)outW * outH); + + // Row-by-row Floyd-Steinberg dithering to 6-color palette + uint8_t* dithered = ditherRowByRow(rgb, outW, outH); + free(rgb); + + if (dithered == nullptr) { + return result; + } + + result.framebuffer = dithered; + result.width = outW; + result.height = outH; + result.valid = true; + Serial.printf("[pipeline] Processing complete (%dx%d)\n", outW, outH); + return result; +} + +ProcessedImage ImagePipeline::processPortraitPair(uint8_t* jpeg1Data, size_t jpeg1Size, + uint8_t* jpeg2Data, size_t jpeg2Size) { + ProcessedImage result = {nullptr, DISPLAY_WIDTH, DISPLAY_HEIGHT, false}; + + uint16_t portraitW = (DISPLAY_WIDTH - PORTRAIT_GAP_PX) / 2; + uint16_t portraitH = DISPLAY_HEIGHT; + + uint16_t out1W, out1H; + uint8_t* rgb1 = decodeAndFit(jpeg1Data, jpeg1Size, portraitW, portraitH, &out1W, &out1H); + + uint16_t out2W, out2H; + uint8_t* rgb2 = decodeAndFit(jpeg2Data, jpeg2Size, portraitW, portraitH, &out2W, &out2H); + + // Allocate combined RGB buffer + uint8_t* combined = (uint8_t*)ps_calloc(DISPLAY_WIDTH * DISPLAY_HEIGHT, 3); + if (combined == nullptr) { + if (rgb1) free(rgb1); + if (rgb2) free(rgb2); + return result; + } + + if (rgb1 != nullptr) { + for (uint16_t y = 0; y < out1H; y++) { + memcpy(combined + y * DISPLAY_WIDTH * 3, + rgb1 + y * out1W * 3, out1W * 3); + } + free(rgb1); + } + + if (rgb2 != nullptr) { + uint16_t offsetX = portraitW + PORTRAIT_GAP_PX; + for (uint16_t y = 0; y < out2H; y++) { + memcpy(combined + (y * DISPLAY_WIDTH + offsetX) * 3, + rgb2 + y * out2W * 3, out2W * 3); + } + free(rgb2); + } + + enhanceContrast(combined, (size_t)DISPLAY_WIDTH * DISPLAY_HEIGHT); + + uint8_t* dithered = ditherRowByRow(combined, DISPLAY_WIDTH, DISPLAY_HEIGHT); + free(combined); + + if (dithered == nullptr) return result; + + result.framebuffer = dithered; + result.width = DISPLAY_WIDTH; + result.height = DISPLAY_HEIGHT; + result.valid = true; + return result; +} + +void ImagePipeline::freeImage(ProcessedImage& img) { + if (img.framebuffer) { + free(img.framebuffer); + img.framebuffer = nullptr; + } + img.valid = false; +} + +uint8_t* ImagePipeline::ditherRowByRow(uint8_t* rgb, uint16_t width, uint16_t height) { + size_t pixelCount = (size_t)width * height; + uint8_t* output = (uint8_t*)ps_malloc(pixelCount); + if (output == nullptr) { + Serial.println("[pipeline] Dither output alloc failed"); + return nullptr; + } + + size_t rowBytes = (size_t)width * 3 * sizeof(int16_t); + int16_t* errCurrent = (int16_t*)ps_malloc(rowBytes); + int16_t* errNext = (int16_t*)ps_malloc(rowBytes); + if (errCurrent == nullptr || errNext == nullptr) { + Serial.println("[pipeline] Dither error buffer alloc failed"); + free(output); + if (errCurrent) free(errCurrent); + if (errNext) free(errNext); + return nullptr; + } + + // Initialize first row + for (uint16_t x = 0; x < width; x++) { + size_t srcIdx = x * 3; + size_t errIdx = x * 3; + errCurrent[errIdx] = rgb[srcIdx]; + errCurrent[errIdx + 1] = rgb[srcIdx + 1]; + errCurrent[errIdx + 2] = rgb[srcIdx + 2]; } - // Floyd-Steinberg dithering for (uint16_t y = 0; y < height; y++) { + // Prepare next row from source + if (y + 1 < height) { + size_t nextRowOffset = (size_t)(y + 1) * width * 3; + for (uint16_t x = 0; x < width; x++) { + size_t srcIdx = nextRowOffset + x * 3; + size_t errIdx = x * 3; + errNext[errIdx] = rgb[srcIdx]; + errNext[errIdx + 1] = rgb[srcIdx + 1]; + errNext[errIdx + 2] = rgb[srcIdx + 2]; + } + } else { + memset(errNext, 0, rowBytes); + } + for (uint16_t x = 0; x < width; x++) { - size_t idx = (y * width + x) * 3; - int r = constrain(work[idx], 0, 255); - int g = constrain(work[idx + 1], 0, 255); - int b = constrain(work[idx + 2], 0, 255); + size_t errIdx = x * 3; + + int r = constrain(errCurrent[errIdx], 0, 255); + int g = constrain(errCurrent[errIdx + 1], 0, 255); + int b = constrain(errCurrent[errIdx + 2], 0, 255); uint8_t nearest = findNearest(r, g, b); output[y * width + x] = nearest; - int errR = r - PALETTE_RGB[nearest][0]; - int errG = g - PALETTE_RGB[nearest][1]; - int errB = b - PALETTE_RGB[nearest][2]; + int errR = r - PALETTE_CALIBRATED[nearest][0]; + int errG = g - PALETTE_CALIBRATED[nearest][1]; + int errB = b - PALETTE_CALIBRATED[nearest][2]; - // Distribute error if (x + 1 < width) { - size_t ni = (y * width + (x + 1)) * 3; - work[ni] += errR * 7 / 16; - work[ni + 1] += errG * 7 / 16; - work[ni + 2] += errB * 7 / 16; + size_t ni = (x + 1) * 3; + errCurrent[ni] += errR * 7 / 16; + errCurrent[ni + 1] += errG * 7 / 16; + errCurrent[ni + 2] += errB * 7 / 16; + } + if (y + 1 < height && x > 0) { + size_t ni = (x - 1) * 3; + errNext[ni] += errR * 3 / 16; + errNext[ni + 1] += errG * 3 / 16; + errNext[ni + 2] += errB * 3 / 16; } if (y + 1 < height) { - if (x > 0) { - size_t ni = ((y + 1) * width + (x - 1)) * 3; - work[ni] += errR * 3 / 16; - work[ni + 1] += errG * 3 / 16; - work[ni + 2] += errB * 3 / 16; - } - { - size_t ni = ((y + 1) * width + x) * 3; - work[ni] += errR * 5 / 16; - work[ni + 1] += errG * 5 / 16; - work[ni + 2] += errB * 5 / 16; - } - if (x + 1 < width) { - size_t ni = ((y + 1) * width + (x + 1)) * 3; - work[ni] += errR * 1 / 16; - work[ni + 1] += errG * 1 / 16; - work[ni + 2] += errB * 1 / 16; - } + size_t ni = x * 3; + errNext[ni] += errR * 5 / 16; + errNext[ni + 1] += errG * 5 / 16; + errNext[ni + 2] += errB * 5 / 16; + } + if (y + 1 < height && x + 1 < width) { + size_t ni = (x + 1) * 3; + errNext[ni] += errR * 1 / 16; + errNext[ni + 1] += errG * 1 / 16; + errNext[ni + 2] += errB * 1 / 16; } } + + int16_t* tmp = errCurrent; + errCurrent = errNext; + errNext = tmp; } - free(work); + free(errCurrent); + free(errNext); return output; } @@ -365,9 +662,9 @@ uint8_t ImagePipeline::findNearest(int r, int g, int b) { uint8_t best = 0; int bestDist = INT32_MAX; for (int i = 0; i < DISPLAY_COLORS; i++) { - int dr = r - PALETTE_RGB[i][0]; - int dg = g - PALETTE_RGB[i][1]; - int db = b - PALETTE_RGB[i][2]; + int dr = r - PALETTE_CALIBRATED[i][0]; + int dg = g - PALETTE_CALIBRATED[i][1]; + int db = b - PALETTE_CALIBRATED[i][2]; int dist = dr * dr + dg * dg + db * db; if (dist < bestDist) { bestDist = dist; diff --git a/src/image_pipeline.h b/src/image_pipeline.h index 5935cd0..2979a9f 100644 --- a/src/image_pipeline.h +++ b/src/image_pipeline.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include "config.h" struct ProcessedImage { @@ -10,9 +11,16 @@ struct ProcessedImage { bool valid; }; +// Context passed through JPEGDEC's user pointer for the draw callback +struct DecodeContext { + uint8_t* rgbBuffer; // Destination RGB888 buffer in PSRAM + uint16_t bufferWidth; // Width of the output buffer + uint16_t bufferHeight; // Height of the output buffer +}; + class ImagePipeline { public: - // Process single landscape photo + // Process single landscape photo (fit-contain with edge-color letterbox) ProcessedImage process(uint8_t* jpegData, size_t jpegSize); // Process portrait pair (two photos side by side) @@ -22,22 +30,14 @@ public: void freeImage(ProcessedImage& img); private: - // Decode JPEG into RGB888 buffer in PSRAM - uint8_t* decodeJpeg(uint8_t* data, size_t size, uint16_t* outWidth, uint16_t* outHeight); + // Decode JPEG and fit into target dimensions with letterboxing + uint8_t* decodeAndFit(uint8_t* data, size_t size, + uint16_t targetW, uint16_t targetH, + uint16_t* outW, uint16_t* outH); - // Resize RGB buffer to target dimensions (bilinear) - uint8_t* resize(uint8_t* rgb, uint16_t srcW, uint16_t srcH, - uint16_t dstW, uint16_t dstH); + // Row-by-row Floyd-Steinberg dither RGB888 to 6-color palette indices + uint8_t* ditherRowByRow(uint8_t* rgb, uint16_t width, uint16_t height); - // Center-crop to target aspect ratio - void centerCrop(uint8_t* rgb, uint16_t srcW, uint16_t srcH, - uint16_t targetW, uint16_t targetH, - uint16_t* cropX, uint16_t* cropY, - uint16_t* cropW, uint16_t* cropH); - - // Floyd-Steinberg dither RGB888 to 6-color palette indices - uint8_t* dither(uint8_t* rgb, uint16_t width, uint16_t height); - - // Find nearest palette color + // Find nearest palette color (Euclidean distance in RGB space) uint8_t findNearest(int r, int g, int b); }; diff --git a/src/immich_client.cpp b/src/immich_client.cpp index 1361bdb..ab6976f 100644 --- a/src/immich_client.cpp +++ b/src/immich_client.cpp @@ -11,7 +11,13 @@ void ImmichClient::begin(const String& baseUrl, const String& apiKey) { _baseUrl.remove(_baseUrl.length() - 1); } _apiKey = apiKey; - Serial.printf("[immich] Configured: %s\n", _baseUrl.c_str()); + + // Debug: show masked key so we can verify it's non-empty and correct prefix + String maskedKey = _apiKey.length() > 8 + ? _apiKey.substring(0, 8) + "..." + : (_apiKey.length() > 0 ? "***" : "(empty)"); + Serial.printf("[immich] Configured: %s | key: %s (len=%d)\n", + _baseUrl.c_str(), maskedKey.c_str(), _apiKey.length()); } std::vector ImmichClient::fetchAlbums() { @@ -47,12 +53,23 @@ std::vector ImmichClient::fetchAlbums() { std::vector ImmichClient::fetchAlbumAssetIds(const String& albumId) { std::vector ids; String url = buildUrl("/api/albums/" + albumId); - String response = httpGet(url); - if (response.isEmpty()) return ids; + // Album responses can be very large (full asset metadata per photo). + // Use PSRAM-backed fetch to avoid exhausting regular heap. + size_t responseLen = 0; + char* response = httpGetPsram(url, &responseLen); + if (response == nullptr || responseLen == 0) { + Serial.printf("[immich] Album %s: empty response\n", albumId.c_str()); + if (response) free(response); + return ids; + } + + Serial.printf("[immich] Album %s: response %d bytes\n", albumId.c_str(), responseLen); JsonDocument doc; - DeserializationError err = deserializeJson(doc, response); + DeserializationError err = deserializeJson(doc, response, responseLen); + free(response); // Free PSRAM buffer immediately after parsing + if (err) { Serial.printf("[immich] fetchAlbumAssets JSON error: %s\n", err.c_str()); return ids; @@ -60,13 +77,67 @@ std::vector ImmichClient::fetchAlbumAssetIds(const String& albumId) { JsonArray assets = doc["assets"].as(); for (JsonObject asset : assets) { - ids.push_back(asset["id"].as()); + String id = asset["id"].as(); + if (id.length() > 0) { + ids.push_back(id); + } } Serial.printf("[immich] Album %s: %d assets\n", albumId.c_str(), ids.size()); return ids; } +std::vector ImmichClient::fetchRandomAssetIds(int count) { + std::vector ids; + String url = buildUrl("/api/search/random"); + + // Build POST body — filter to images only (excludes videos, audio) + JsonDocument reqDoc; + reqDoc["size"] = count; + reqDoc["type"] = "IMAGE"; + String body; + serializeJson(reqDoc, body); + + String response = httpPost(url, body); + + if (response.isEmpty()) { + Serial.println("[immich] fetchRandom: empty response"); + return ids; + } + + JsonDocument doc; + DeserializationError err = deserializeJson(doc, response); + if (err) { + Serial.printf("[immich] fetchRandom JSON error: %s\n", err.c_str()); + return ids; + } + + JsonArray arr = doc.as(); + int skippedRaw = 0; + for (JsonObject asset : arr) { + String id = asset["id"].as(); + if (id.length() == 0) continue; + + // Filter out RAW/DNG files — their previews work but we prefer actual photos + String filename = asset["originalFileName"] | ""; + filename.toLowerCase(); + if (filename.endsWith(".dng") || filename.endsWith(".raw") || + filename.endsWith(".cr2") || filename.endsWith(".cr3") || + filename.endsWith(".nef") || filename.endsWith(".arw") || + filename.endsWith(".orf") || filename.endsWith(".rw2") || + filename.endsWith(".raf") || filename.endsWith(".srw")) { + skippedRaw++; + continue; + } + + ids.push_back(id); + } + + Serial.printf("[immich] Fetched %d random assets (skipped %d RAW)\n", + ids.size(), skippedRaw); + return ids; +} + std::vector ImmichClient::fetchFavoriteAssetIds() { std::vector ids; String url = buildUrl("/api/assets?isFavorite=true"); @@ -147,14 +218,29 @@ AssetInfo ImmichClient::fetchAssetInfo(const String& assetId) { bool ImmichClient::downloadAsset(const String& assetId, ImageQuality quality, uint8_t** outBuffer, size_t* outSize) { - String url; - if (quality == ImageQuality::Original) { - url = buildUrl("/api/assets/" + assetId + "/original"); - } else { - url = buildUrl("/api/assets/" + assetId + "/thumbnail?size=preview"); + // Always request preview size with JPEG format. + // Server controls actual pixel dimensions via its image config (720px baseline). + // The "quality" setting is kept for future use but we always use preview for now + // to stay within PSRAM budget. + (void)quality; + String url = buildUrl("/api/assets/" + assetId + "/thumbnail?size=preview&format=jpeg"); + + if (!httpGetBinary(url, outBuffer, outSize)) { + return false; } - return httpGetBinary(url, outBuffer, outSize); + // Verify JPEG magic bytes (FF D8 FF) + if (*outSize < 3 || (*outBuffer)[0] != 0xFF || + (*outBuffer)[1] != 0xD8 || (*outBuffer)[2] != 0xFF) { + Serial.printf("[immich] Not JPEG data (magic: %02X %02X %02X)\n", + (*outBuffer)[0], (*outBuffer)[1], (*outBuffer)[2]); + free(*outBuffer); + *outBuffer = nullptr; + *outSize = 0; + return false; + } + + return true; } String ImmichClient::buildUrl(const String& path) { @@ -183,6 +269,102 @@ String ImmichClient::httpGet(const String& url) { return result; } +String ImmichClient::httpPost(const String& url, const String& body) { + WiFiClientSecure client; + client.setInsecure(); + + HTTPClient http; + http.begin(client, url); + http.addHeader("x-api-key", _apiKey); + http.addHeader("Content-Type", "application/json"); + http.setTimeout(30000); + + int code = http.POST(body); + String result = ""; + + if (code == HTTP_CODE_OK) { + result = http.getString(); + } else { + Serial.printf("[immich] HTTP POST %s failed: %d\n", url.c_str(), code); + } + + http.end(); + return result; +} + +char* ImmichClient::httpGetPsram(const String& url, size_t* outLen) { + *outLen = 0; + WiFiClientSecure client; + client.setInsecure(); + + HTTPClient http; + http.begin(client, url); + http.addHeader("x-api-key", _apiKey); + http.setTimeout(60000); + + int code = http.GET(); + if (code != HTTP_CODE_OK) { + Serial.printf("[immich] PSRAM GET %s failed: %d\n", url.c_str(), code); + http.end(); + return nullptr; + } + + int contentLength = http.getSize(); + WiFiClient* stream = http.getStreamPtr(); + + // For chunked responses without Content-Length, read in chunks + if (contentLength <= 0) { + // Read incrementally into PSRAM, up to 4MB max + const size_t maxSize = 4 * 1024 * 1024; + size_t capacity = 64 * 1024; + char* buf = (char*)ps_malloc(capacity); + if (!buf) { + Serial.println("[immich] PSRAM alloc failed"); + http.end(); + return nullptr; + } + size_t total = 0; + while (stream->connected() || stream->available()) { + int avail = stream->available(); + if (avail <= 0) { delay(1); continue; } + if (total + avail >= capacity) { + capacity = min(capacity * 2, maxSize); + char* newBuf = (char*)ps_realloc(buf, capacity); + if (!newBuf) { free(buf); http.end(); return nullptr; } + buf = newBuf; + } + int read = stream->readBytes(buf + total, avail); + total += read; + if (total >= maxSize) break; + } + buf[total] = '\0'; + *outLen = total; + http.end(); + return buf; + } + + // Known content length — single PSRAM allocation + char* buf = (char*)ps_malloc(contentLength + 1); + if (!buf) { + Serial.printf("[immich] PSRAM alloc failed for %d bytes\n", contentLength); + http.end(); + return nullptr; + } + + size_t bytesRead = 0; + while (bytesRead < (size_t)contentLength && (stream->connected() || stream->available())) { + int avail = stream->available(); + if (avail <= 0) { delay(1); continue; } + int read = stream->readBytes(buf + bytesRead, min(avail, (int)(contentLength - bytesRead))); + bytesRead += read; + } + buf[bytesRead] = '\0'; + *outLen = bytesRead; + + http.end(); + return buf; +} + bool ImmichClient::httpGetBinary(const String& url, uint8_t** outBuffer, size_t* outSize) { WiFiClientSecure client; client.setInsecure(); diff --git a/src/immich_client.h b/src/immich_client.h index ca6289a..9dccab1 100644 --- a/src/immich_client.h +++ b/src/immich_client.h @@ -26,6 +26,7 @@ public: void begin(const String& baseUrl, const String& apiKey); std::vector fetchAlbums(); std::vector fetchAlbumAssetIds(const String& albumId); + std::vector fetchRandomAssetIds(int count = 50); std::vector fetchFavoriteAssetIds(); AssetInfo fetchAssetInfo(const String& assetId); bool downloadAsset(const String& assetId, ImageQuality quality, @@ -37,5 +38,7 @@ private: String buildUrl(const String& path); String httpGet(const String& url); + String httpPost(const String& url, const String& body); + char* httpGetPsram(const String& url, size_t* outLen); bool httpGetBinary(const String& url, uint8_t** outBuffer, size_t* outSize); }; diff --git a/src/main.cpp b/src/main.cpp index 17e10d5..28342d9 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -47,6 +48,11 @@ void displayTask(void* param) { if (photoQueue.sync()) { Serial.printf("[display_task] Queue ready: %d photos\n", photoQueue.size()); + // Trigger first photo immediately + if (xSemaphoreTake(stateMutex, pdMS_TO_TICKS(100)) == pdTRUE) { + refreshRequested = true; + xSemaphoreGive(stateMutex); + } } else { displayManager.showMessage("No Photos", "Select albums in web UI"); } @@ -80,16 +86,22 @@ void displayTask(void* param) { } if (shouldRefresh && photoQueue.size() > 0 && wifiManager.isConnected()) { - // Get next photo ID - String assetId; - if (wantsRandom) { - assetId = photoQueue.random(); - } else { - assetId = photoQueue.next(); - } + // Try up to 3 assets in case some fail to decode + static constexpr int MAX_RETRIES = 10; + bool displayed = false; - if (assetId.length() > 0) { - Serial.printf("[display_task] Loading asset: %s\n", assetId.c_str()); + for (int attempt = 0; attempt < MAX_RETRIES && !displayed; attempt++) { + String assetId; + if (wantsRandom) { + assetId = photoQueue.random(); + } else { + assetId = photoQueue.next(); + } + + if (assetId.length() == 0) break; + + Serial.printf("[display_task] Loading asset: %s (attempt %d)\n", + assetId.c_str(), attempt + 1); // Fetch asset info for portrait detection and metadata AssetInfo info = immichClient.fetchAssetInfo(assetId); @@ -100,55 +112,37 @@ void displayTask(void* param) { bool downloaded = immichClient.downloadAsset( assetId, s.img_quality, &jpegBuf, &jpegSize); - if (downloaded && jpegBuf != nullptr) { - ProcessedImage img; - - if (info.isPortrait) { - // Try to find a portrait pair - String pairId = photoQueue.findPortraitPair( - settingsManager.get().queue_cursor); - if (pairId.length() > 0) { - AssetInfo pairInfo = immichClient.fetchAssetInfo(pairId); - if (pairInfo.isPortrait) { - uint8_t* jpeg2Buf = nullptr; - size_t jpeg2Size = 0; - if (immichClient.downloadAsset(pairId, s.img_quality, - &jpeg2Buf, &jpeg2Size)) { - img = imagePipeline.processPortraitPair( - jpegBuf, jpegSize, jpeg2Buf, jpeg2Size); - free(jpeg2Buf); - } else { - img = imagePipeline.process(jpegBuf, jpegSize); - } - } else { - img = imagePipeline.process(jpegBuf, jpegSize); - } - } else { - img = imagePipeline.process(jpegBuf, jpegSize); - } - } else { - img = imagePipeline.process(jpegBuf, jpegSize); - } - - free(jpegBuf); - - if (img.valid) { - displayManager.showImage(img); - // Show metadata overlay if enabled - if (s.meta_flags != 0) { - displayManager.showMetadata(info, s.meta_flags, s.meta_pos); - } - imagePipeline.freeImage(img); - } else { - Serial.println("[display_task] Image processing failed"); - } + if (!downloaded || jpegBuf == nullptr) { + Serial.println("[display_task] Download failed, trying next"); + continue; } - if (xSemaphoreTake(stateMutex, pdMS_TO_TICKS(100)) == pdTRUE) { - lastRefreshTime = millis(); - xSemaphoreGive(stateMutex); + Serial.printf("[display_task] JPEG: %u KB, PSRAM free: %u KB\n", + (unsigned)(jpegSize / 1024), + (unsigned)(heap_caps_get_free_size(MALLOC_CAP_SPIRAM) / 1024)); + + ProcessedImage img = imagePipeline.process(jpegBuf, jpegSize); + free(jpegBuf); + + Serial.printf("[display_task] Post-process PSRAM free: %u KB\n", + (unsigned)(heap_caps_get_free_size(MALLOC_CAP_SPIRAM) / 1024)); + + if (img.valid) { + displayManager.showImage(img); + if (s.meta_flags != 0) { + displayManager.showMetadata(info, s.meta_flags, s.meta_pos); + } + imagePipeline.freeImage(img); + displayed = true; + } else { + Serial.println("[display_task] Processing failed, trying next"); } } + + if (xSemaphoreTake(stateMutex, pdMS_TO_TICKS(100)) == pdTRUE) { + lastRefreshTime = millis(); + xSemaphoreGive(stateMutex); + } } // Yield — check every second @@ -181,6 +175,11 @@ void setup() { Serial.begin(115200); Serial.println("[main] Immich Frame v1.0 booting..."); + Serial.printf("[main] PSRAM: %u KB free / %u KB total\n", + (unsigned)(heap_caps_get_free_size(MALLOC_CAP_SPIRAM) / 1024), + (unsigned)(heap_caps_get_total_size(MALLOC_CAP_SPIRAM) / 1024)); + Serial.printf("[main] Internal RAM: %u KB free\n", + (unsigned)(heap_caps_get_free_size(MALLOC_CAP_INTERNAL) / 1024)); // Create state mutex stateMutex = xSemaphoreCreateMutex(); diff --git a/src/photo_queue.cpp b/src/photo_queue.cpp index 4835249..0e76f62 100644 --- a/src/photo_queue.cpp +++ b/src/photo_queue.cpp @@ -1,6 +1,7 @@ #include "photo_queue.h" #include "immich_client.h" #include +#include #include "config.h" void PhotoQueue::begin(ImmichClient& client, SettingsManager& settings) { @@ -14,37 +15,45 @@ void PhotoQueue::begin(ImmichClient& client, SettingsManager& settings) { bool PhotoQueue::sync() { if (_client == nullptr || _settings == nullptr) return false; + _lastSyncAttempt = millis(); + Settings s = _settings->get(); + + // Fetch in batches until we have TARGET_QUEUE_SIZE usable, non-duplicate IDs std::vector newIds; + int rounds = 0; - // Get selected album IDs - std::vector albumIds = getSelectedAlbumIds(); + Serial.printf("[queue] Fetching until %d usable assets (excluding %d shown)\n", + TARGET_QUEUE_SIZE, _shown.size()); - if (albumIds.empty()) { - // "All photos" mode — fetch from all albums - auto albums = _client->fetchAlbums(); - for (auto& album : albums) { - auto ids = _client->fetchAlbumAssetIds(album.id); - for (auto& id : ids) { - newIds.push_back(id); - } + while (newIds.size() < TARGET_QUEUE_SIZE && rounds < MAX_FETCH_ROUNDS) { + auto batch = _client->fetchRandomAssetIds(50); + if (batch.empty()) { + Serial.printf("[queue] Fetch round %d returned empty\n", rounds + 1); + break; } - } else { - // Fetch from selected albums only - for (auto& albumId : albumIds) { - auto ids = _client->fetchAlbumAssetIds(albumId); - for (auto& id : ids) { - newIds.push_back(id); - } + + for (auto& id : batch) { + if (newIds.size() >= TARGET_QUEUE_SIZE) break; + + // Skip if already in this batch + if (std::find(newIds.begin(), newIds.end(), id) != newIds.end()) continue; + + // Skip if recently shown + if (std::find(_shown.begin(), _shown.end(), id) != _shown.end()) continue; + + newIds.push_back(id); } + + rounds++; + Serial.printf("[queue] Round %d: have %d/%d usable assets\n", + rounds, newIds.size(), TARGET_QUEUE_SIZE); } - // Deduplicate - std::sort(newIds.begin(), newIds.end()); - newIds.erase(std::unique(newIds.begin(), newIds.end()), newIds.end()); - if (newIds.empty()) { - Serial.println("[queue] No assets found"); + _syncRetryDelay = min(_syncRetryDelay * 2, SYNC_RETRY_MAX); + Serial.printf("[queue] Sync failed after %d rounds, retry in %lus\n", + rounds, _syncRetryDelay / 1000); return false; } @@ -68,24 +77,34 @@ bool PhotoQueue::sync() { break; } - // Clamp cursor - if (_cursor >= _queue.size()) { - _cursor = 0; - } + // Reset cursor for fresh queue + _cursor = 0; _lastSyncTime = millis(); - Serial.printf("[queue] Synced: %d assets, cursor at %d\n", _queue.size(), _cursor); + _syncRetryDelay = SYNC_RETRY_MIN; + Serial.printf("[queue] Synced: %d assets in %d rounds, cursor at %d\n", + _queue.size(), rounds, _cursor); return true; } String PhotoQueue::next() { - if (_queue.empty()) return ""; - if (_cursor >= _queue.size()) { + // Auto-refill when queue is exhausted + if (_queue.empty() || _cursor >= _queue.size()) { + Serial.println("[queue] Queue exhausted, fetching new batch"); + sync(); _cursor = 0; - shuffle(); // Reshuffle on wrap for random mode } + + if (_queue.empty()) return ""; + String id = _queue[_cursor++]; + // Track shown IDs for dedup on next refill + _shown.push_back(id); + if (_shown.size() > MAX_SHOWN_HISTORY) { + _shown.erase(_shown.begin()); + } + // Persist cursor _settings->saveField("queue_cursor", static_cast(_cursor)); @@ -109,17 +128,21 @@ size_t PhotoQueue::size() { } bool PhotoQueue::needsResync() { - if (_lastSyncTime == 0) return true; - unsigned long elapsed = millis() - _lastSyncTime; + unsigned long now = millis(); + + // If never synced successfully, use retry backoff + if (_lastSyncTime == 0) { + if (_lastSyncAttempt == 0) return true; + return (now - _lastSyncAttempt) >= _syncRetryDelay; + } + + // Normal resync interval + unsigned long elapsed = now - _lastSyncTime; return elapsed >= (QUEUE_RESYNC_HOURS * 3600000UL); } String PhotoQueue::findPortraitPair(size_t startIndex) { - // Look ahead up to PORTRAIT_LOOKAHEAD items for another portrait for (size_t i = 1; i <= PORTRAIT_LOOKAHEAD && (startIndex + i) < _queue.size(); i++) { - // We'd need asset info to determine portrait status - // This will be called by the display task which fetches AssetInfo - // Return the ID — caller checks isPortrait from AssetInfo return _queue[startIndex + i]; } return ""; @@ -133,9 +156,6 @@ void PhotoQueue::shuffle() { } void PhotoQueue::sortChronological(bool reverse) { - // For chronological sort, we'd need timestamps which we don't store in the queue - // The IDs from Immich are UUIDs, not sortable by time - // For now, keep the order returned by Immich (which is chronological within albums) if (reverse) { std::reverse(_queue.begin(), _queue.end()); } @@ -148,9 +168,7 @@ void PhotoQueue::applyFavoritesWeighting() { if (s.cycle_mode != CycleMode::FavoritesWeighted) return; auto favorites = _client->fetchFavoriteAssetIds(); - // Add favorites 2 more times (total 3× appearance) for (auto& fav : favorites) { - // Only add if already in queue bool inQueue = false; for (auto& id : _queue) { if (id == fav) { inQueue = true; break; } diff --git a/src/photo_queue.h b/src/photo_queue.h index 39476aa..0a2d208 100644 --- a/src/photo_queue.h +++ b/src/photo_queue.h @@ -22,8 +22,17 @@ private: SettingsManager* _settings = nullptr; std::vector _queue; + std::vector _shown; // Recently shown IDs for dedup on refill size_t _cursor = 0; unsigned long _lastSyncTime = 0; + unsigned long _lastSyncAttempt = 0; + unsigned long _syncRetryDelay = 10000; + + static constexpr unsigned long SYNC_RETRY_MIN = 10000; // 10 seconds + static constexpr unsigned long SYNC_RETRY_MAX = 300000; // 5 minutes + static constexpr size_t TARGET_QUEUE_SIZE = 50; + static constexpr int MAX_FETCH_ROUNDS = 5; + static constexpr size_t MAX_SHOWN_HISTORY = 200; void shuffle(); void sortChronological(bool reverse); diff --git a/src/web_server.cpp b/src/web_server.cpp index 9eb7186..a787e0c 100644 --- a/src/web_server.cpp +++ b/src/web_server.cpp @@ -157,6 +157,31 @@ void AppWebServer::setupAPIRoutes() { _power->enterDeepSleep(); }); + // Test Immich connectivity + _server.on("/api/test-connection", HTTP_POST, + [this](AsyncWebServerRequest* req) { + Settings s = _settings->get(); + Serial.printf("[web] Testing connection to: %s (key: %s...)\n", + s.immich_url.c_str(), + s.immich_key.substring(0, 8).c_str()); + + // Reinitialize immich client with current settings + _immich->begin(s.immich_url, s.immich_key); + auto albums = _immich->fetchAlbums(); + + JsonDocument doc; + if (!albums.empty()) { + doc["ok"] = true; + doc["msg"] = String("Connected! Found ") + String(albums.size()) + " albums."; + } else { + doc["ok"] = false; + doc["msg"] = "Failed to connect. Check URL and API key."; + } + String response; + serializeJson(doc, response); + req->send(200, "application/json", response); + }); + _server.on("/api/action/reboot", HTTP_POST, [](AsyncWebServerRequest* req) { req->send(200, "application/json", "{\"ok\":true,\"msg\":\"Rebooting...\"}"); @@ -259,6 +284,7 @@ void AppWebServer::handleGetSettings(AsyncWebServerRequest* request) { doc["meta_pos"] = static_cast(s.meta_pos); doc["led_brightness"] = s.led_brightness; doc["immich_url"] = s.immich_url; + doc["immich_key"] = s.immich_key; doc["albums_json"] = s.albums_json; String response;