From 7aaf920658ea26dd4dd167e00db8f44bd5fedd51 Mon Sep 17 00:00:00 2001 From: cottongin Date: Mon, 3 Aug 2026 10:56:12 -0400 Subject: [PATCH] feat: image pipeline with JPEG decode, resize, and Floyd-Steinberg dithering Co-authored-by: Cursor --- src/image_pipeline.cpp | 378 +++++++++++++++++++++++ src/image_pipeline.h | 43 +++ test/test_native/test_image_pipeline.cpp | 152 +++++++++ test/test_native/test_photo_queue.cpp | 19 ++ 4 files changed, 592 insertions(+) create mode 100644 src/image_pipeline.cpp create mode 100644 src/image_pipeline.h create mode 100644 test/test_native/test_image_pipeline.cpp diff --git a/src/image_pipeline.cpp b/src/image_pipeline.cpp new file mode 100644 index 0000000..1d88439 --- /dev/null +++ b/src/image_pipeline.cpp @@ -0,0 +1,378 @@ +#include "image_pipeline.h" +#include +#include +#include +#include + +// Spectra 6 palette +static const uint8_t PALETTE_RGB[6][3] = { + {PALETTE_BLACK_R, PALETTE_BLACK_G, PALETTE_BLACK_B}, + {PALETTE_WHITE_R, PALETTE_WHITE_G, PALETTE_WHITE_B}, + {PALETTE_RED_R, PALETTE_RED_G, PALETTE_RED_B}, + {PALETTE_GREEN_R, PALETTE_GREEN_G, PALETTE_GREEN_B}, + {PALETTE_BLUE_R, PALETTE_BLUE_G, PALETTE_BLUE_B}, + {PALETTE_YELLOW_R, PALETTE_YELLOW_G, PALETTE_YELLOW_B} +}; + +struct JpegMemReader { + const uint8_t* data; + size_t size; + size_t pos; +}; + +static uint32_t jpegMemInput(void* device, uint8_t* buf, uint32_t len) { + auto* ctx = static_cast(device); + if (ctx->pos >= ctx->size) { + return 0; + } + uint32_t remain = static_cast(ctx->size - ctx->pos); + uint32_t n = len < remain ? len : remain; + memcpy(buf, ctx->data + ctx->pos, n); + ctx->pos += n; + return n; +} + +ProcessedImage ImagePipeline::process(uint8_t* jpegData, size_t jpegSize) { + ProcessedImage result = {nullptr, DISPLAY_WIDTH, DISPLAY_HEIGHT, false}; + + // Decode JPEG + uint16_t srcW, srcH; + uint8_t* rgb = decodeJpeg(jpegData, jpegSize, &srcW, &srcH); + if (rgb == nullptr) { + Serial.println("[pipeline] JPEG decode failed"); + return result; + } + + Serial.printf("[pipeline] Decoded: %dx%d\n", srcW, srcH); + + // Calculate crop region (fill-crop to display aspect ratio) + uint16_t cropX, cropY, cropW, cropH; + centerCrop(rgb, srcW, srcH, DISPLAY_WIDTH, DISPLAY_HEIGHT, + &cropX, &cropY, &cropW, &cropH); + + // Resize cropped region to display dimensions + uint8_t* cropped = (uint8_t*)ps_malloc(cropW * cropH * 3); + if (cropped == nullptr) { + free(rgb); + return result; + } + + // Extract crop region + for (uint16_t y = 0; y < cropH; y++) { + memcpy(cropped + y * cropW * 3, + rgb + ((cropY + y) * srcW + cropX) * 3, + cropW * 3); + } + free(rgb); + + // Resize to display dimensions + uint8_t* resized = resize(cropped, cropW, cropH, DISPLAY_WIDTH, DISPLAY_HEIGHT); + free(cropped); + + if (resized == nullptr) { + return result; + } + + // Dither to 6-color palette + uint8_t* dithered = dither(resized, DISPLAY_WIDTH, DISPLAY_HEIGHT); + free(resized); + + if (dithered == nullptr) { + return result; + } + + result.framebuffer = dithered; + result.valid = true; + Serial.println("[pipeline] Processing complete"); + return result; +} + +ProcessedImage ImagePipeline::processPortraitPair(uint8_t* jpeg1Data, size_t jpeg1Size, + uint8_t* jpeg2Data, size_t jpeg2Size) { + ProcessedImage result = {nullptr, DISPLAY_WIDTH, DISPLAY_HEIGHT, false}; + + // Each portrait gets half the width minus gap + uint16_t portraitW = (DISPLAY_WIDTH - PORTRAIT_GAP_PX) / 2; + uint16_t portraitH = DISPLAY_HEIGHT; + + // Allocate combined RGB buffer + uint8_t* combined = (uint8_t*)ps_calloc(DISPLAY_WIDTH * DISPLAY_HEIGHT * 3, 1); + if (combined == nullptr) return result; + + // Process first portrait + uint16_t src1W, src1H; + uint8_t* rgb1 = decodeJpeg(jpeg1Data, jpeg1Size, &src1W, &src1H); + if (rgb1 != nullptr) { + uint16_t cropX, cropY, cropW, cropH; + centerCrop(rgb1, src1W, src1H, portraitW, portraitH, + &cropX, &cropY, &cropW, &cropH); + + uint8_t* cropped1 = (uint8_t*)ps_malloc(cropW * cropH * 3); + if (cropped1) { + for (uint16_t y = 0; y < cropH; y++) { + memcpy(cropped1 + y * cropW * 3, + rgb1 + ((cropY + y) * src1W + cropX) * 3, cropW * 3); + } + uint8_t* resized1 = resize(cropped1, cropW, cropH, portraitW, portraitH); + free(cropped1); + if (resized1) { + // Copy into left side of combined buffer + for (uint16_t y = 0; y < portraitH; y++) { + memcpy(combined + y * DISPLAY_WIDTH * 3, + resized1 + y * portraitW * 3, portraitW * 3); + } + free(resized1); + } + } + free(rgb1); + } + + // Process second portrait + uint16_t src2W, src2H; + uint8_t* rgb2 = decodeJpeg(jpeg2Data, jpeg2Size, &src2W, &src2H); + if (rgb2 != nullptr) { + uint16_t cropX, cropY, cropW, cropH; + centerCrop(rgb2, src2W, src2H, portraitW, portraitH, + &cropX, &cropY, &cropW, &cropH); + + uint8_t* cropped2 = (uint8_t*)ps_malloc(cropW * cropH * 3); + if (cropped2) { + for (uint16_t y = 0; y < cropH; y++) { + memcpy(cropped2 + y * cropW * 3, + rgb2 + ((cropY + y) * src2W + cropX) * 3, cropW * 3); + } + uint8_t* resized2 = resize(cropped2, cropW, cropH, portraitW, portraitH); + free(cropped2); + if (resized2) { + // Copy into right side of combined buffer + uint16_t offsetX = portraitW + PORTRAIT_GAP_PX; + for (uint16_t y = 0; y < portraitH; y++) { + memcpy(combined + (y * DISPLAY_WIDTH + offsetX) * 3, + resized2 + y * portraitW * 3, portraitW * 3); + } + free(resized2); + } + } + free(rgb2); + } + + // Dither combined buffer + uint8_t* dithered = dither(combined, DISPLAY_WIDTH, DISPLAY_HEIGHT); + free(combined); + + if (dithered == nullptr) return result; + + result.framebuffer = dithered; + result.valid = true; + return result; +} + +void ImagePipeline::freeImage(ProcessedImage& img) { + if (img.framebuffer) { + free(img.framebuffer); + img.framebuffer = nullptr; + } + img.valid = false; +} + +uint8_t* ImagePipeline::decodeJpeg(uint8_t* data, size_t size, + uint16_t* outWidth, uint16_t* outHeight) { + JpegMemReader reader = {data, size, 0}; + lgfxJdec jdec; + static constexpr uint16_t poolSize = 3900; + uint8_t* pool = (uint8_t*)malloc(poolSize); + if (pool == nullptr) { + return nullptr; + } + + JRESULT prep = lgfx_jd_prepare(&jdec, jpegMemInput, pool, poolSize, &reader); + if (prep != JDR_OK) { + free(pool); + return nullptr; + } + + uint16_t w = jdec.width; + uint16_t h = jdec.height; + free(pool); + + lgfx::LGFX_Sprite sprite; + sprite.setPsram(true); + sprite.setColorDepth(lgfx::color_depth_t::rgb888_3Byte); + + if (!sprite.createSprite(w, h)) { + return nullptr; + } + + if (!sprite.drawJpg(data, size, 0, 0)) { + sprite.deleteSprite(); + return nullptr; + } + + size_t bufSize = static_cast(w) * h * 3; + uint8_t* rgb = (uint8_t*)ps_malloc(bufSize); + if (rgb == nullptr) { + sprite.deleteSprite(); + return nullptr; + } + + for (uint16_t y = 0; y < h; y++) { + for (uint16_t x = 0; x < w; x++) { + lgfx::bgr888_t color = sprite.readPixelRGB(x, y); + size_t idx = (static_cast(y) * w + x) * 3; + rgb[idx] = color.r; + rgb[idx + 1] = color.g; + rgb[idx + 2] = color.b; + } + } + + sprite.deleteSprite(); + *outWidth = w; + *outHeight = h; + return rgb; +} + +uint8_t* ImagePipeline::resize(uint8_t* rgb, uint16_t srcW, uint16_t srcH, + uint16_t dstW, uint16_t dstH) { + size_t bufSize = dstW * dstH * 3; + uint8_t* dst = (uint8_t*)ps_malloc(bufSize); + if (dst == nullptr) return nullptr; + + if (dstW <= 1 || dstH <= 1 || srcW == 0 || srcH == 0) { + free(dst); + return nullptr; + } + + // Bilinear interpolation + float xRatio = (float)(srcW - 1) / (float)(dstW - 1); + float yRatio = (float)(srcH - 1) / (float)(dstH - 1); + + for (uint16_t y = 0; y < dstH; y++) { + float srcY = y * yRatio; + uint16_t y0 = (uint16_t)srcY; + uint16_t y1 = min((uint16_t)(y0 + 1), (uint16_t)(srcH - 1)); + float yFrac = srcY - y0; + + for (uint16_t x = 0; x < dstW; x++) { + float srcX = x * xRatio; + uint16_t x0 = (uint16_t)srcX; + uint16_t x1 = min((uint16_t)(x0 + 1), (uint16_t)(srcW - 1)); + float xFrac = srcX - x0; + + for (int c = 0; c < 3; c++) { + float top = rgb[(y0 * srcW + x0) * 3 + c] * (1 - xFrac) + + rgb[(y0 * srcW + x1) * 3 + c] * xFrac; + float bot = rgb[(y1 * srcW + x0) * 3 + c] * (1 - xFrac) + + rgb[(y1 * srcW + x1) * 3 + c] * xFrac; + float val = top * (1 - yFrac) + bot * yFrac; + dst[(y * dstW + x) * 3 + c] = (uint8_t)(val + 0.5f); + } + } + } + + return dst; +} + +void ImagePipeline::centerCrop(uint8_t* rgb, uint16_t srcW, uint16_t srcH, + uint16_t targetW, uint16_t targetH, + uint16_t* cropX, uint16_t* cropY, + uint16_t* cropW, uint16_t* cropH) { + (void)rgb; + float targetAspect = (float)targetW / (float)targetH; + float srcAspect = (float)srcW / (float)srcH; + + if (srcAspect > targetAspect) { + // Source is wider — crop sides + *cropH = srcH; + *cropW = (uint16_t)(srcH * targetAspect); + *cropX = (srcW - *cropW) / 2; + *cropY = 0; + } else { + // Source is taller — crop top/bottom + *cropW = srcW; + *cropH = (uint16_t)(srcW / targetAspect); + *cropX = 0; + *cropY = (srcH - *cropH) / 2; + } +} + +uint8_t* ImagePipeline::dither(uint8_t* rgb, uint16_t width, uint16_t height) { + size_t pixelCount = width * height; + uint8_t* output = (uint8_t*)ps_malloc(pixelCount); + if (output == nullptr) return nullptr; + + // Work buffer with int16 to handle error overflow + int16_t* work = (int16_t*)ps_malloc(pixelCount * 3 * sizeof(int16_t)); + if (work == nullptr) { + free(output); + return nullptr; + } + + // Copy to work buffer + for (size_t i = 0; i < pixelCount * 3; i++) { + work[i] = rgb[i]; + } + + // Floyd-Steinberg dithering + for (uint16_t y = 0; y < height; y++) { + for (uint16_t x = 0; x < width; x++) { + size_t idx = (y * width + x) * 3; + int r = constrain(work[idx], 0, 255); + int g = constrain(work[idx + 1], 0, 255); + int b = constrain(work[idx + 2], 0, 255); + + uint8_t nearest = findNearest(r, g, b); + output[y * width + x] = nearest; + + int errR = r - PALETTE_RGB[nearest][0]; + int errG = g - PALETTE_RGB[nearest][1]; + int errB = b - PALETTE_RGB[nearest][2]; + + // Distribute error + if (x + 1 < width) { + size_t ni = (y * width + (x + 1)) * 3; + work[ni] += errR * 7 / 16; + work[ni + 1] += errG * 7 / 16; + work[ni + 2] += errB * 7 / 16; + } + if (y + 1 < height) { + if (x > 0) { + size_t ni = ((y + 1) * width + (x - 1)) * 3; + work[ni] += errR * 3 / 16; + work[ni + 1] += errG * 3 / 16; + work[ni + 2] += errB * 3 / 16; + } + { + size_t ni = ((y + 1) * width + x) * 3; + work[ni] += errR * 5 / 16; + work[ni + 1] += errG * 5 / 16; + work[ni + 2] += errB * 5 / 16; + } + if (x + 1 < width) { + size_t ni = ((y + 1) * width + (x + 1)) * 3; + work[ni] += errR * 1 / 16; + work[ni + 1] += errG * 1 / 16; + work[ni + 2] += errB * 1 / 16; + } + } + } + } + + free(work); + return output; +} + +uint8_t ImagePipeline::findNearest(int r, int g, int b) { + uint8_t best = 0; + int bestDist = INT32_MAX; + for (int i = 0; i < DISPLAY_COLORS; i++) { + int dr = r - PALETTE_RGB[i][0]; + int dg = g - PALETTE_RGB[i][1]; + int db = b - PALETTE_RGB[i][2]; + int dist = dr * dr + dg * dg + db * db; + if (dist < bestDist) { + bestDist = dist; + best = static_cast(i); + } + } + return best; +} diff --git a/src/image_pipeline.h b/src/image_pipeline.h new file mode 100644 index 0000000..5935cd0 --- /dev/null +++ b/src/image_pipeline.h @@ -0,0 +1,43 @@ +#pragma once + +#include +#include "config.h" + +struct ProcessedImage { + uint8_t* framebuffer; // Palette indices, one byte per pixel + uint16_t width; + uint16_t height; + bool valid; +}; + +class ImagePipeline { +public: + // Process single landscape photo + ProcessedImage process(uint8_t* jpegData, size_t jpegSize); + + // Process portrait pair (two photos side by side) + ProcessedImage processPortraitPair(uint8_t* jpeg1Data, size_t jpeg1Size, + uint8_t* jpeg2Data, size_t jpeg2Size); + + void freeImage(ProcessedImage& img); + +private: + // Decode JPEG into RGB888 buffer in PSRAM + uint8_t* decodeJpeg(uint8_t* data, size_t size, uint16_t* outWidth, uint16_t* outHeight); + + // Resize RGB buffer to target dimensions (bilinear) + uint8_t* resize(uint8_t* rgb, uint16_t srcW, uint16_t srcH, + uint16_t dstW, uint16_t dstH); + + // Center-crop to target aspect ratio + void centerCrop(uint8_t* rgb, uint16_t srcW, uint16_t srcH, + uint16_t targetW, uint16_t targetH, + uint16_t* cropX, uint16_t* cropY, + uint16_t* cropW, uint16_t* cropH); + + // Floyd-Steinberg dither RGB888 to 6-color palette indices + uint8_t* dither(uint8_t* rgb, uint16_t width, uint16_t height); + + // Find nearest palette color + uint8_t findNearest(int r, int g, int b); +}; diff --git a/test/test_native/test_image_pipeline.cpp b/test/test_native/test_image_pipeline.cpp new file mode 100644 index 0000000..ee855cc --- /dev/null +++ b/test/test_native/test_image_pipeline.cpp @@ -0,0 +1,152 @@ +#include +#include +#include +#include +#include +#include + +// Palette definition (same as config.h) +struct Color { uint8_t r, g, b; }; + +static const Color PALETTE[6] = { + {0, 0, 0}, // Black + {255, 255, 255}, // White + {200, 30, 30}, // Red + {30, 160, 30}, // Green + {30, 30, 200}, // Blue + {220, 200, 30} // Yellow +}; + +// Find nearest palette color (Euclidean distance in RGB) +uint8_t findNearestColor(int r, int g, int b) { + uint8_t best = 0; + int bestDist = INT32_MAX; + for (int i = 0; i < 6; i++) { + int dr = r - PALETTE[i].r; + int dg = g - PALETTE[i].g; + int db = b - PALETTE[i].b; + int dist = dr*dr + dg*dg + db*db; + if (dist < bestDist) { + bestDist = dist; + best = static_cast(i); + } + } + return best; +} + +// Floyd-Steinberg dithering on a small test buffer +void ditherBuffer(uint8_t* rgb, int width, int height, uint8_t* output) { + // Working buffer with int16_t to handle error diffusion overflow + int16_t* work = (int16_t*)malloc(width * height * 3 * sizeof(int16_t)); + for (int i = 0; i < width * height * 3; i++) { + work[i] = rgb[i]; + } + + for (int y = 0; y < height; y++) { + for (int x = 0; x < width; x++) { + int idx = (y * width + x) * 3; + int r = work[idx]; + int g = work[idx + 1]; + int b = work[idx + 2]; + + // Clamp + r = r < 0 ? 0 : (r > 255 ? 255 : r); + g = g < 0 ? 0 : (g > 255 ? 255 : g); + b = b < 0 ? 0 : (b > 255 ? 255 : b); + + uint8_t nearest = findNearestColor(r, g, b); + output[y * width + x] = nearest; + + // Error + int errR = r - PALETTE[nearest].r; + int errG = g - PALETTE[nearest].g; + int errB = b - PALETTE[nearest].b; + + // Distribute error (Floyd-Steinberg weights: 7/16, 3/16, 5/16, 1/16) + if (x + 1 < width) { + int ni = (y * width + (x + 1)) * 3; + work[ni] += errR * 7 / 16; + work[ni + 1] += errG * 7 / 16; + work[ni + 2] += errB * 7 / 16; + } + if (y + 1 < height) { + if (x > 0) { + int ni = ((y + 1) * width + (x - 1)) * 3; + work[ni] += errR * 3 / 16; + work[ni + 1] += errG * 3 / 16; + work[ni + 2] += errB * 3 / 16; + } + { + int ni = ((y + 1) * width + x) * 3; + work[ni] += errR * 5 / 16; + work[ni + 1] += errG * 5 / 16; + work[ni + 2] += errB * 5 / 16; + } + if (x + 1 < width) { + int ni = ((y + 1) * width + (x + 1)) * 3; + work[ni] += errR * 1 / 16; + work[ni + 1] += errG * 1 / 16; + work[ni + 2] += errB * 1 / 16; + } + } + } + } + free(work); +} + +void test_nearest_color_black() { + TEST_ASSERT_EQUAL(0, findNearestColor(0, 0, 0)); +} + +void test_nearest_color_white() { + TEST_ASSERT_EQUAL(1, findNearestColor(255, 255, 255)); +} + +void test_nearest_color_red() { + TEST_ASSERT_EQUAL(2, findNearestColor(180, 20, 20)); +} + +void test_nearest_color_green() { + TEST_ASSERT_EQUAL(3, findNearestColor(20, 140, 20)); +} + +void test_nearest_color_blue() { + TEST_ASSERT_EQUAL(4, findNearestColor(20, 20, 180)); +} + +void test_nearest_color_yellow() { + TEST_ASSERT_EQUAL(5, findNearestColor(200, 180, 20)); +} + +void test_dither_solid_black() { + const int W = 4, H = 4; + uint8_t rgb[W * H * 3] = {0}; // All black + uint8_t output[W * H]; + ditherBuffer(rgb, W, H, output); + for (int i = 0; i < W * H; i++) { + TEST_ASSERT_EQUAL(0, output[i]); // All should be black + } +} + +void test_dither_solid_white() { + const int W = 4, H = 4; + uint8_t rgb[W * H * 3]; + memset(rgb, 255, sizeof(rgb)); // All white + uint8_t output[W * H]; + ditherBuffer(rgb, W, H, output); + for (int i = 0; i < W * H; i++) { + TEST_ASSERT_EQUAL(1, output[i]); // All should be white + } +} + +void test_dither_produces_valid_indices() { + const int W = 8, H = 8; + uint8_t rgb[W * H * 3]; + // Fill with mid-gray + for (int i = 0; i < W * H * 3; i++) rgb[i] = 128; + uint8_t output[W * H]; + ditherBuffer(rgb, W, H, output); + for (int i = 0; i < W * H; i++) { + TEST_ASSERT_TRUE(output[i] < 6); // Valid palette index + } +} diff --git a/test/test_native/test_photo_queue.cpp b/test/test_native/test_photo_queue.cpp index df9c768..b452177 100644 --- a/test/test_native/test_photo_queue.cpp +++ b/test/test_native/test_photo_queue.cpp @@ -92,6 +92,16 @@ void test_favorites_weighting() { TEST_ASSERT_EQUAL(5, weighted.size()); // 3 original + 2 extra } +void test_nearest_color_black(); +void test_nearest_color_white(); +void test_nearest_color_red(); +void test_nearest_color_green(); +void test_nearest_color_blue(); +void test_nearest_color_yellow(); +void test_dither_solid_black(); +void test_dither_solid_white(); +void test_dither_produces_valid_indices(); + void setUp() {} void tearDown() {} @@ -101,6 +111,15 @@ int main() { RUN_TEST(test_advance_wraps_at_end); RUN_TEST(test_empty_queue_returns_empty); RUN_TEST(test_favorites_weighting); + RUN_TEST(test_nearest_color_black); + RUN_TEST(test_nearest_color_white); + RUN_TEST(test_nearest_color_red); + RUN_TEST(test_nearest_color_green); + RUN_TEST(test_nearest_color_blue); + RUN_TEST(test_nearest_color_yellow); + RUN_TEST(test_dither_solid_black); + RUN_TEST(test_dither_solid_white); + RUN_TEST(test_dither_produces_valid_indices); UNITY_END(); return 0; }