From 702eaaaa879cfbe6c3aca26b34bf5c20f56a3370 Mon Sep 17 00:00:00 2001 From: Roman Pustobaiev Date: Mon, 29 Sep 2025 21:45:30 +0200 Subject: [PATCH 01/18] Adding smile detection Refactoring --- host/.gitignore | 1 + host/CMakeLists.txt | 36 ++++ host/HaarPath.hpp.in | 3 + host/Src/main.cpp | 84 +++++++++ u5/app/CMakeLists.txt | 2 +- u5/app/opencv.cmake | 18 +- u5/app/prj.conf | 30 +++- u5/app/src/cascades.h.in | 40 +++++ u5/app/src/grinreflex.h | 4 +- u5/app/src/grinreflex_v2.cpp | 162 +++++++++++++++++- u5/app/src/opencv_utils.cpp | 160 ++++++++++++++--- u5/app/src/opencv_utils.hpp | 43 ++++- .../grinreflex_dk2/grinreflex_dk2_defconfig | 3 +- u5/lib/cv/CMakeLists.txt | 2 +- 14 files changed, 539 insertions(+), 49 deletions(-) create mode 100644 host/.gitignore create mode 100644 host/CMakeLists.txt create mode 100644 host/HaarPath.hpp.in create mode 100644 host/Src/main.cpp create mode 100644 u5/app/src/cascades.h.in diff --git a/host/.gitignore b/host/.gitignore new file mode 100644 index 0000000..378eac2 --- /dev/null +++ b/host/.gitignore @@ -0,0 +1 @@ +build diff --git a/host/CMakeLists.txt b/host/CMakeLists.txt new file mode 100644 index 0000000..9542c02 --- /dev/null +++ b/host/CMakeLists.txt @@ -0,0 +1,36 @@ +cmake_minimum_required(VERSION 3.10) +project(host_app) + +# Find OpenCV4 +find_package(OpenCV 4 REQUIRED) + +set(OPENCV_HAAR_PATH "/usr/share/opencv4/") +#set(OPENCV_HAAR_PATH "${CMAKE_SOURCE_DIR}/Haar") + +set(HAAR_HEADER "${CMAKE_BINARY_DIR}/Include/HaarPath.hpp") +file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/Include") + +configure_file( + ${CMAKE_SOURCE_DIR}/HaarPath.hpp.in + ${HAAR_HEADER} + @ONLY +) + +set(OPENCV_UTILS_DIR_PATH "../u5/app/src") + +set(HOST_APP_SRCS + Src/main.cpp + ${OPENCV_UTILS_DIR_PATH}/opencv_utils.cpp + ${OPENCV_UTILS_DIR_PATH}/opencv_utils.hpp +) + +# Add executable +add_executable(host_app ${HOST_APP_SRCS}) + +# Link OpenCV libraries +target_include_directories(host_app + PRIVATE "${CMAKE_BINARY_DIR}/Include" + PRIVATE ${OpenCV_INCLUDE_DIRS} + PRIVATE ${OPENCV_UTILS_DIR_PATH}) + +target_link_libraries(host_app PRIVATE ${OpenCV_LIBS}) diff --git a/host/HaarPath.hpp.in b/host/HaarPath.hpp.in new file mode 100644 index 0000000..9db1dfa --- /dev/null +++ b/host/HaarPath.hpp.in @@ -0,0 +1,3 @@ +#pragma once + +constexpr const char* HAAR_PATH = "@OPENCV_HAAR_PATH@"; \ No newline at end of file diff --git a/host/Src/main.cpp b/host/Src/main.cpp new file mode 100644 index 0000000..52805e2 --- /dev/null +++ b/host/Src/main.cpp @@ -0,0 +1,84 @@ +#include +#include + +#include "HaarPath.hpp" + +#include "opencv_utils.hpp" + +cv::String haarPath = HAAR_PATH; +cv::String face_cascade_name = haarPath + "lbpcascades/lbpcascade_frontalface_improved.xml"; +cv::String smile_cascade_name = haarPath + "haarcascades/haarcascade_smile.xml"; +cv::CascadeClassifier face_cascade; +cv::CascadeClassifier smile_cascade; +cv::String window_name = "Capture - Face detection"; + +int main(int, char**) { + // open the first webcam plugged in the computer + cv::VideoCapture camera(0); // in linux check $ ls /dev/video0 + if (!camera.isOpened()) { + std::cerr << "ERROR: Could not open camera" << std::endl; + return 1; + } + + // array to hold image + cv::Mat frameOriginal; + cv::Mat frameScreenGray; + + if (!face_cascade.load(face_cascade_name)) + { + printf("--(!)Error loading face cascade\n"); + return -1; + }; + if (!smile_cascade.load(smile_cascade_name)) + { + printf("--(!)Error loading eyes cascade\n"); + return -1; + }; + + cv::namedWindow(window_name, 2); + // display the frame until you press a key + + std::cout << "Start capturing" << std::endl; + while (camera.read(frameOriginal)) { + + if (frameOriginal.empty()) + { + printf(" --(!) No captured frame -- Break!"); + break; + } + + cv::Mat frameOriginalGray; + cv::cvtColor(frameOriginal, frameOriginalGray, cv::COLOR_BGR2GRAY); + cv::equalizeHist(frameOriginalGray, frameOriginalGray); + + cv::Rect faceROIMax{}; + faceROIMax.width = 80; + faceROIMax.height = 80; + + cv::resize(frameOriginalGray, frameScreenGray, cv::Size(faceROIMax.width, faceROIMax.height), 0.0f, 0.0f, cv::INTER_LINEAR); + + std::vector ROIs = detectFaceAndSmile( + face_cascade, + smile_cascade, + frameScreenGray, + frameOriginalGray, + faceROIMax + ); + + for (const auto &ROI : ROIs) { + rectangle(frameScreenGray, ROI, cv::Scalar(0, 0, 255), 2, 8, 0); + } + + cv::imshow(window_name, frameScreenGray); + int c = cv::waitKey(100); + if ((char)c == 27) { return 0; } + } + + return 0; +} + +// CMD to generate executable: +// g++ webcam_opencv.cpp -o webcam_demo -I/usr/include/opencv4 -lopencv_core -lopencv_videoio -lopencv_highgui + +// Note: check your opencv hpp files - for many users it is at /usr/local/include/opencv4 +// Add more packages during compilation from the list obtained by $ pkg-config --cflags --libs opencv4 diff --git a/u5/app/CMakeLists.txt b/u5/app/CMakeLists.txt index d671cb2..3ad0d3e 100644 --- a/u5/app/CMakeLists.txt +++ b/u5/app/CMakeLists.txt @@ -49,4 +49,4 @@ if (CONFIG_BOARD_GRINREFLEX_DK2) target_sources(app PRIVATE src/grinreflex_v2.cpp ) -endif() \ No newline at end of file +endif() diff --git a/u5/app/opencv.cmake b/u5/app/opencv.cmake index d890f63..e5c4ddb 100644 --- a/u5/app/opencv.cmake +++ b/u5/app/opencv.cmake @@ -7,4 +7,20 @@ target_sources(app PRIVATE src/opencv_utils.cpp ) -target_link_libraries(app PUBLIC opencv_lib) +if (CONFIG_BOARD_GRINREFLEX_DK2) +set(OPENCV_LBP_CASCADES ${OPENCV_INSTALL_DIR}/share/opencv4/lbpcascades) +set(OPENCV_HAAR_CASCADES ${OPENCV_INSTALL_DIR}/share/opencv4/haarcascades) + +set(OPENCV_CASCADE_FACE_PATH ${OPENCV_LBP_CASCADES}/lbpcascade_frontalface_improved.xml) +set(OPENCV_CASCADE_SMILE_PATH ${OPENCV_HAAR_CASCADES}/haarcascade_smile.xml) + +configure_file( + src/cascades.h.in + src/cascades.h + @ONLY +) + +target_include_directories(app PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/src") +endif() + +target_link_libraries(app PUBLIC opencv_lib) \ No newline at end of file diff --git a/u5/app/prj.conf b/u5/app/prj.conf index fb3091f..fa4f882 100644 --- a/u5/app/prj.conf +++ b/u5/app/prj.conf @@ -40,7 +40,8 @@ CONFIG_DEBUG_OPTIMIZATIONS=y CONFIG_COMMON_LIBC_MALLOC=y CONFIG_HEAP_MEM_POOL_SIZE=49152 -CONFIG_COMMON_LIBC_MALLOC_ARENA_SIZE=1966080 +CONFIG_COMMON_LIBC_MALLOC_ARENA_SIZE=2228224 +#CONFIG_COMMON_LIBC_MALLOC_ARENA_SIZE=1048576 #CONFIG_TFLITE_LIB=y @@ -76,9 +77,28 @@ CONFIG_VIDEO_LOG_LEVEL_INF=y CONFIG_STM32_JPEG_RGB_FORMAT=3 CONFIG_GRINREFLEX_JPEG_VIDEO=y -CONFIG_GRINREFLEX_JPEG_VIDEO_QUALITY=10 -CONFIG_GRINREFLEX_VIDEO_WIDTH=160 -CONFIG_GRINREFLEX_VIDEO_HEIGHT=120 +CONFIG_GRINREFLEX_JPEG_VIDEO_QUALITY=50 +CONFIG_GRINREFLEX_VIDEO_WIDTH=320 +CONFIG_GRINREFLEX_VIDEO_HEIGHT=240 -CONFIG_HW_STACK_PROTECTION=y \ No newline at end of file +CONFIG_HW_STACK_PROTECTION=y + + +# Make asserts print file:line and message +CONFIG_ASSERT=y +CONFIG_ASSERT_VERBOSE=y + +# Ensure logs/printk come out before the crash +CONFIG_PRINTK=y +CONFIG_LOG=y +CONFIG_LOG_PRINTK=y +CONFIG_LOG_MODE_IMMEDIATE=y + +# Better symbols for gdb +#CONFIG_DEBUG_INFO=y +# (Optional) minimal optimizations for nicer backtraces +CONFIG_NO_OPTIMIZATIONS=y + +# Richer fault dump on ARM (registers, etc.) +CONFIG_FAULT_DUMP=2 diff --git a/u5/app/src/cascades.h.in b/u5/app/src/cascades.h.in new file mode 100644 index 0000000..e74f257 --- /dev/null +++ b/u5/app/src/cascades.h.in @@ -0,0 +1,40 @@ +#ifndef GRINREFLEX_CASCADES_H +#define GRINREFLEX_CASCADES_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define _EVAL(__x) __x + +#define _INCBIN(__name, __path) \ +extern const uint8_t _cascade_##__name##_start[]; \ +extern const uint8_t _cascade_##__name##_end[]; \ + \ +__asm__ ( \ + ".section .rodata.cascades\n" \ + ".global _cascade_" #__name "_start\n" \ + ".global _cascade_" #__name "_end\n" \ + ".balign 4\n" \ + "_cascade_" #__name "_start:\n" \ + ".incbin \"" #__path "\"\n" \ + "_cascade_" #__name "_end:\n" \ + ".previous\n" \ +); \ + \ +static inline size_t cascade_##__name##_len() { \ + return (size_t)_cascade_##__name##_end - (size_t)_cascade_##__name##_start; \ +} \ + \ +static inline const uint8_t *cascade_##__name##_data() { \ + return _cascade_##__name##_start; \ +} + +_INCBIN(face, @OPENCV_CASCADE_FACE_PATH@) +_INCBIN(smile, @OPENCV_CASCADE_SMILE_PATH@) + +#ifdef __cplusplus +} +#endif + +#endif /* GRINREFLEX_CASCADES_H */ \ No newline at end of file diff --git a/u5/app/src/grinreflex.h b/u5/app/src/grinreflex.h index 281cd96..d152488 100644 --- a/u5/app/src/grinreflex.h +++ b/u5/app/src/grinreflex.h @@ -30,8 +30,8 @@ extern "C" { #endif /* defined(CONFIG_GRINREFLEX_JPEG_VIDEO) */ -#define GRAY_FRAME_WIDTH 120 -#define GRAY_FRAME_HEIGHT 120 +#define GRAY_FRAME_WIDTH 80 +#define GRAY_FRAME_HEIGHT 80 int init(); diff --git a/u5/app/src/grinreflex_v2.cpp b/u5/app/src/grinreflex_v2.cpp index 838894f..dac70e2 100644 --- a/u5/app/src/grinreflex_v2.cpp +++ b/u5/app/src/grinreflex_v2.cpp @@ -2,6 +2,12 @@ #include #include +#if defined(CONFIG_OPENCV_LIB) +#include "opencv2/opencv.hpp" +#include "opencv_utils.hpp" +#include "cascades.h" +#endif + #include #include #include @@ -23,11 +29,6 @@ LOG_MODULE_REGISTER(grinreflex_app); #include -#if defined(CONFIG_OPENCV_LIB) -#include "opencv2/opencv.hpp" -#include "opencv_utils.hpp" -#endif - #include "grinreflex.h" static const struct device *video_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_camera)); @@ -38,10 +39,17 @@ static lv_img_dsc_t video_img; static lv_obj_t *vid_canvas; static lv_obj_t *screen; -static uint8_t rgb_frame_buffer[CONFIG_GRINREFLEX_VIDEO_WIDTH * CONFIG_GRINREFLEX_VIDEO_HEIGHT * LV_COLOR_FORMAT_GET_SIZE(JPEG_COLOR_FORMAT)]; +static uint8_t input_frame_buffer[CONFIG_GRINREFLEX_VIDEO_WIDTH * CONFIG_GRINREFLEX_VIDEO_HEIGHT * LV_COLOR_FORMAT_GET_SIZE(JPEG_COLOR_FORMAT)]; static uint8_t gray_frame_buffer[GRAY_FRAME_WIDTH * GRAY_FRAME_HEIGHT * LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8)]; +#if defined(CONFIG_OPENCV_LIB) +static lv_obj_t *RedGreenDot; +static lv_obj_t *ROIRectFace; +static lv_obj_t *ROIRectSmile; +static cv::CascadeClassifier faceCascade; +static cv::CascadeClassifier smileCascade; +#endif /* defined(CONFIG_OPENCV_LIB) */ int init() { @@ -60,6 +68,22 @@ int init() return -ENODEV; } +#if defined(CONFIG_OPENCV_LIB) + + cv::setNumThreads(1); + cv::setUseOptimized(false); + + + if (false == loadCascade(faceCascade, cascade_face_data(), cascade_face_len())) { + return -3; + } + + if (false == loadCascade(smileCascade, cascade_smile_data(), cascade_smile_len())) { + return -3; + } + +#endif /* defined(CONFIG_OPENCV_LIB) */ + Video::setup(); vid_canvas = lv_canvas_create(lv_screen_active()); @@ -78,6 +102,52 @@ int init() video_img.header.cf = LV_COLOR_FORMAT_NATIVE; #endif screen = lv_img_create(lv_scr_act()); + +#if defined(CONFIG_OPENCV_LIB) + RedGreenDot = lv_obj_create(lv_scr_act()); + lv_obj_set_size(RedGreenDot, 30, 30); // dot size + lv_obj_set_style_radius(RedGreenDot, LV_RADIUS_CIRCLE, 0); // make it round + lv_obj_set_style_border_width(RedGreenDot, 0, 0); + lv_obj_set_style_bg_opa(RedGreenDot, LV_OPA_COVER, 0); + lv_obj_clear_flag(RedGreenDot, LV_OBJ_FLAG_SCROLLABLE); + lv_obj_align(RedGreenDot, LV_ALIGN_LEFT_MID, 4, 0); // 4px from left + lv_obj_move_foreground(RedGreenDot); // keep it on top + + ROIRectFace = lv_obj_create(lv_scr_act()); + lv_obj_set_pos(ROIRectFace, 0, 0); + lv_obj_set_size(ROIRectFace, 1, 1); + + // Unfilled, just a thick border + lv_obj_set_style_bg_opa(ROIRectFace, LV_OPA_TRANSP, LV_PART_MAIN); + lv_obj_set_style_border_width(ROIRectFace, 4, LV_PART_MAIN); + lv_obj_set_style_border_opa(ROIRectFace, LV_OPA_COVER, LV_PART_MAIN); + lv_obj_set_style_border_color(ROIRectFace, lv_palette_main(LV_PALETTE_GREEN), LV_PART_MAIN); + lv_obj_set_style_radius(ROIRectFace, 0, LV_PART_MAIN); // 0 = sharp corners + lv_obj_set_style_pad_all(ROIRectFace, 0, LV_PART_MAIN); + lv_obj_set_style_outline_opa(ROIRectFace, LV_OPA_TRANSP, LV_PART_MAIN); // no focus outline + lv_obj_set_style_shadow_opa(ROIRectFace, LV_OPA_TRANSP, LV_PART_MAIN); // no shadow + lv_obj_move_foreground(ROIRectFace); + lv_obj_add_flag(ROIRectFace, LV_OBJ_FLAG_HIDDEN); + + ROIRectSmile = lv_obj_create(lv_scr_act()); + lv_obj_set_pos(ROIRectSmile, 0, 0); + lv_obj_set_size(ROIRectSmile, 1, 1); + + // Unfilled, just a thick border + lv_obj_set_style_bg_opa(ROIRectSmile, LV_OPA_TRANSP, LV_PART_MAIN); + lv_obj_set_style_border_width(ROIRectSmile, 2, LV_PART_MAIN); + lv_obj_set_style_border_opa(ROIRectSmile, LV_OPA_COVER, LV_PART_MAIN); + lv_obj_set_style_border_color(ROIRectSmile, lv_palette_main(LV_PALETTE_GREEN), LV_PART_MAIN); + lv_obj_set_style_radius(ROIRectSmile, 0, LV_PART_MAIN); // 0 = sharp corners + lv_obj_set_style_pad_all(ROIRectSmile, 0, LV_PART_MAIN); + lv_obj_set_style_outline_opa(ROIRectSmile, LV_OPA_TRANSP, LV_PART_MAIN); // no focus outline + lv_obj_set_style_shadow_opa(ROIRectSmile, LV_OPA_TRANSP, LV_PART_MAIN); // no shadow + lv_obj_move_foreground(ROIRectSmile); + lv_obj_add_flag(ROIRectSmile, LV_OBJ_FLAG_HIDDEN); + +#endif + + return 0; } int loop() @@ -103,7 +173,7 @@ int loop() //printf("JPEG done w = %d, h = %d, color = %d chroma = %d\n", jpeg_prop.width, // jpeg_prop.height, jpeg_prop.color_space, jpeg_prop.chroma); - jpeg_color_convert_helper(jpeg_dev, &jpeg_prop, vbuf_ptr->buffer, rgb_frame_buffer); + jpeg_color_convert_helper(jpeg_dev, &jpeg_prop, vbuf_ptr->buffer, input_frame_buffer); #else video_img.data = (uint8_t *)vbuf_ptr->buffer; @@ -112,10 +182,11 @@ int loop() lv_task_handler(); #endif +#if !defined(CONFIG_OPENCV_LIB) Gfx::GfxBuffer src{}; Gfx::GfxBuffer dst{}; - src.buf = rgb_frame_buffer; + src.buf = input_frame_buffer; src.width = CONFIG_GRINREFLEX_VIDEO_WIDTH; src.height = CONFIG_GRINREFLEX_VIDEO_HEIGHT; src.stride = CONFIG_GRINREFLEX_VIDEO_WIDTH * LV_COLOR_FORMAT_GET_SIZE(JPEG_COLOR_FORMAT); @@ -137,5 +208,80 @@ int loop() LOG_ERR("Unable to requeue video buf"); return 0; } +#else /* !defined(CONFIG_OPENCV_LIB) */ + + Gfx::GfxBuffer src{}; + Gfx::GfxBuffer dst{}; + + src.buf = input_frame_buffer; + src.width = CONFIG_GRINREFLEX_VIDEO_WIDTH; + src.height = CONFIG_GRINREFLEX_VIDEO_HEIGHT; + src.stride = CONFIG_GRINREFLEX_VIDEO_WIDTH * LV_COLOR_FORMAT_GET_SIZE(JPEG_COLOR_FORMAT); + src.cf = JPEG_COLOR_FORMAT; + src.size = sizeof(input_frame_buffer); + + dst.buf = gray_frame_buffer; + dst.width = GRAY_FRAME_WIDTH; + dst.height = GRAY_FRAME_HEIGHT; + dst.stride = GRAY_FRAME_WIDTH * LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8); + dst.cf = LV_COLOR_FORMAT_L8; + dst.size = sizeof(gray_frame_buffer); + + Gfx::fit(vid_canvas, src, dst); + lv_task_handler(); + + err = video_enqueue(video_dev, vbuf_ptr); + if (err) { + LOG_ERR("Unable to requeue video buf"); + return 0; + } + + cv::Mat matGraySmall(GRAY_FRAME_HEIGHT, GRAY_FRAME_WIDTH, CV_8UC1, gray_frame_buffer); + + if (JPEG_COLOR_FORMAT != LV_COLOR_FORMAT_L8) { + LOG_ERR("Format is not supported"); + return -3; + } + + cv::Mat matGrayFull(CONFIG_GRINREFLEX_VIDEO_HEIGHT, CONFIG_GRINREFLEX_VIDEO_WIDTH, CV_8UC1, input_frame_buffer); + + cv::Rect faceROIMax{}; + faceROIMax.width = GRAY_FRAME_WIDTH; + faceROIMax.height = GRAY_FRAME_HEIGHT; + + std::vector objects = detectFaceAndSmile( + faceCascade, + smileCascade, + matGraySmall, + matGrayFull, + faceROIMax + ); + + lv_color_t c = !objects.empty() ? lv_palette_main(LV_PALETTE_GREEN) + : lv_palette_main(LV_PALETTE_RED); + lv_obj_set_style_bg_color(RedGreenDot, c, 0); + + if (objects.size()) { + lv_obj_remove_flag(ROIRectFace, LV_OBJ_FLAG_HIDDEN); + + auto object = objects[0]; + lv_obj_align_to(ROIRectFace, vid_canvas, LV_ALIGN_TOP_LEFT, object.x, object.y); + lv_obj_set_size(ROIRectFace, object.width, object.height); + + } else { + lv_obj_add_flag(ROIRectFace, LV_OBJ_FLAG_HIDDEN); + } + + if (objects.size() > 1) { + auto object = objects[1]; + lv_obj_align_to(ROIRectSmile, vid_canvas, LV_ALIGN_TOP_LEFT, object.x, object.y); + lv_obj_set_size(ROIRectSmile, object.width, object.height); + + lv_obj_remove_flag(ROIRectSmile, LV_OBJ_FLAG_HIDDEN); + } else { + lv_obj_add_flag(ROIRectSmile, LV_OBJ_FLAG_HIDDEN); + } +#endif /* !defined(CONFIG_OPENCV_LIB) */ + return 0; } \ No newline at end of file diff --git a/u5/app/src/opencv_utils.cpp b/u5/app/src/opencv_utils.cpp index 3b003f7..6e32d35 100644 --- a/u5/app/src/opencv_utils.cpp +++ b/u5/app/src/opencv_utils.cpp @@ -7,47 +7,40 @@ #include #include -#include - #include "opencv_utils.hpp" bool loadCascade(cv::CascadeClassifier &cascade, const char *path) { - uint32_t start_ms = k_uptime_get_32(); - if (!cascade.load(path)) { std::cerr << "Cannot load cascade: " << path << "\n"; return false; } - - uint32_t end_ms = k_uptime_get_32(); - uint32_t elapsed = end_ms - start_ms; - - std::cout << "cascade.load() Took : " << elapsed << " ms" << std::endl; return true; } -bool detectFaces(cv::CascadeClassifier &cascade, cv::Mat &gray, std::vector &faces) { - const cv::Size minSize(24, 24); - const cv::Size maxSize; - const double scaleFactor = 1.04; - const int minNeighbors = 1; - const int flags = 0; - - uint32_t start_ms = k_uptime_get_32(); - - cascade.detectMultiScale(gray, faces, scaleFactor, minNeighbors, - flags, minSize, maxSize); +bool loadCascade(cv::CascadeClassifier &cascade, cv::FileStorage &fs) { + if (!cascade.read(fs.getFirstTopLevelNode())) { + std::cerr << "Cannot load cascade\n"; + return false; + } + return true; +} - uint32_t end_ms = k_uptime_get_32(); - uint32_t elapsed = end_ms - start_ms; +bool loadCascade(cv::CascadeClassifier &cascade, const uint8_t *buffer, size_t len) { + cv::FileStorage fs(std::string((const char *)buffer, len), cv::FileStorage::READ | cv::FileStorage::MEMORY); + return loadCascade(cascade, fs); +} - bool facesFound = ! faces.empty(); +void detectObjects(cv::CascadeClassifier &cascade, cv::Mat &gray, std::vector &objects) { + const double scaleFactor = 1.1; + const int minNeighbors = 3; + const int flags = 0; - if (facesFound) { - printf("detectMultiScale : %d ms\n", elapsed); - } + cascade.detectMultiScale(gray, objects, scaleFactor, minNeighbors, + flags); +} - return facesFound; +void detectFaces(cv::CascadeClassifier &cascade, cv::Mat &gray, std::vector &faces) { + detectObjects(cascade, gray, faces); } cv::Mat cv_preprocessForQR(const cv::Mat& bgr) { @@ -58,4 +51,115 @@ cv::Mat cv_preprocessForQR(const cv::Mat& bgr) { cv::equalizeHist(gray, eq); cv::adaptiveThreshold(eq, bin, 255, cv::ADAPTIVE_THRESH_MEAN_C, cv::THRESH_BINARY, 31, 5); return bin; -} \ No newline at end of file +} + +std::vector detectFaceAndSmile( + cv::CascadeClassifier &faceCascade, + cv::CascadeClassifier &smileCascade, + cv::Mat &grayScreen, + cv::Mat &grayFull, + cv::Rect &faceROIMax) { + + std::vector outROIs; + std::vector faceROIs; + std::vector smileROIs; + + const double scaleFactor = 1.05; + const int minNeighbors = 3; + const int flags = 0; + + faceCascade.detectMultiScale(grayScreen, faceROIs, scaleFactor, minNeighbors, + flags); + + for (auto & faceROI : faceROIs) { + outROIs.push_back(faceROI); + + try { + + cv::Rect faceROI_Orig = remapROI(faceROI, grayScreen, grayFull); + + /* Get lower half/third of face ROI + some gap where the smile is most likelly to be */ + const uint32_t offset = (faceROI_Orig.height * 2) / 3; + faceROI_Orig.y = faceROI_Orig.y + offset; + //faceROI_Orig.height = faceROI_Orig.height - faceROI_Orig.height / 3; + if (faceROI_Orig.height + faceROI_Orig.y > grayFull.rows) { + faceROI_Orig.height = grayFull.rows - faceROI_Orig.y; + } + + float sx = 1.0f; + float sy = 1.0f; + + cv::Mat faceROIFrame = grayFull(faceROI_Orig); + cv::Mat faceROIFrameResized; + + // Get area with minimum number of pixels + if (faceROIMax.width * faceROIMax.height < faceROI_Orig.width * faceROI_Orig.height) { + sx = (float)faceROIMax.width / (float)faceROI_Orig.width; + sy = (float)faceROIMax.height / (float)faceROI_Orig.height; + } + cv::resize(faceROIFrame, faceROIFrameResized, cv::Size(), sx, sy, cv::INTER_LINEAR); + + const double scaleFactor = 1.05; + const int minNeighbors = 3; + const int flags = 0; + + smileCascade.detectMultiScale(faceROIFrameResized, smileROIs, scaleFactor, minNeighbors, + flags); + + for (auto & smileROI : smileROIs) { + cv::Rect smileROIRemapped = translateScaleROI(smileROI, 1.0f / sx, 1.0f / sy, faceROI_Orig.x, faceROI_Orig.y); + + float ssx_inv = (float)grayScreen.cols / (float)grayFull.cols; + float ssy_inv = (float)grayScreen.rows / (float)grayFull.rows; + + smileROIRemapped = translateScaleROI(smileROIRemapped, ssx_inv, ssy_inv, 0.0f, 0.0f); + + outROIs.push_back(smileROIRemapped); + } + + } catch (cv::Exception &ce) { + std::cerr << "Exception: " << ce.what() << std::endl; + } catch ( ... ) { + std::cerr << "Unknown Exception"; + } + } + + return outROIs; +} + +cv::Rect remapROI(cv::Rect &ROI, cv::Mat &inFrame, cv::Mat &outFrame) { + float sx = (float)outFrame.cols / (float)inFrame.cols; + float sy = (float)outFrame.rows / (float)inFrame.rows; + return translateScaleROI(ROI, sx, sy, 0.0f, 0.0f); +} + +cv::Rect translateScaleROI(cv::Rect &ROI, float sx, float sy, float tx, float ty) { + cv::Rect OutROI; + + float inROIVecData[6] = { + (float)ROI.x, (float)(ROI.x + ROI.width), + (float)ROI.y, (float)(ROI.y + ROI.height), + 1.0f, 1.0f, + }; + + float scaleVecData[9] = { + sx, 0.0f, tx, + 0.0f, sy, ty, + 0.0f, 0.0f, 1.0f + }; + + cv::Mat scaleMat(3, 3, CV_32FC1, &scaleVecData[0]); + cv::Mat inROIMat(3, 2, CV_32FC1, &inROIVecData[0]); + + float outROIVecData[6]{}; + cv::Mat outROIMat(3, 2, CV_32FC1, &outROIVecData[0]); + + cv::gemm(scaleMat, inROIMat, 1.0f, cv::Mat(), 1.0f, outROIMat); + + OutROI.x = outROIVecData[0]; + OutROI.y = outROIVecData[2]; + OutROI.width = outROIVecData[1] - outROIVecData[0]; + OutROI.height = outROIVecData[3] - outROIVecData[2]; + + return OutROI; +} diff --git a/u5/app/src/opencv_utils.hpp b/u5/app/src/opencv_utils.hpp index 4232eea..ed2f715 100644 --- a/u5/app/src/opencv_utils.hpp +++ b/u5/app/src/opencv_utils.hpp @@ -10,7 +10,46 @@ #endif bool loadCascade(cv::CascadeClassifier &cascade, const char *path); - -bool detectFaces(cv::CascadeClassifier &cascade, cv::Mat &gray, std::vector &faces); +bool loadCascade(cv::CascadeClassifier &cascade, cv::FileStorage &fs); +bool loadCascade(cv::CascadeClassifier &cascade, const uint8_t *buffer, size_t len); +/** + * @brief + * + * @param faceCascade + * @param smileCascade + * @param grayScreen this is the frame to be displayed on the screen, usually smaller that original one + * @param grayFull this is the original frame, to crop from to detect smile + * @param faceROIMax maximum faceROI to be resized to if detected ROI is largen than this + * @return std::vector bunch of ROI's to be drawn on the screen (grayScreen) + */ +std::vector detectFaceAndSmile( + cv::CascadeClassifier &faceCascade, + cv::CascadeClassifier &smileCascade, + cv::Mat &grayScreen, + cv::Mat &grayFull, + cv::Rect &faceROIMax); cv::Mat cv_preprocessForQR(const cv::Mat& bgr); + +/** + * @brief Remaps ROI that was defined withing inFrame to the outFrame + * + * @param ROI original ROI rect + * @param inFrame input frame + * @param outFrame output frame + * @return cv::Rect remapped ROI rect + */ +cv::Rect remapROI(cv::Rect &ROI, cv::Mat &inFrame, cv::Mat &outFrame); + +/** + * @brief Translate and scale ROI using homogenous transformations + * + * @param ROI + * @param sx + * @param sy + * @param tx + * @param ty + * @return cv::Rect + */ +cv::Rect translateScaleROI(cv::Rect &ROI, float sx, float sy, float tx, float ty); + diff --git a/u5/boards/vendor/grinreflex_dk2/grinreflex_dk2_defconfig b/u5/boards/vendor/grinreflex_dk2/grinreflex_dk2_defconfig index 608a2ca..380feb8 100644 --- a/u5/boards/vendor/grinreflex_dk2/grinreflex_dk2_defconfig +++ b/u5/boards/vendor/grinreflex_dk2/grinreflex_dk2_defconfig @@ -21,6 +21,7 @@ CONFIG_GPIO=y CONFIG_VIDEO=y CONFIG_VIDEO_LOG_LEVEL_DBG=y -CONFIG_VIDEO_BUFFER_POOL_SZ_MAX=76800 +#CONFIG_VIDEO_BUFFER_POOL_SZ_MAX=76800 +CONFIG_VIDEO_BUFFER_POOL_SZ_MAX=131072 #CONFIG_LV_COLOR_16_SWAP=y \ No newline at end of file diff --git a/u5/lib/cv/CMakeLists.txt b/u5/lib/cv/CMakeLists.txt index 4ad51b9..6cc67cb 100644 --- a/u5/lib/cv/CMakeLists.txt +++ b/u5/lib/cv/CMakeLists.txt @@ -24,7 +24,7 @@ set(CMAKE_CXX_FLAGS \"${external_project_cxxflags} -DOPENCV_FLANN_USE_STD_RAND\" ") set(OPENCV_BINARY_DIR ${CMAKE_BINARY_DIR}/opencv) -set(OPENCV_INSTALL_DIR ${OPENCV_BINARY_DIR}/install) +set(OPENCV_INSTALL_DIR ${OPENCV_BINARY_DIR}/install CACHE INTERNAL "") add_custom_target(opencv_cmake DEPENDS ${OPENCV_BINARY_DIR}/CMakeCache.txt From 48a58c41b2ff508e556f9e4649fb3dc2877e55fd Mon Sep 17 00:00:00 2001 From: Roman Pustobaiev Date: Wed, 1 Oct 2025 18:17:25 +0200 Subject: [PATCH 02/18] Refactoring, improvements, tweaks with memory Face and Smile detection works more or less reliable now Needs further work --- host/Src/main.cpp | 27 ++- u5/app/CMakeLists.txt | 6 + u5/app/prj.conf | 12 +- u5/app/src/gfx_utils.cpp | 4 +- u5/app/src/gfx_utils.hpp | 7 +- u5/app/src/grinreflex.h | 25 --- u5/app/src/grinreflex_config.hpp | 34 ++++ u5/app/src/grinreflex_utils.cpp | 81 +++++++++ u5/app/src/grinreflex_utils.hpp | 10 ++ u5/app/src/grinreflex_v2.cpp | 164 ++++-------------- u5/app/src/opencv_utils.cpp | 51 +++--- u5/app/src/opencv_utils.hpp | 17 +- u5/app/src/video.cpp | 33 +++- .../grinreflex/grinreflex_dk1_defconfig | 12 +- .../grinreflex_dk2/grinreflex_dk2_defconfig | 11 +- u5/lib/cv/opencv | 2 +- 16 files changed, 284 insertions(+), 212 deletions(-) create mode 100644 u5/app/src/grinreflex_config.hpp create mode 100644 u5/app/src/grinreflex_utils.cpp create mode 100644 u5/app/src/grinreflex_utils.hpp diff --git a/host/Src/main.cpp b/host/Src/main.cpp index 52805e2..3db0284 100644 --- a/host/Src/main.cpp +++ b/host/Src/main.cpp @@ -49,24 +49,37 @@ int main(int, char**) { cv::Mat frameOriginalGray; cv::cvtColor(frameOriginal, frameOriginalGray, cv::COLOR_BGR2GRAY); - cv::equalizeHist(frameOriginalGray, frameOriginalGray); + //cv::equalizeHist(frameOriginalGray, frameOriginalGray); + + cv::Mat frameCroppedGray; + + cv::Rect crop; + crop.x = (frameOriginalGray.cols - 320) / 2; + crop.width = 320; + crop.y = (frameOriginalGray.rows - 240) / 2; + crop.height = 240; + frameCroppedGray = frameOriginalGray(crop); cv::Rect faceROIMax{}; - faceROIMax.width = 80; - faceROIMax.height = 80; + faceROIMax.width = 60; + faceROIMax.height = 60; + + cv::resize(frameCroppedGray, frameScreenGray, cv::Size(120, 120), 0.0f, 0.0f, cv::INTER_NEAREST ); - cv::resize(frameOriginalGray, frameScreenGray, cv::Size(faceROIMax.width, faceROIMax.height), 0.0f, 0.0f, cv::INTER_LINEAR); + std::vector faces, smiles; std::vector ROIs = detectFaceAndSmile( face_cascade, smile_cascade, frameScreenGray, - frameOriginalGray, - faceROIMax + frameCroppedGray, + faceROIMax, + faces, + smiles ); for (const auto &ROI : ROIs) { - rectangle(frameScreenGray, ROI, cv::Scalar(0, 0, 255), 2, 8, 0); + rectangle(frameScreenGray, ROI, cv::Scalar(0, 0, 255), 1, 1, 0); } cv::imshow(window_name, frameScreenGray); diff --git a/u5/app/CMakeLists.txt b/u5/app/CMakeLists.txt index 3ad0d3e..d3f5f6e 100644 --- a/u5/app/CMakeLists.txt +++ b/u5/app/CMakeLists.txt @@ -36,6 +36,12 @@ target_sources(app PRIVATE src/video.cpp ) +if (CONFIG_BOARD_GRINREFLEX_DK1 OR CONFIG_BOARD_GRINREFLEX_DK2) +target_sources(app PRIVATE + src/grinreflex_utils.cpp +) +endif() + if (CONFIG_BOARD_GRINREFLEX_DK1) target_sources(app PRIVATE src/sample_usbd_init.c diff --git a/u5/app/prj.conf b/u5/app/prj.conf index fa4f882..07a4b56 100644 --- a/u5/app/prj.conf +++ b/u5/app/prj.conf @@ -40,10 +40,9 @@ CONFIG_DEBUG_OPTIMIZATIONS=y CONFIG_COMMON_LIBC_MALLOC=y CONFIG_HEAP_MEM_POOL_SIZE=49152 -CONFIG_COMMON_LIBC_MALLOC_ARENA_SIZE=2228224 +CONFIG_COMMON_LIBC_MALLOC_ARENA_SIZE=2097152 #CONFIG_COMMON_LIBC_MALLOC_ARENA_SIZE=1048576 - #CONFIG_TFLITE_LIB=y CONFIG_OPENCV_LIB=y #CONFIG_QUIRC_LIB=y @@ -73,15 +72,6 @@ CONFIG_CPP_RTTI=y #CONFIG_DISPLAY_LOG_LEVEL_DBG=y CONFIG_DMA_LOG_LEVEL_INF=y -CONFIG_VIDEO_LOG_LEVEL_INF=y - -CONFIG_STM32_JPEG_RGB_FORMAT=3 -CONFIG_GRINREFLEX_JPEG_VIDEO=y -CONFIG_GRINREFLEX_JPEG_VIDEO_QUALITY=50 -CONFIG_GRINREFLEX_VIDEO_WIDTH=320 -CONFIG_GRINREFLEX_VIDEO_HEIGHT=240 - - CONFIG_HW_STACK_PROTECTION=y diff --git a/u5/app/src/gfx_utils.cpp b/u5/app/src/gfx_utils.cpp index 5d3de9f..e378c3e 100644 --- a/u5/app/src/gfx_utils.cpp +++ b/u5/app/src/gfx_utils.cpp @@ -3,7 +3,7 @@ namespace Gfx { -void fit(lv_obj_t *canvas, const GfxBuffer &src, const GfxBuffer &dst) { +void fit(lv_obj_t *canvas, const GfxBuffer &src, const GfxBuffer &dst, const int32_t x, const int32_t y) { lv_image_dsc_t src_img = { .header = @@ -31,7 +31,7 @@ void fit(lv_obj_t *canvas, const GfxBuffer &src, const GfxBuffer &dst) { img_dsc.scale_x = (LV_SCALE_NONE * (int32_t)(dst.width)) / src.width; img_dsc.scale_y = (LV_SCALE_NONE * (int32_t)(dst.height)) / src.height; - lv_area_t coords_img = {0, 0, src.width - 1, src.height - 1}; + lv_area_t coords_img = {x, y, src.width - 1, src.height - 1}; lv_draw_image(&layer, &img_dsc, &coords_img); diff --git a/u5/app/src/gfx_utils.hpp b/u5/app/src/gfx_utils.hpp index 4f6d177..fc662e9 100644 --- a/u5/app/src/gfx_utils.hpp +++ b/u5/app/src/gfx_utils.hpp @@ -8,6 +8,11 @@ namespace Gfx { +struct GfxRect { + uint32_t x, y; + uint32_t width, height; +}; + struct GfxBuffer { uint8_t *buf; uint32_t width, height, stride; @@ -19,6 +24,6 @@ struct GfxBuffer { * @brief Fit source buffer onto destination * */ -void fit(lv_obj_t *canvas, const GfxBuffer &src, const GfxBuffer &dst); +void fit(lv_obj_t *canvas, const GfxBuffer &src, const GfxBuffer &dst, const int32_t x = 0, const int32_t y = 0); } // namespace Gfx diff --git a/u5/app/src/grinreflex.h b/u5/app/src/grinreflex.h index d152488..73afd7a 100644 --- a/u5/app/src/grinreflex.h +++ b/u5/app/src/grinreflex.h @@ -9,31 +9,6 @@ extern "C" { #endif -#if defined(CONFIG_GRINREFLEX_JPEG_VIDEO) - -#if defined(CONFIG_STM32_JPEG_RGB_FORMAT) -#if CONFIG_STM32_JPEG_RGB_FORMAT == 0 -#define JPEG_COLOR_FORMAT LV_COLOR_FORMAT_ARGB8888 -#elif CONFIG_STM32_JPEG_RGB_FORMAT == 1 -#define JPEG_COLOR_FORMAT LV_COLOR_FORMAT_RGB888 -#elif CONFIG_STM32_JPEG_RGB_FORMAT == 2 -#define JPEG_COLOR_FORMAT LV_COLOR_FORMAT_RGB565 -#elif CONFIG_STM32_JPEG_RGB_FORMAT == 3 -#define JPEG_COLOR_FORMAT LV_COLOR_FORMAT_L8 -#else -#error "CONFIG_STM32_JPEG_RGB_FORMAT is not supported" -#endif - -#else -#define JPEG_COLOR_FORMAT LV_COLOR_FORMAT_RGB888 -#endif /* defined(CONFIG_STM32_JPEG_RGB_FORMAT) */ - -#endif /* defined(CONFIG_GRINREFLEX_JPEG_VIDEO) */ - -#define GRAY_FRAME_WIDTH 80 -#define GRAY_FRAME_HEIGHT 80 - - int init(); int loop(); diff --git a/u5/app/src/grinreflex_config.hpp b/u5/app/src/grinreflex_config.hpp new file mode 100644 index 0000000..7de3e79 --- /dev/null +++ b/u5/app/src/grinreflex_config.hpp @@ -0,0 +1,34 @@ + + +#pragma once + + +#if !defined(CONFIG_GRINREFLEX_JPEG_VIDEO) +#error "Not supported yet" +#endif + +#if defined(CONFIG_STM32_JPEG_RGB_FORMAT) +#if CONFIG_STM32_JPEG_RGB_FORMAT == 0 +#define FULL_FRAME_COLOR_FORMAT LV_COLOR_FORMAT_ARGB8888 +#elif CONFIG_STM32_JPEG_RGB_FORMAT == 1 +#define FULL_FRAME_COLOR_FORMAT LV_COLOR_FORMAT_RGB888 +#elif CONFIG_STM32_JPEG_RGB_FORMAT == 2 +#define FULL_FRAME_COLOR_FORMAT LV_COLOR_FORMAT_RGB565 +#elif CONFIG_STM32_JPEG_RGB_FORMAT == 3 +#define FULL_FRAME_COLOR_FORMAT LV_COLOR_FORMAT_L8 +#else +#error "CONFIG_STM32_JPEG_RGB_FORMAT is not supported" +#endif + +#else +#define FULL_FRAME_COLOR_FORMAT LV_COLOR_FORMAT_RGB888 +#endif /* defined(CONFIG_STM32_JPEG_RGB_FORMAT) */ + +#define FULL_FRAME_WIDTH CONFIG_GRINREFLEX_VIDEO_WIDTH +#define FULL_FRAME_HEIGHT CONFIG_GRINREFLEX_VIDEO_HEIGHT + +#define ROI_FRAME_WIDTH 320 +#define ROI_FRAME_HEIGHT 240 + +#define THUMBNAIL_FRAME_WIDTH 80 +#define THUMBNAIL_FRAME_HEIGHT 80 \ No newline at end of file diff --git a/u5/app/src/grinreflex_utils.cpp b/u5/app/src/grinreflex_utils.cpp new file mode 100644 index 0000000..d9db464 --- /dev/null +++ b/u5/app/src/grinreflex_utils.cpp @@ -0,0 +1,81 @@ +#include "grinreflex_config.hpp" +#include "gfx_utils.hpp" + +#include "grinreflex_utils.hpp" + +lv_obj_t *allocLvROIRect(lv_obj_t *parent, uint32_t borderWidth) { + lv_obj_t *rect = lv_obj_create(parent); + lv_obj_set_pos(rect, 0, 0); + lv_obj_set_size(rect, 1, 1); + + // Unfilled, just a thick border + lv_obj_set_style_bg_opa(rect, LV_OPA_TRANSP, LV_PART_MAIN); + lv_obj_set_style_border_width(rect, borderWidth, LV_PART_MAIN); + lv_obj_set_style_border_opa(rect, LV_OPA_COVER, LV_PART_MAIN); + lv_obj_set_style_border_color(rect, lv_palette_main(LV_PALETTE_GREEN), LV_PART_MAIN); + lv_obj_set_style_radius(rect, 0, LV_PART_MAIN); // 0 = sharp corners + lv_obj_set_style_pad_all(rect, 0, LV_PART_MAIN); + lv_obj_set_style_outline_opa(rect, LV_OPA_TRANSP, LV_PART_MAIN); // no focus outline + lv_obj_set_style_shadow_opa(rect, LV_OPA_TRANSP, LV_PART_MAIN); // no shadow + lv_obj_move_foreground(rect); + lv_obj_add_flag(rect, LV_OBJ_FLAG_HIDDEN); + + return rect; +} + +void cropFullFrameToRoi(lv_obj_t *parent, uint8_t *fullFb, uint8_t *roiFb) { + Gfx::GfxBuffer src{}; + Gfx::GfxBuffer dst{}; + + uint32_t x = (FULL_FRAME_WIDTH - ROI_FRAME_WIDTH) / 2; + uint32_t y = (FULL_FRAME_HEIGHT - ROI_FRAME_HEIGHT) / 2; + + uint32_t fullFbStride = FULL_FRAME_WIDTH * LV_COLOR_FORMAT_GET_SIZE(FULL_FRAME_COLOR_FORMAT); + uint32_t fullFbOffset = (y * fullFbStride) + x; + uint32_t fullFbSize = ROI_FRAME_HEIGHT * fullFbStride; + + src.buf = fullFb + fullFbOffset; + src.width = ROI_FRAME_WIDTH; + src.height = ROI_FRAME_HEIGHT; + src.stride = fullFbStride; + src.cf = FULL_FRAME_COLOR_FORMAT; + src.size = fullFbSize - fullFbOffset; + + uint32_t roiFbStride = ROI_FRAME_WIDTH * LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8); + + dst.buf = roiFb; + dst.width = ROI_FRAME_WIDTH; + dst.height = ROI_FRAME_HEIGHT; + dst.stride = roiFbStride; + dst.cf = LV_COLOR_FORMAT_L8; + dst.size = ROI_FRAME_HEIGHT * roiFbStride; + + Gfx::fit(parent, src, dst); + lv_task_handler(); +} + +void fitRoiFrameToThumbnail(lv_obj_t *parent, uint8_t *roiFb, uint8_t *tFb) { + Gfx::GfxBuffer src{}; + Gfx::GfxBuffer dst{}; + + uint32_t roiFbStride = ROI_FRAME_WIDTH * LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8); + + src.buf = roiFb; + src.width = ROI_FRAME_WIDTH; + src.height = ROI_FRAME_HEIGHT; + src.stride = roiFbStride; + src.cf = LV_COLOR_FORMAT_L8; + src.size = ROI_FRAME_HEIGHT * roiFbStride; + + uint32_t tnFbStride = THUMBNAIL_FRAME_WIDTH * LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8); + + dst.buf = tFb; + dst.width = THUMBNAIL_FRAME_WIDTH; + dst.height = THUMBNAIL_FRAME_HEIGHT; + dst.stride = tnFbStride; + dst.cf = LV_COLOR_FORMAT_L8; + dst.size = THUMBNAIL_FRAME_HEIGHT * tnFbStride; + + Gfx::fit(parent, src, dst); + lv_task_handler(); +} diff --git a/u5/app/src/grinreflex_utils.hpp b/u5/app/src/grinreflex_utils.hpp new file mode 100644 index 0000000..0cba9ad --- /dev/null +++ b/u5/app/src/grinreflex_utils.hpp @@ -0,0 +1,10 @@ + + +#pragma once + +#include + +lv_obj_t *allocLvROIRect(lv_obj_t *parent, uint32_t borderWidth); + +void cropFullFrameToRoi(lv_obj_t *parent, uint8_t *fullFb, uint8_t *roiFb); +void fitRoiFrameToThumbnail(lv_obj_t *parent, uint8_t *roiFb, uint8_t *tFb); diff --git a/u5/app/src/grinreflex_v2.cpp b/u5/app/src/grinreflex_v2.cpp index dac70e2..5f71dfc 100644 --- a/u5/app/src/grinreflex_v2.cpp +++ b/u5/app/src/grinreflex_v2.cpp @@ -29,22 +29,23 @@ LOG_MODULE_REGISTER(grinreflex_app); #include +#include "grinreflex_config.hpp" +#include "grinreflex_utils.hpp" #include "grinreflex.h" static const struct device *video_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_camera)); static const struct device *display_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_display)); const struct device *jpeg_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_jpeg)); -static lv_img_dsc_t video_img; -static lv_obj_t *vid_canvas; -static lv_obj_t *screen; - -static uint8_t input_frame_buffer[CONFIG_GRINREFLEX_VIDEO_WIDTH * CONFIG_GRINREFLEX_VIDEO_HEIGHT * LV_COLOR_FORMAT_GET_SIZE(JPEG_COLOR_FORMAT)]; +static uint8_t fullFrameBuffer[FULL_FRAME_WIDTH * FULL_FRAME_HEIGHT * LV_COLOR_FORMAT_GET_SIZE(FULL_FRAME_COLOR_FORMAT)]; +static uint8_t roiFrameBuffer[ROI_FRAME_WIDTH * ROI_FRAME_HEIGHT * LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8)]; +static uint8_t thumbnailFrameBuffer[THUMBNAIL_FRAME_WIDTH * THUMBNAIL_FRAME_HEIGHT * LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8)]; -static uint8_t gray_frame_buffer[GRAY_FRAME_WIDTH * GRAY_FRAME_HEIGHT * LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8)]; +static lv_obj_t *screen_canvas; +static lv_obj_t *dummy_canvas; +static lv_obj_t *screen; #if defined(CONFIG_OPENCV_LIB) -static lv_obj_t *RedGreenDot; static lv_obj_t *ROIRectFace; static lv_obj_t *ROIRectSmile; static cv::CascadeClassifier faceCascade; @@ -86,65 +87,18 @@ int init() Video::setup(); - vid_canvas = lv_canvas_create(lv_screen_active()); - lv_canvas_fill_bg(vid_canvas, lv_color_hex3(0xccc), LV_OPA_COVER); - lv_obj_center(vid_canvas); + dummy_canvas = lv_canvas_create(NULL); + screen_canvas = lv_canvas_create(lv_screen_active()); + lv_canvas_fill_bg(screen_canvas, lv_color_hex3(0xccc), LV_OPA_COVER); + lv_obj_center(screen_canvas); display_blanking_off(display_dev); - video_img.header.w = CONFIG_GRINREFLEX_VIDEO_WIDTH; - video_img.header.h = CONFIG_GRINREFLEX_VIDEO_HEIGHT; -#if defined(CONFIG_GRINREFLEX_JPEG_VIDEO) - video_img.data_size = CONFIG_GRINREFLEX_VIDEO_WIDTH * CONFIG_GRINREFLEX_VIDEO_HEIGHT * LV_COLOR_FORMAT_GET_SIZE(JPEG_COLOR_FORMAT); - video_img.header.cf = JPEG_COLOR_FORMAT; -#else - video_img.data_size = CONFIG_GRINREFLEX_VIDEO_WIDTH * CONFIG_GRINREFLEX_VIDEO_HEIGHT * LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_NATIVE); - video_img.header.cf = LV_COLOR_FORMAT_NATIVE; -#endif screen = lv_img_create(lv_scr_act()); #if defined(CONFIG_OPENCV_LIB) - RedGreenDot = lv_obj_create(lv_scr_act()); - lv_obj_set_size(RedGreenDot, 30, 30); // dot size - lv_obj_set_style_radius(RedGreenDot, LV_RADIUS_CIRCLE, 0); // make it round - lv_obj_set_style_border_width(RedGreenDot, 0, 0); - lv_obj_set_style_bg_opa(RedGreenDot, LV_OPA_COVER, 0); - lv_obj_clear_flag(RedGreenDot, LV_OBJ_FLAG_SCROLLABLE); - lv_obj_align(RedGreenDot, LV_ALIGN_LEFT_MID, 4, 0); // 4px from left - lv_obj_move_foreground(RedGreenDot); // keep it on top - - ROIRectFace = lv_obj_create(lv_scr_act()); - lv_obj_set_pos(ROIRectFace, 0, 0); - lv_obj_set_size(ROIRectFace, 1, 1); - - // Unfilled, just a thick border - lv_obj_set_style_bg_opa(ROIRectFace, LV_OPA_TRANSP, LV_PART_MAIN); - lv_obj_set_style_border_width(ROIRectFace, 4, LV_PART_MAIN); - lv_obj_set_style_border_opa(ROIRectFace, LV_OPA_COVER, LV_PART_MAIN); - lv_obj_set_style_border_color(ROIRectFace, lv_palette_main(LV_PALETTE_GREEN), LV_PART_MAIN); - lv_obj_set_style_radius(ROIRectFace, 0, LV_PART_MAIN); // 0 = sharp corners - lv_obj_set_style_pad_all(ROIRectFace, 0, LV_PART_MAIN); - lv_obj_set_style_outline_opa(ROIRectFace, LV_OPA_TRANSP, LV_PART_MAIN); // no focus outline - lv_obj_set_style_shadow_opa(ROIRectFace, LV_OPA_TRANSP, LV_PART_MAIN); // no shadow - lv_obj_move_foreground(ROIRectFace); - lv_obj_add_flag(ROIRectFace, LV_OBJ_FLAG_HIDDEN); - - ROIRectSmile = lv_obj_create(lv_scr_act()); - lv_obj_set_pos(ROIRectSmile, 0, 0); - lv_obj_set_size(ROIRectSmile, 1, 1); - - // Unfilled, just a thick border - lv_obj_set_style_bg_opa(ROIRectSmile, LV_OPA_TRANSP, LV_PART_MAIN); - lv_obj_set_style_border_width(ROIRectSmile, 2, LV_PART_MAIN); - lv_obj_set_style_border_opa(ROIRectSmile, LV_OPA_COVER, LV_PART_MAIN); - lv_obj_set_style_border_color(ROIRectSmile, lv_palette_main(LV_PALETTE_GREEN), LV_PART_MAIN); - lv_obj_set_style_radius(ROIRectSmile, 0, LV_PART_MAIN); // 0 = sharp corners - lv_obj_set_style_pad_all(ROIRectSmile, 0, LV_PART_MAIN); - lv_obj_set_style_outline_opa(ROIRectSmile, LV_OPA_TRANSP, LV_PART_MAIN); // no focus outline - lv_obj_set_style_shadow_opa(ROIRectSmile, LV_OPA_TRANSP, LV_PART_MAIN); // no shadow - lv_obj_move_foreground(ROIRectSmile); - lv_obj_add_flag(ROIRectSmile, LV_OBJ_FLAG_HIDDEN); - + ROIRectFace = allocLvROIRect(lv_scr_act(), 4); + ROIRectSmile = allocLvROIRect(lv_scr_act(), 2); #endif return 0; @@ -165,7 +119,6 @@ int loop() return 0; } -#if defined(CONFIG_GRINREFLEX_JPEG_VIDEO) //jpeg_hw_decode(jpeg_dev, (uint8_t *)vbuf_ptr->buffer, vbuf_ptr->bytesused, jpeg_frame_buffer); jpeg_hw_poll(jpeg_dev, 0, &jpeg_prop); @@ -173,35 +126,12 @@ int loop() //printf("JPEG done w = %d, h = %d, color = %d chroma = %d\n", jpeg_prop.width, // jpeg_prop.height, jpeg_prop.color_space, jpeg_prop.chroma); - jpeg_color_convert_helper(jpeg_dev, &jpeg_prop, vbuf_ptr->buffer, input_frame_buffer); - -#else - video_img.data = (uint8_t *)vbuf_ptr->buffer; - lv_img_set_src(screen, &video_img); - lv_obj_align(screen, LV_ALIGN_BOTTOM_LEFT, 0, 0); - lv_task_handler(); -#endif + jpeg_color_convert_helper(jpeg_dev, &jpeg_prop, vbuf_ptr->buffer, fullFrameBuffer); #if !defined(CONFIG_OPENCV_LIB) - Gfx::GfxBuffer src{}; - Gfx::GfxBuffer dst{}; - - src.buf = input_frame_buffer; - src.width = CONFIG_GRINREFLEX_VIDEO_WIDTH; - src.height = CONFIG_GRINREFLEX_VIDEO_HEIGHT; - src.stride = CONFIG_GRINREFLEX_VIDEO_WIDTH * LV_COLOR_FORMAT_GET_SIZE(JPEG_COLOR_FORMAT); - src.cf = JPEG_COLOR_FORMAT; - src.size = CONFIG_GRINREFLEX_VIDEO_HEIGHT * CONFIG_GRINREFLEX_VIDEO_WIDTH * LV_COLOR_FORMAT_GET_SIZE(JPEG_COLOR_FORMAT); - - dst.buf = gray_frame_buffer; - dst.width = GRAY_FRAME_WIDTH; - dst.height = GRAY_FRAME_HEIGHT; - dst.stride = GRAY_FRAME_WIDTH * LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8); - dst.cf = LV_COLOR_FORMAT_L8; - dst.size = sizeof(gray_frame_buffer); - - Gfx::fit(vid_canvas, src, dst); - lv_task_handler(); + + cropFullFrameToRoi(dummy_canvas, fullFrameBuffer, roiFrameBuffer); + fitRoiFrameToThumbnail(screen_canvas, roiFrameBuffer, thumbnailFrameBuffer); err = video_enqueue(video_dev, vbuf_ptr); if (err) { @@ -210,25 +140,8 @@ int loop() } #else /* !defined(CONFIG_OPENCV_LIB) */ - Gfx::GfxBuffer src{}; - Gfx::GfxBuffer dst{}; - - src.buf = input_frame_buffer; - src.width = CONFIG_GRINREFLEX_VIDEO_WIDTH; - src.height = CONFIG_GRINREFLEX_VIDEO_HEIGHT; - src.stride = CONFIG_GRINREFLEX_VIDEO_WIDTH * LV_COLOR_FORMAT_GET_SIZE(JPEG_COLOR_FORMAT); - src.cf = JPEG_COLOR_FORMAT; - src.size = sizeof(input_frame_buffer); - - dst.buf = gray_frame_buffer; - dst.width = GRAY_FRAME_WIDTH; - dst.height = GRAY_FRAME_HEIGHT; - dst.stride = GRAY_FRAME_WIDTH * LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8); - dst.cf = LV_COLOR_FORMAT_L8; - dst.size = sizeof(gray_frame_buffer); - - Gfx::fit(vid_canvas, src, dst); - lv_task_handler(); + cropFullFrameToRoi(dummy_canvas, fullFrameBuffer, roiFrameBuffer); + fitRoiFrameToThumbnail(screen_canvas, roiFrameBuffer, thumbnailFrameBuffer); err = video_enqueue(video_dev, vbuf_ptr); if (err) { @@ -236,45 +149,40 @@ int loop() return 0; } - cv::Mat matGraySmall(GRAY_FRAME_HEIGHT, GRAY_FRAME_WIDTH, CV_8UC1, gray_frame_buffer); - - if (JPEG_COLOR_FORMAT != LV_COLOR_FORMAT_L8) { - LOG_ERR("Format is not supported"); - return -3; - } - - cv::Mat matGrayFull(CONFIG_GRINREFLEX_VIDEO_HEIGHT, CONFIG_GRINREFLEX_VIDEO_WIDTH, CV_8UC1, input_frame_buffer); + cv::Mat matGraySmall(THUMBNAIL_FRAME_HEIGHT, THUMBNAIL_FRAME_WIDTH, CV_8UC1, thumbnailFrameBuffer); + cv::Mat matGrayFull(ROI_FRAME_HEIGHT, ROI_FRAME_WIDTH, CV_8UC1, roiFrameBuffer); cv::Rect faceROIMax{}; - faceROIMax.width = GRAY_FRAME_WIDTH; - faceROIMax.height = GRAY_FRAME_HEIGHT; + faceROIMax.width = 60; + faceROIMax.height = 60; + + std::vector faces; + std::vector smiles; std::vector objects = detectFaceAndSmile( faceCascade, smileCascade, matGraySmall, matGrayFull, - faceROIMax + faceROIMax, + faces, + smiles ); - lv_color_t c = !objects.empty() ? lv_palette_main(LV_PALETTE_GREEN) - : lv_palette_main(LV_PALETTE_RED); - lv_obj_set_style_bg_color(RedGreenDot, c, 0); - - if (objects.size()) { + if (faces.size()) { lv_obj_remove_flag(ROIRectFace, LV_OBJ_FLAG_HIDDEN); - auto object = objects[0]; - lv_obj_align_to(ROIRectFace, vid_canvas, LV_ALIGN_TOP_LEFT, object.x, object.y); + auto object = faces[0]; + lv_obj_align_to(ROIRectFace, screen_canvas, LV_ALIGN_TOP_LEFT, object.x, object.y); lv_obj_set_size(ROIRectFace, object.width, object.height); } else { lv_obj_add_flag(ROIRectFace, LV_OBJ_FLAG_HIDDEN); } - if (objects.size() > 1) { - auto object = objects[1]; - lv_obj_align_to(ROIRectSmile, vid_canvas, LV_ALIGN_TOP_LEFT, object.x, object.y); + if (smiles.size()) { + auto object = smiles[0]; + lv_obj_align_to(ROIRectSmile, screen_canvas, LV_ALIGN_TOP_LEFT, object.x, object.y); lv_obj_set_size(ROIRectSmile, object.width, object.height); lv_obj_remove_flag(ROIRectSmile, LV_OBJ_FLAG_HIDDEN); diff --git a/u5/app/src/opencv_utils.cpp b/u5/app/src/opencv_utils.cpp index 6e32d35..d76217e 100644 --- a/u5/app/src/opencv_utils.cpp +++ b/u5/app/src/opencv_utils.cpp @@ -56,19 +56,19 @@ cv::Mat cv_preprocessForQR(const cv::Mat& bgr) { std::vector detectFaceAndSmile( cv::CascadeClassifier &faceCascade, cv::CascadeClassifier &smileCascade, - cv::Mat &grayScreen, - cv::Mat &grayFull, - cv::Rect &faceROIMax) { + cv::Mat &thumbnailFrame, + cv::Mat &fullFrame, + cv::Rect &faceROIMax, + std::vector &faceROIs, + std::vector &smileROIs) { std::vector outROIs; - std::vector faceROIs; - std::vector smileROIs; - const double scaleFactor = 1.05; + const double scaleFactor = 1.04; const int minNeighbors = 3; const int flags = 0; - faceCascade.detectMultiScale(grayScreen, faceROIs, scaleFactor, minNeighbors, + faceCascade.detectMultiScale(thumbnailFrame, faceROIs, scaleFactor, minNeighbors, flags); for (auto & faceROI : faceROIs) { @@ -76,28 +76,33 @@ std::vector detectFaceAndSmile( try { - cv::Rect faceROI_Orig = remapROI(faceROI, grayScreen, grayFull); + cv::Rect faceROIFull = remapROI(faceROI, thumbnailFrame, fullFrame); /* Get lower half/third of face ROI + some gap where the smile is most likelly to be */ - const uint32_t offset = (faceROI_Orig.height * 2) / 3; - faceROI_Orig.y = faceROI_Orig.y + offset; - //faceROI_Orig.height = faceROI_Orig.height - faceROI_Orig.height / 3; - if (faceROI_Orig.height + faceROI_Orig.y > grayFull.rows) { - faceROI_Orig.height = grayFull.rows - faceROI_Orig.y; + const uint32_t offset = (faceROIFull.height * 2) / 3; + faceROIFull.y = faceROIFull.y + offset; + faceROIFull.height = faceROIFull.height - faceROIFull.height / 2; + if (faceROIFull.height + faceROIFull.y > fullFrame.rows) { + faceROIFull.height = fullFrame.rows - faceROIFull.y; } + cv::Rect roiThumb = remapROI(faceROIFull, fullFrame, thumbnailFrame); + outROIs.push_back(roiThumb); + float sx = 1.0f; float sy = 1.0f; - cv::Mat faceROIFrame = grayFull(faceROI_Orig); + cv::Mat faceROIFrame = fullFrame(faceROIFull); cv::Mat faceROIFrameResized; // Get area with minimum number of pixels - if (faceROIMax.width * faceROIMax.height < faceROI_Orig.width * faceROI_Orig.height) { - sx = (float)faceROIMax.width / (float)faceROI_Orig.width; - sy = (float)faceROIMax.height / (float)faceROI_Orig.height; + if (faceROIMax.width * faceROIMax.height < faceROIFull.width * faceROIFull.height) { + sx = (float)faceROIMax.width / (float)faceROIFull.width; + sy = (float)faceROIMax.height / (float)faceROIFull.height; + cv::resize(faceROIFrame, faceROIFrameResized, cv::Size(), sx, sy, cv::INTER_LINEAR); + } else { + faceROIFrameResized = faceROIFrame; } - cv::resize(faceROIFrame, faceROIFrameResized, cv::Size(), sx, sy, cv::INTER_LINEAR); const double scaleFactor = 1.05; const int minNeighbors = 3; @@ -106,15 +111,15 @@ std::vector detectFaceAndSmile( smileCascade.detectMultiScale(faceROIFrameResized, smileROIs, scaleFactor, minNeighbors, flags); - for (auto & smileROI : smileROIs) { - cv::Rect smileROIRemapped = translateScaleROI(smileROI, 1.0f / sx, 1.0f / sy, faceROI_Orig.x, faceROI_Orig.y); - - float ssx_inv = (float)grayScreen.cols / (float)grayFull.cols; - float ssy_inv = (float)grayScreen.rows / (float)grayFull.rows; + float ssx_inv = (float)thumbnailFrame.cols / (float)fullFrame.cols; + float ssy_inv = (float)thumbnailFrame.rows / (float)fullFrame.rows; + for (auto & smileROI : smileROIs) { + cv::Rect smileROIRemapped = translateScaleROI(smileROI, 1.0f / sx, 1.0f / sy, faceROIFull.x, faceROIFull.y); smileROIRemapped = translateScaleROI(smileROIRemapped, ssx_inv, ssy_inv, 0.0f, 0.0f); outROIs.push_back(smileROIRemapped); + smileROI = smileROIRemapped; } } catch (cv::Exception &ce) { diff --git a/u5/app/src/opencv_utils.hpp b/u5/app/src/opencv_utils.hpp index ed2f715..3fa8719 100644 --- a/u5/app/src/opencv_utils.hpp +++ b/u5/app/src/opencv_utils.hpp @@ -17,17 +17,22 @@ bool loadCascade(cv::CascadeClassifier &cascade, const uint8_t *buffer, size_t l * * @param faceCascade * @param smileCascade - * @param grayScreen this is the frame to be displayed on the screen, usually smaller that original one - * @param grayFull this is the original frame, to crop from to detect smile + * @param thumbnailFrame frame to detect face in, thumbnail, small enough to save memory + * @param fullFrame frame to detect face in, roi is cropped from to have finer frained area * @param faceROIMax maximum faceROI to be resized to if detected ROI is largen than this - * @return std::vector bunch of ROI's to be drawn on the screen (grayScreen) + * @param faces vector with faces rects + * @param smiles vector with sniles rects + * @return std::vector all ROI's ever considered during run */ std::vector detectFaceAndSmile( cv::CascadeClassifier &faceCascade, cv::CascadeClassifier &smileCascade, - cv::Mat &grayScreen, - cv::Mat &grayFull, - cv::Rect &faceROIMax); + cv::Mat &thumbnailFrame, + cv::Mat &fullFrame, + cv::Rect &faceROIMax, + std::vector &faces, + std::vector &smiles +); cv::Mat cv_preprocessForQR(const cv::Mat& bgr); diff --git a/u5/app/src/video.cpp b/u5/app/src/video.cpp index 419fd82..5e770ef 100644 --- a/u5/app/src/video.cpp +++ b/u5/app/src/video.cpp @@ -17,6 +17,10 @@ namespace Video static const struct device *video_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_camera)); static struct video_buffer *buffers[2]; +#if defined(CONFIG_GRINREFLEX_JPEG_VIDEO) +static struct video_buffer second_buffer{}; +#endif + void setup() { struct video_format fmt; @@ -127,11 +131,31 @@ void setup() bsize = fmt.pitch * fmt.height; #if defined(CONFIG_GRINREFLEX_JPEG_VIDEO) - bsize = CONFIG_VIDEO_BUFFER_POOL_SZ_MAX / 2; -#else - bsize = fmt.pitch * fmt.height; -#endif + bsize = CONFIG_VIDEO_BUFFER_POOL_SZ_MAX; + + buffers[0] = video_buffer_aligned_alloc(bsize, CONFIG_VIDEO_BUFFER_POOL_ALIGN, + K_FOREVER); + if (buffers[0] == NULL) { + LOG_ERR("Unable to alloc video buffer"); + return; + } + buffers[0]->type = type; + video_enqueue(video_dev, buffers[0]); + + buffers[1] = &second_buffer; + + bsize = CONFIG_GRINREFLEX_VIDEO_WIDTH * CONFIG_GRINREFLEX_VIDEO_HEIGHT * 2; + buffers[1]->type = type; + buffers[1]->buffer = new uint8_t[bsize]; + buffers[1]->size = bsize; + + if (buffers[1]->buffer == NULL) { + LOG_ERR("Unable to alloc video buffer"); + return; + } + video_enqueue(video_dev, buffers[1]); +#else /* !defined(CONFIG_GRINREFLEX_JPEG_VIDEO) */ /* Alloc video buffers and enqueue for capture */ for (i = 0; i < ARRAY_SIZE(buffers); i++) { buffers[i] = video_buffer_aligned_alloc(bsize, CONFIG_VIDEO_BUFFER_POOL_ALIGN, @@ -143,6 +167,7 @@ void setup() buffers[i]->type = type; video_enqueue(video_dev, buffers[i]); } +#endif /* defined(CONFIG_GRINREFLEX_JPEG_VIDEO) */ /* Set controls */ struct video_control ctrl = {.id = VIDEO_CID_HFLIP, .val = 1}; diff --git a/u5/boards/vendor/grinreflex/grinreflex_dk1_defconfig b/u5/boards/vendor/grinreflex/grinreflex_dk1_defconfig index 608a2ca..695db2c 100644 --- a/u5/boards/vendor/grinreflex/grinreflex_dk1_defconfig +++ b/u5/boards/vendor/grinreflex/grinreflex_dk1_defconfig @@ -17,10 +17,18 @@ CONFIG_UART_CONSOLE=y # Enable GPIO CONFIG_GPIO=y - CONFIG_VIDEO=y CONFIG_VIDEO_LOG_LEVEL_DBG=y -CONFIG_VIDEO_BUFFER_POOL_SZ_MAX=76800 +CONFIG_VIDEO_LOG_LEVEL_INF=y + +CONFIG_STM32_JPEG_RGB_FORMAT=3 +CONFIG_GRINREFLEX_JPEG_VIDEO=y +CONFIG_GRINREFLEX_JPEG_VIDEO_QUALITY=50 +CONFIG_GRINREFLEX_VIDEO_WIDTH=640 +CONFIG_GRINREFLEX_VIDEO_HEIGHT=480 + +#CONFIG_VIDEO_BUFFER_POOL_SZ_MAX=76800 +CONFIG_VIDEO_BUFFER_POOL_SZ_MAX=614400 #CONFIG_LV_COLOR_16_SWAP=y \ No newline at end of file diff --git a/u5/boards/vendor/grinreflex_dk2/grinreflex_dk2_defconfig b/u5/boards/vendor/grinreflex_dk2/grinreflex_dk2_defconfig index 380feb8..18b10d4 100644 --- a/u5/boards/vendor/grinreflex_dk2/grinreflex_dk2_defconfig +++ b/u5/boards/vendor/grinreflex_dk2/grinreflex_dk2_defconfig @@ -17,11 +17,18 @@ CONFIG_UART_CONSOLE=y # Enable GPIO CONFIG_GPIO=y - CONFIG_VIDEO=y CONFIG_VIDEO_LOG_LEVEL_DBG=y +CONFIG_VIDEO_LOG_LEVEL_INF=y + +CONFIG_STM32_JPEG_RGB_FORMAT=3 +CONFIG_GRINREFLEX_JPEG_VIDEO=y +CONFIG_GRINREFLEX_JPEG_VIDEO_QUALITY=50 +CONFIG_GRINREFLEX_VIDEO_WIDTH=640 +CONFIG_GRINREFLEX_VIDEO_HEIGHT=480 + #CONFIG_VIDEO_BUFFER_POOL_SZ_MAX=76800 -CONFIG_VIDEO_BUFFER_POOL_SZ_MAX=131072 +CONFIG_VIDEO_BUFFER_POOL_SZ_MAX=153600 #CONFIG_LV_COLOR_16_SWAP=y \ No newline at end of file diff --git a/u5/lib/cv/opencv b/u5/lib/cv/opencv index cd8f1a9..7baf8f3 160000 --- a/u5/lib/cv/opencv +++ b/u5/lib/cv/opencv @@ -1 +1 @@ -Subproject commit cd8f1a9e20efc0c6d3a2a4a05dc6b542612be082 +Subproject commit 7baf8f38e889a9e87e4b5f568a596fbdbcdc2763 From 0e64547f1bfa19ddd8fb54a01602f1c1636073d0 Mon Sep 17 00:00:00 2001 From: Roman Pustobaiev Date: Mon, 13 Oct 2025 14:10:09 +0200 Subject: [PATCH 03/18] Adding gpio to indicate smile detected Refactoring, improving performance Adding FLIP_Y in case different camera is used: some OV2640 produce y-flipped image --- README.md | 4 + host/Src/main.cpp | 12 +- stm32hal.patch | 20 + u5/app/prj.conf | 11 +- u5/app/src/gfx_utils.cpp | 10 +- u5/app/src/gfx_utils.hpp | 2 +- u5/app/src/grinreflex_config.hpp | 4 +- u5/app/src/grinreflex_utils.cpp | 16 +- u5/app/src/grinreflex_v2.cpp | 20 +- u5/app/src/main.c | 2 +- .../vendor/grinreflex_dk2/grinreflex_dk2.dts | 28 +- .../grinreflex_dk2/grinreflex_dk2_defconfig | 2 +- u5/drivers/jpeg/jpeg_utils_conf.h | 2 +- zephyr.patch | 471 ------------------ 14 files changed, 103 insertions(+), 501 deletions(-) create mode 100644 stm32hal.patch delete mode 100644 zephyr.patch diff --git a/README.md b/README.md index 356e925..a4e04f5 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,10 @@ For a fresh start ### Important note: since stm32u5g9j_dk2 doesn't have camera sensor, you need to find out your own way how to transfer images to the board; I used uart for that purpose +### Apply stm32hal.patch, in case -O2 (CONFIG_SPEED_OPTIMIZATIONS=y) is used +* `modules/hal/stm32/` +* `git apply ../../../stm32hal.patch` + ## To enable OpenCV library and example code * in `u5/app/prj.conf` : CONFIG_OPENCV_LIB=y diff --git a/host/Src/main.cpp b/host/Src/main.cpp index 3db0284..ac42de1 100644 --- a/host/Src/main.cpp +++ b/host/Src/main.cpp @@ -57,14 +57,15 @@ int main(int, char**) { crop.x = (frameOriginalGray.cols - 320) / 2; crop.width = 320; crop.y = (frameOriginalGray.rows - 240) / 2; + //crop.y = 240; crop.height = 240; frameCroppedGray = frameOriginalGray(crop); cv::Rect faceROIMax{}; - faceROIMax.width = 60; - faceROIMax.height = 60; + faceROIMax.width = 64; + faceROIMax.height = 64; - cv::resize(frameCroppedGray, frameScreenGray, cv::Size(120, 120), 0.0f, 0.0f, cv::INTER_NEAREST ); + cv::resize(frameCroppedGray, frameScreenGray, cv::Size(80, 80), 0.0f, 0.0f, cv::INTER_NEAREST ); std::vector faces, smiles; @@ -78,7 +79,10 @@ int main(int, char**) { smiles ); - for (const auto &ROI : ROIs) { + for (const auto &ROI : faces) { + rectangle(frameScreenGray, ROI, cv::Scalar(0, 0, 255), 1, 1, 0); + } + for (const auto &ROI : smiles) { rectangle(frameScreenGray, ROI, cv::Scalar(0, 0, 255), 1, 1, 0); } diff --git a/stm32hal.patch b/stm32hal.patch new file mode 100644 index 0000000..4b7fe0e --- /dev/null +++ b/stm32hal.patch @@ -0,0 +1,20 @@ +diff --git a/stm32cube/stm32u5xx/drivers/src/stm32u5xx_hal_dma_ex.c b/stm32cube/stm32u5xx/drivers/src/stm32u5xx_hal_dma_ex.c +index d60eca23..708ff379 100644 +--- a/stm32cube/stm32u5xx/drivers/src/stm32u5xx_hal_dma_ex.c ++++ b/stm32cube/stm32u5xx/drivers/src/stm32u5xx_hal_dma_ex.c +@@ -516,6 +516,9 @@ + /* Includes ----------------------------------------------------------------------------------------------------------*/ + #include "stm32u5xx_hal.h" + ++#pragma GCC push_options ++#pragma GCC optimize ("O0") ++ + /** @addtogroup STM32U5xx_HAL_Driver + * @{ + */ +@@ -4717,3 +4720,5 @@ static void DMA_List_CleanQueue(DMA_QListTypeDef *const pQList) + /** + * @} + */ ++ ++#pragma GCC pop_options diff --git a/u5/app/prj.conf b/u5/app/prj.conf index 07a4b56..d0d7494 100644 --- a/u5/app/prj.conf +++ b/u5/app/prj.conf @@ -72,8 +72,7 @@ CONFIG_CPP_RTTI=y #CONFIG_DISPLAY_LOG_LEVEL_DBG=y CONFIG_DMA_LOG_LEVEL_INF=y -CONFIG_HW_STACK_PROTECTION=y - +#CONFIG_HW_STACK_PROTECTION=y # Make asserts print file:line and message CONFIG_ASSERT=y @@ -83,12 +82,16 @@ CONFIG_ASSERT_VERBOSE=y CONFIG_PRINTK=y CONFIG_LOG=y CONFIG_LOG_PRINTK=y -CONFIG_LOG_MODE_IMMEDIATE=y +#CONFIG_LOG_MODE_IMMEDIATE=y # Better symbols for gdb #CONFIG_DEBUG_INFO=y # (Optional) minimal optimizations for nicer backtraces -CONFIG_NO_OPTIMIZATIONS=y +# CONFIG_NO_OPTIMIZATIONS=y # Richer fault dump on ARM (registers, etc.) CONFIG_FAULT_DUMP=2 + +CONFIG_SPEED_OPTIMIZATIONS=y + +CONFIG_VIDEO_SHELL=y diff --git a/u5/app/src/gfx_utils.cpp b/u5/app/src/gfx_utils.cpp index e378c3e..fb897f2 100644 --- a/u5/app/src/gfx_utils.cpp +++ b/u5/app/src/gfx_utils.cpp @@ -3,7 +3,7 @@ namespace Gfx { -void fit(lv_obj_t *canvas, const GfxBuffer &src, const GfxBuffer &dst, const int32_t x, const int32_t y) { +void fit(lv_obj_t *canvas, const GfxBuffer &src, const GfxBuffer &dst, bool flip_y) { lv_image_dsc_t src_img = { .header = @@ -31,7 +31,13 @@ void fit(lv_obj_t *canvas, const GfxBuffer &src, const GfxBuffer &dst, const int img_dsc.scale_x = (LV_SCALE_NONE * (int32_t)(dst.width)) / src.width; img_dsc.scale_y = (LV_SCALE_NONE * (int32_t)(dst.height)) / src.height; - lv_area_t coords_img = {x, y, src.width - 1, src.height - 1}; + if (flip_y) { + img_dsc.rotation = 1800; + img_dsc.pivot.x = src.width / 2; + img_dsc.pivot.y = src.height / 2; + } + + lv_area_t coords_img = {0, 0, src.width - 1, src.height - 1}; lv_draw_image(&layer, &img_dsc, &coords_img); diff --git a/u5/app/src/gfx_utils.hpp b/u5/app/src/gfx_utils.hpp index fc662e9..0963f20 100644 --- a/u5/app/src/gfx_utils.hpp +++ b/u5/app/src/gfx_utils.hpp @@ -24,6 +24,6 @@ struct GfxBuffer { * @brief Fit source buffer onto destination * */ -void fit(lv_obj_t *canvas, const GfxBuffer &src, const GfxBuffer &dst, const int32_t x = 0, const int32_t y = 0); +void fit(lv_obj_t *canvas, const GfxBuffer &src, const GfxBuffer &dst, bool flip_y = false); } // namespace Gfx diff --git a/u5/app/src/grinreflex_config.hpp b/u5/app/src/grinreflex_config.hpp index 7de3e79..5f64bac 100644 --- a/u5/app/src/grinreflex_config.hpp +++ b/u5/app/src/grinreflex_config.hpp @@ -31,4 +31,6 @@ #define ROI_FRAME_HEIGHT 240 #define THUMBNAIL_FRAME_WIDTH 80 -#define THUMBNAIL_FRAME_HEIGHT 80 \ No newline at end of file +#define THUMBNAIL_FRAME_HEIGHT 80 + +//#define FULL_FRAME_FLIP_Y (1) \ No newline at end of file diff --git a/u5/app/src/grinreflex_utils.cpp b/u5/app/src/grinreflex_utils.cpp index d9db464..c38b6a0 100644 --- a/u5/app/src/grinreflex_utils.cpp +++ b/u5/app/src/grinreflex_utils.cpp @@ -28,7 +28,12 @@ void cropFullFrameToRoi(lv_obj_t *parent, uint8_t *fullFb, uint8_t *roiFb) { Gfx::GfxBuffer dst{}; uint32_t x = (FULL_FRAME_WIDTH - ROI_FRAME_WIDTH) / 2; - uint32_t y = (FULL_FRAME_HEIGHT - ROI_FRAME_HEIGHT) / 2; + //uint32_t y = (FULL_FRAME_HEIGHT - ROI_FRAME_HEIGHT) / 2; +#if defined(FULL_FRAME_FLIP_Y) + uint32_t y = 0; +#else + uint32_t y = FULL_FRAME_HEIGHT - ROI_FRAME_HEIGHT; +#endif uint32_t fullFbStride = FULL_FRAME_WIDTH * LV_COLOR_FORMAT_GET_SIZE(FULL_FRAME_COLOR_FORMAT); uint32_t fullFbOffset = (y * fullFbStride) + x; @@ -50,8 +55,13 @@ void cropFullFrameToRoi(lv_obj_t *parent, uint8_t *fullFb, uint8_t *roiFb) { dst.cf = LV_COLOR_FORMAT_L8; dst.size = ROI_FRAME_HEIGHT * roiFbStride; + +#if defined(FULL_FRAME_FLIP_Y) + Gfx::fit(parent, src, dst, true); +#else Gfx::fit(parent, src, dst); - lv_task_handler(); +#endif + //lv_task_handler(); } void fitRoiFrameToThumbnail(lv_obj_t *parent, uint8_t *roiFb, uint8_t *tFb) { @@ -77,5 +87,5 @@ void fitRoiFrameToThumbnail(lv_obj_t *parent, uint8_t *roiFb, uint8_t *tFb) { dst.size = THUMBNAIL_FRAME_HEIGHT * tnFbStride; Gfx::fit(parent, src, dst); - lv_task_handler(); + //lv_task_handler(); } diff --git a/u5/app/src/grinreflex_v2.cpp b/u5/app/src/grinreflex_v2.cpp index 5f71dfc..da69665 100644 --- a/u5/app/src/grinreflex_v2.cpp +++ b/u5/app/src/grinreflex_v2.cpp @@ -36,6 +36,7 @@ LOG_MODULE_REGISTER(grinreflex_app); static const struct device *video_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_camera)); static const struct device *display_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_display)); const struct device *jpeg_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_jpeg)); +static const struct gpio_dt_spec led = GPIO_DT_SPEC_GET(DT_ALIAS(led0), gpios); static uint8_t fullFrameBuffer[FULL_FRAME_WIDTH * FULL_FRAME_HEIGHT * LV_COLOR_FORMAT_GET_SIZE(FULL_FRAME_COLOR_FORMAT)]; static uint8_t roiFrameBuffer[ROI_FRAME_WIDTH * ROI_FRAME_HEIGHT * LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8)]; @@ -69,6 +70,14 @@ int init() return -ENODEV; } + if (!gpio_is_ready_dt(&led)) { + return -ENODEV; + } + + if (gpio_pin_configure_dt(&led, GPIO_OUTPUT_ACTIVE) < 0) { + return -ENODEV; + } + #if defined(CONFIG_OPENCV_LIB) cv::setNumThreads(1); @@ -112,7 +121,7 @@ int loop() struct jpeg_out_prop jpeg_prop; vbuf_ptr->type = VIDEO_BUF_TYPE_OUTPUT; - + int err = video_dequeue(video_dev, &vbuf_ptr, K_FOREVER); if (err) { LOG_ERR("Unable to dequeue video buf"); @@ -132,6 +141,7 @@ int loop() cropFullFrameToRoi(dummy_canvas, fullFrameBuffer, roiFrameBuffer); fitRoiFrameToThumbnail(screen_canvas, roiFrameBuffer, thumbnailFrameBuffer); + lv_task_handler(); err = video_enqueue(video_dev, vbuf_ptr); if (err) { @@ -142,7 +152,9 @@ int loop() cropFullFrameToRoi(dummy_canvas, fullFrameBuffer, roiFrameBuffer); fitRoiFrameToThumbnail(screen_canvas, roiFrameBuffer, thumbnailFrameBuffer); + lv_task_handler(); + // Return buffer back, we don't need it anymore, thank you err = video_enqueue(video_dev, vbuf_ptr); if (err) { LOG_ERR("Unable to requeue video buf"); @@ -153,8 +165,8 @@ int loop() cv::Mat matGrayFull(ROI_FRAME_HEIGHT, ROI_FRAME_WIDTH, CV_8UC1, roiFrameBuffer); cv::Rect faceROIMax{}; - faceROIMax.width = 60; - faceROIMax.height = 60; + faceROIMax.width = 64; + faceROIMax.height = 64; std::vector faces; std::vector smiles; @@ -186,6 +198,8 @@ int loop() lv_obj_set_size(ROIRectSmile, object.width, object.height); lv_obj_remove_flag(ROIRectSmile, LV_OBJ_FLAG_HIDDEN); + + gpio_pin_set_dt(&led, 0); } else { lv_obj_add_flag(ROIRectSmile, LV_OBJ_FLAG_HIDDEN); } diff --git a/u5/app/src/main.c b/u5/app/src/main.c index 5238d7a..672166a 100644 --- a/u5/app/src/main.c +++ b/u5/app/src/main.c @@ -33,6 +33,6 @@ int main(void) while (1) { - loop(1); + loop(); } } diff --git a/u5/boards/vendor/grinreflex_dk2/grinreflex_dk2.dts b/u5/boards/vendor/grinreflex_dk2/grinreflex_dk2.dts index c66b6f1..c24fab1 100644 --- a/u5/boards/vendor/grinreflex_dk2/grinreflex_dk2.dts +++ b/u5/boards/vendor/grinreflex_dk2/grinreflex_dk2.dts @@ -25,11 +25,21 @@ zephyr,camera = &zephyr_camera_dvp; }; + led_gpios { + compatible = "gpio-leds"; + + green_led_0: led_0 { + gpios = <&gpiob 11 GPIO_ACTIVE_HIGH>; + label = "Face detect OK"; + }; + }; + aliases { watchdog0 = &iwdg; die-temp0 = &die_temp; volt-sensor0 = &vref1; volt-sensor1 = &vbat4; + led0 = &green_led_0; }; mipi_dbi_st7735r_160x80 { @@ -153,7 +163,7 @@ &i2c1 { pinctrl-names = "default"; pinctrl-0 = <&i2c1_scl_pb8 &i2c1_sda_pb3>; - status = "okay"; + status = "disabled"; clock-frequency = ; }; @@ -193,12 +203,12 @@ pinctrl-names = "default"; cs-gpios = <&gpiob 11 (GPIO_ACTIVE_LOW | GPIO_PULL_UP)>; clock-frequency = ; - status = "okay"; + status = "disabled"; }; &timers1 { st,prescaler = <1>; - status = "okay"; + status = "disabled"; pwm1: pwm { status = "okay"; @@ -209,7 +219,7 @@ &timers2 { st,prescaler = <1>; - status = "okay"; + status = "disabled"; pwm2: pwm { status = "okay"; @@ -274,23 +284,23 @@ }; &iwdg { - status = "okay"; + status = "disabled"; }; &rng { - status = "okay"; + status = "disabled"; }; &die_temp { - status = "okay"; + status = "disabled"; }; &vref1 { - status = "okay"; + status = "disabled"; }; &vbat4 { - status = "okay"; + status = "disabled"; }; diff --git a/u5/boards/vendor/grinreflex_dk2/grinreflex_dk2_defconfig b/u5/boards/vendor/grinreflex_dk2/grinreflex_dk2_defconfig index 18b10d4..90b8554 100644 --- a/u5/boards/vendor/grinreflex_dk2/grinreflex_dk2_defconfig +++ b/u5/boards/vendor/grinreflex_dk2/grinreflex_dk2_defconfig @@ -24,7 +24,7 @@ CONFIG_VIDEO_LOG_LEVEL_INF=y CONFIG_STM32_JPEG_RGB_FORMAT=3 CONFIG_GRINREFLEX_JPEG_VIDEO=y -CONFIG_GRINREFLEX_JPEG_VIDEO_QUALITY=50 +CONFIG_GRINREFLEX_JPEG_VIDEO_QUALITY=10 CONFIG_GRINREFLEX_VIDEO_WIDTH=640 CONFIG_GRINREFLEX_VIDEO_HEIGHT=480 diff --git a/u5/drivers/jpeg/jpeg_utils_conf.h b/u5/drivers/jpeg/jpeg_utils_conf.h index 8d7c5af..b542a08 100644 --- a/u5/drivers/jpeg/jpeg_utils_conf.h +++ b/u5/drivers/jpeg/jpeg_utils_conf.h @@ -57,7 +57,7 @@ * Define JPEG_RGB_FORMAT */ #define JPEG_RGB_FORMAT \ - CONFIG_STM32_JPEG_RGB_FORMAT /* JPEG_ARGB8888, JPEG_RGB888, JPEG_RGB565 ********* Value \ + CONFIG_STM32_JPEG_RGB_FORMAT /* JPEG_ARGB8888, JPEG_RGB888, JPEG_RGB565, Gray ********* Value \ different from default value : 0 ********** */ /* diff --git a/zephyr.patch b/zephyr.patch deleted file mode 100644 index dc22575..0000000 --- a/zephyr.patch +++ /dev/null @@ -1,471 +0,0 @@ -diff --git a/drivers/video/ov2640.c b/drivers/video/ov2640.c -index 08527db125c..33e0530e6fe 100644 ---- a/drivers/video/ov2640.c -+++ b/drivers/video/ov2640.c -@@ -158,11 +158,113 @@ LOG_MODULE_REGISTER(video_ov2640, CONFIG_VIDEO_LOG_LEVEL); - #define UXGA_HSIZE (1600) - #define UXGA_VSIZE (1200) - -+#define VAL_SET(x, mask, rshift, lshift) \ -+ ((((x) >> rshift) & mask) << lshift) -+ -+#define CTRLI_V_DIV_SET(x) VAL_SET(x, 0x3, 0, 3) -+#define CTRLI_H_DIV_SET(x) VAL_SET(x, 0x3, 0, 0) -+#define ZMOW_OUTW_SET(x) VAL_SET(x, 0xFF, 2, 0) -+#define ZMOH_OUTH_SET(x) VAL_SET(x, 0xFF, 2, 0) -+ -+#define ZMHH_ZSPEED_SET(x) VAL_SET(x, 0x0F, 0, 4) -+#define ZMHH_OUTH_SET(x) VAL_SET(x, 0x1, (8+2), 2) -+#define ZMHH_OUTW_SET(x) VAL_SET(x, 0x3, (8+2), 0) -+ -+#define CIF_WIDTH 352 -+#define CIF_HEIGHT 288 -+#define HD_720_WIDTH 1280 -+#define HD_720_HEIGHT 720 -+#define HD_1080_WIDTH 1920 -+#define HD_1080_HEIGHT 1080 -+#define QCIF_WIDTH 176 -+#define QCIF_HEIGHT 144 -+#define QQCIF_WIDTH 88 -+#define QQCIF_HEIGHT 72 -+#define QQVGA_WIDTH 160 -+#define QQVGA_HEIGHT 120 -+#define QVGA_WIDTH 320 -+#define QVGA_HEIGHT 240 -+#define SVGA_WIDTH 800 -+#define SVGA_HEIGHT 600 -+#define SXGA_WIDTH 1280 -+#define SXGA_HEIGHT 1024 -+#define VGA_WIDTH 640 -+#define VGA_HEIGHT 480 -+#define UXGA_WIDTH 1600 -+#define UXGA_HEIGHT 1200 -+#define XGA_WIDTH 1024 -+#define XGA_HEIGHT 768 -+ - struct ov2640_reg { - uint8_t addr; - uint8_t value; - }; - -+struct ov2640_win_size { -+ char *name; -+ uint32_t width; -+ uint32_t height; -+ const struct ov2640_reg *regs; -+ uint32_t regs_size; -+}; -+ -+#define PER_SIZE_REG_SEQ(x, y, v_div, h_div, pclk_div) \ -+{ CTRLI, CTRLI_LP_DP | CTRLI_V_DIV_SET(v_div) | \ -+ CTRLI_H_DIV_SET(h_div)}, \ -+{ ZMOW, ZMOW_OUTW_SET(x) }, \ -+{ ZMOH, ZMOH_OUTH_SET(y) }, \ -+{ ZMHH, ZMHH_OUTW_SET(x) | ZMHH_OUTH_SET(y) }, \ -+{ R_DVP_SP, pclk_div }, \ -+{ RESET, 0x00} -+ -+static const struct ov2640_reg ov2640_qqvga_regs[] = { -+ PER_SIZE_REG_SEQ(QQVGA_WIDTH, QQVGA_HEIGHT, 3, 3, 8), -+}; -+static const struct ov2640_reg ov2640_qcif_regs[] = { -+ PER_SIZE_REG_SEQ(QCIF_WIDTH, QCIF_HEIGHT, 3, 3, 4), -+}; -+static const struct ov2640_reg ov2640_qvga_regs[] = { -+ PER_SIZE_REG_SEQ(QVGA_WIDTH, QVGA_HEIGHT, 2, 2, 4), -+}; -+static const struct ov2640_reg ov2640_cif_regs[] = { -+ PER_SIZE_REG_SEQ(CIF_WIDTH, CIF_HEIGHT, 2, 2, 8), -+}; -+static const struct ov2640_reg ov2640_vga_regs[] = { -+ PER_SIZE_REG_SEQ(VGA_WIDTH, VGA_HEIGHT, 0, 0, 2), -+}; -+static const struct ov2640_reg ov2640_svga_regs[] = { -+ PER_SIZE_REG_SEQ(SVGA_WIDTH, SVGA_HEIGHT, 1, 1, 2), -+}; -+static const struct ov2640_reg ov2640_xga_regs[] = { -+ PER_SIZE_REG_SEQ(XGA_WIDTH, XGA_HEIGHT, 0, 0, 2), -+ { CTRLI, 0x00}, -+}; -+static const struct ov2640_reg ov2640_sxga_regs[] = { -+ PER_SIZE_REG_SEQ(SXGA_WIDTH, SXGA_HEIGHT, 0, 0, 2), -+ { CTRLI, 0x00}, -+ { R_DVP_SP, 2 | R_DVP_SP_AUTO_MODE }, -+}; -+static const struct ov2640_reg ov2640_uxga_regs[] = { -+ PER_SIZE_REG_SEQ(UXGA_WIDTH, UXGA_HEIGHT, 0, 0, 0), -+ { CTRLI, 0x00}, -+ { R_DVP_SP, 0 | R_DVP_SP_AUTO_MODE }, -+}; -+ -+#define OV2640_SIZE(n, w, h, r) \ -+ {.name = n, .width = w , .height = h, .regs = r, .regs_size = ARRAY_SIZE(r) } -+ -+static const struct ov2640_win_size ov2640_supported_win_sizes[] = { -+ OV2640_SIZE("QQVGA", QQVGA_WIDTH, QQVGA_HEIGHT, ov2640_qqvga_regs), -+ OV2640_SIZE("QCIF", QCIF_WIDTH, QCIF_HEIGHT, ov2640_qcif_regs), -+ OV2640_SIZE("QVGA", QVGA_WIDTH, QVGA_HEIGHT, ov2640_qvga_regs), -+ OV2640_SIZE("CIF", CIF_WIDTH, CIF_HEIGHT, ov2640_cif_regs), -+ OV2640_SIZE("VGA", VGA_WIDTH, VGA_HEIGHT, ov2640_vga_regs), -+ OV2640_SIZE("SVGA", SVGA_WIDTH, SVGA_HEIGHT, ov2640_svga_regs), -+ OV2640_SIZE("XGA", XGA_WIDTH, XGA_HEIGHT, ov2640_xga_regs), -+ OV2640_SIZE("SXGA", SXGA_WIDTH, SXGA_HEIGHT, ov2640_sxga_regs), -+ OV2640_SIZE("UXGA", UXGA_WIDTH, UXGA_HEIGHT, ov2640_uxga_regs), -+}; -+ - static const struct ov2640_reg default_regs[] = { - { BANK_SEL, BANK_SEL_DSP }, - { 0x2c, 0xff }, -@@ -400,7 +502,7 @@ static const struct ov2640_reg uxga_regs[] = { - { R_DVP_SP, R_DVP_SP_AUTO_MODE | 0x04}, - - { R_BYPASS, R_BYPASS_DSP_EN }, -- { RESET, 0x00 }, -+// { RESET, 0x00 }, - {0, 0}, - }; - -@@ -472,27 +574,27 @@ struct ov2640_data { - } - - static const struct video_format_cap fmts[] = { -- OV2640_VIDEO_FORMAT_CAP(160, 120, VIDEO_PIX_FMT_RGB565), /* QQVGA */ -- OV2640_VIDEO_FORMAT_CAP(176, 144, VIDEO_PIX_FMT_RGB565), /* QCIF */ -+ OV2640_VIDEO_FORMAT_CAP(QQVGA_WIDTH, QQVGA_HEIGHT, VIDEO_PIX_FMT_RGB565), /* QQVGA */ -+ OV2640_VIDEO_FORMAT_CAP(QCIF_WIDTH, QCIF_HEIGHT, VIDEO_PIX_FMT_RGB565), /* QCIF */ - OV2640_VIDEO_FORMAT_CAP(240, 160, VIDEO_PIX_FMT_RGB565), /* HQVGA */ - OV2640_VIDEO_FORMAT_CAP(240, 240, VIDEO_PIX_FMT_RGB565), /* 240x240 */ - OV2640_VIDEO_FORMAT_CAP(320, 240, VIDEO_PIX_FMT_RGB565), /* QVGA */ -- OV2640_VIDEO_FORMAT_CAP(352, 288, VIDEO_PIX_FMT_RGB565), /* CIF */ -- OV2640_VIDEO_FORMAT_CAP(640, 480, VIDEO_PIX_FMT_RGB565), /* VGA */ -- OV2640_VIDEO_FORMAT_CAP(800, 600, VIDEO_PIX_FMT_RGB565), /* SVGA */ -- OV2640_VIDEO_FORMAT_CAP(1024, 768, VIDEO_PIX_FMT_RGB565), /* XVGA */ -- OV2640_VIDEO_FORMAT_CAP(1280, 1024, VIDEO_PIX_FMT_RGB565), /* SXGA */ -- OV2640_VIDEO_FORMAT_CAP(1600, 1200, VIDEO_PIX_FMT_RGB565), /* UXGA */ -- OV2640_VIDEO_FORMAT_CAP(160, 120, VIDEO_PIX_FMT_JPEG), /* QQVGA */ -- OV2640_VIDEO_FORMAT_CAP(176, 144, VIDEO_PIX_FMT_JPEG), /* QCIF */ -+ OV2640_VIDEO_FORMAT_CAP(CIF_WIDTH, CIF_HEIGHT, VIDEO_PIX_FMT_RGB565), /* CIF */ -+ OV2640_VIDEO_FORMAT_CAP(VGA_WIDTH, VGA_HEIGHT, VIDEO_PIX_FMT_RGB565), /* VGA */ -+ OV2640_VIDEO_FORMAT_CAP(SVGA_WIDTH, SVGA_HEIGHT, VIDEO_PIX_FMT_RGB565), /* SVGA */ -+ OV2640_VIDEO_FORMAT_CAP(XGA_WIDTH, XGA_HEIGHT, VIDEO_PIX_FMT_RGB565), /* XVGA */ -+ OV2640_VIDEO_FORMAT_CAP(SXGA_WIDTH, SXGA_HEIGHT, VIDEO_PIX_FMT_RGB565), /* SXGA */ -+ OV2640_VIDEO_FORMAT_CAP(UXGA_WIDTH, UXGA_HEIGHT, VIDEO_PIX_FMT_RGB565), /* UXGA */ -+ OV2640_VIDEO_FORMAT_CAP(QQVGA_WIDTH, QQVGA_HEIGHT, VIDEO_PIX_FMT_JPEG), /* QQVGA */ -+ OV2640_VIDEO_FORMAT_CAP(QCIF_WIDTH, QCIF_HEIGHT, VIDEO_PIX_FMT_JPEG), /* QCIF */ - OV2640_VIDEO_FORMAT_CAP(240, 160, VIDEO_PIX_FMT_JPEG), /* HQVGA */ - OV2640_VIDEO_FORMAT_CAP(320, 240, VIDEO_PIX_FMT_JPEG), /* QVGA */ -- OV2640_VIDEO_FORMAT_CAP(352, 288, VIDEO_PIX_FMT_JPEG), /* CIF */ -- OV2640_VIDEO_FORMAT_CAP(640, 480, VIDEO_PIX_FMT_JPEG), /* VGA */ -- OV2640_VIDEO_FORMAT_CAP(800, 600, VIDEO_PIX_FMT_JPEG), /* SVGA */ -- OV2640_VIDEO_FORMAT_CAP(1024, 768, VIDEO_PIX_FMT_JPEG), /* XVGA */ -- OV2640_VIDEO_FORMAT_CAP(1280, 1024, VIDEO_PIX_FMT_JPEG), /* SXGA */ -- OV2640_VIDEO_FORMAT_CAP(1600, 1200, VIDEO_PIX_FMT_JPEG), /* UXGA */ -+ OV2640_VIDEO_FORMAT_CAP(CIF_WIDTH, CIF_HEIGHT, VIDEO_PIX_FMT_JPEG), /* CIF */ -+ OV2640_VIDEO_FORMAT_CAP(VGA_WIDTH, VGA_HEIGHT, VIDEO_PIX_FMT_JPEG), /* VGA */ -+ OV2640_VIDEO_FORMAT_CAP(SVGA_WIDTH, SVGA_HEIGHT, VIDEO_PIX_FMT_JPEG), /* SVGA */ -+ OV2640_VIDEO_FORMAT_CAP(XGA_WIDTH, XGA_HEIGHT, VIDEO_PIX_FMT_JPEG), /* XVGA */ -+ OV2640_VIDEO_FORMAT_CAP(SXGA_WIDTH, SXGA_HEIGHT, VIDEO_PIX_FMT_JPEG), /* SXGA */ -+ OV2640_VIDEO_FORMAT_CAP(UXGA_WIDTH, UXGA_HEIGHT, VIDEO_PIX_FMT_JPEG), /* UXGA */ - { 0 } - }; - -@@ -775,6 +877,17 @@ static int ov2640_set_vertical_flip(const struct device *dev, int enable) - return ret; - } - -+static const struct ov2640_win_size *ov2640_select_win(uint32_t width, uint32_t height) -+{ -+ int i, default_size = ARRAY_SIZE(ov2640_supported_win_sizes) - 1; -+ for (i = 0; i < ARRAY_SIZE(ov2640_supported_win_sizes); i++) { -+ if (ov2640_supported_win_sizes[i].width >= width && -+ ov2640_supported_win_sizes[i].height >= height) -+ return &ov2640_supported_win_sizes[i]; -+ } -+ return &ov2640_supported_win_sizes[default_size]; -+} -+ - static int ov2640_set_resolution(const struct device *dev, - uint16_t img_width, uint16_t img_height) - { -@@ -784,23 +897,23 @@ static int ov2640_set_resolution(const struct device *dev, - uint16_t w = img_width; - uint16_t h = img_height; - -+ const struct ov2640_win_size *win = ov2640_select_win(w, h); -+ -+ /* Write DSP input registers */ -+ ret |= ov2640_write_all(dev, uxga_regs, ARRAY_SIZE(uxga_regs)); -+ - /* Disable DSP */ - ret |= ov2640_write_reg(&cfg->i2c, BANK_SEL, BANK_SEL_DSP); - ret |= ov2640_write_reg(&cfg->i2c, R_BYPASS, R_BYPASS_DSP_BYPAS); - -- /* Write output width */ -- ret |= ov2640_write_reg(&cfg->i2c, ZMOW, (w >> 2) & 0xFF); /* OUTW[7:0] (real/4) */ -- ret |= ov2640_write_reg(&cfg->i2c, ZMOH, (h >> 2) & 0xFF); /* OUTH[7:0] (real/4) */ -- ret |= ov2640_write_reg(&cfg->i2c, ZMHH, ((h >> 8) & 0x04) | -- ((w>>10) & 0x03)); /* OUTH[8]/OUTW[9:8] */ -+ LOG_INF("Selected resolution %s", win->name); -+ -+ ret |= ov2640_write_all(dev, win->regs, win->regs_size); - - /* Set CLKRC */ - ret |= ov2640_write_reg(&cfg->i2c, BANK_SEL, BANK_SEL_SENSOR); - ret |= ov2640_write_reg(&cfg->i2c, CLKRC, cfg->clock_rate_control); - -- /* Write DSP input registers */ -- ov2640_write_all(dev, uxga_regs, ARRAY_SIZE(uxga_regs)); -- - /* Enable DSP */ - ret |= ov2640_write_reg(&cfg->i2c, BANK_SEL, BANK_SEL_DSP); - ret |= ov2640_write_reg(&cfg->i2c, R_BYPASS, R_BYPASS_DSP_EN); -diff --git a/drivers/video/video_stm32_dcmi.c b/drivers/video/video_stm32_dcmi.c -index 363de09717b..11e75e86c7a 100644 ---- a/drivers/video/video_stm32_dcmi.c -+++ b/drivers/video/video_stm32_dcmi.c -@@ -81,7 +81,56 @@ void HAL_DCMI_FrameEventCallback(DCMI_HandleTypeDef *hdcmi) - k_fifo_put(&dev_data->fifo_out, vbuf); - - resume: -+#if defined(CONFIG_SOC_SERIES_STM32U5X) -+ int err = HAL_DCMI_Start_DMA(&dev_data->hdcmi, DCMI_MODE_SNAPSHOT, -+ (uint32_t)dev_data->vbuf->buffer, dev_data->vbuf->bytesused / 4); -+ if (err != HAL_OK) { -+ LOG_ERR("Failed to start DCMI DMA"); -+ } -+#else -+ HAL_DCMI_Resume(hdcmi); -+#endif -+} -+ -+void HAL_DCMI_VsyncEventCallback(DCMI_HandleTypeDef *hdcmi) -+{ -+ struct video_stm32_dcmi_data *dev_data = -+ CONTAINER_OF(hdcmi, struct video_stm32_dcmi_data, hdcmi); -+ struct video_buffer *vbuf; -+ -+ if (dev_data->fmt.pixelformat != VIDEO_PIX_FMT_JPEG) { -+ return; -+ } -+ -+ HAL_DCMI_Suspend(hdcmi); -+ -+ vbuf = k_fifo_get(&dev_data->fifo_in, K_NO_WAIT); -+ -+ if (vbuf == NULL) { -+ LOG_DBG("Failed to get buffer from fifo"); -+ goto resume; -+ } -+ -+ vbuf->bytesused = vbuf->size - __HAL_DMA_GET_COUNTER(dev_data->hdcmi.DMA_Handle); -+ -+ vbuf->timestamp = k_uptime_get_32(); -+ memcpy(vbuf->buffer, dev_data->vbuf->buffer, vbuf->bytesused); -+ -+ k_fifo_put(&dev_data->fifo_out, vbuf); -+ -+resume: -+#if defined(CONFIG_SOC_SERIES_STM32U5X) -+ // Stop DMA as in JPEG mode it may be still waiting for more data to come, so we can't restart it directly -+ HAL_DCMI_Stop(&dev_data->hdcmi); -+ int err = HAL_DCMI_Start_DMA(&dev_data->hdcmi, DCMI_MODE_SNAPSHOT, -+ (uint32_t)dev_data->vbuf->buffer, dev_data->vbuf->bytesused / 4); -+ if (err != HAL_OK) { -+ LOG_ERR("Failed to start DCMI DMA"); -+ } -+#else -+ - HAL_DCMI_Resume(hdcmi); -+#endif - } - - static void stm32_dcmi_isr(const struct device *dev) -@@ -109,6 +158,47 @@ void HAL_DMA_ErrorCallback(DMA_HandleTypeDef *hdma) - LOG_WRN("%s", __func__); - } - -+#if defined(CONFIG_SOC_SERIES_STM32U5X) -+static int stm32_dma_list_init(DMA_HandleTypeDef *hdma) -+{ -+ static DMA_NodeTypeDef Node; -+ static DMA_QListTypeDef Queue; -+ -+ HAL_StatusTypeDef ret = HAL_OK; -+ /* DMA node configuration declaration */ -+ DMA_NodeConfTypeDef pNodeConfig; -+ -+ /* Set node configuration ################################################*/ -+ pNodeConfig.NodeType = DMA_GPDMA_LINEAR_NODE; -+ pNodeConfig.Init.Request = GPDMA1_REQUEST_DCMI_PSSI; -+ pNodeConfig.Init.BlkHWRequest = DMA_BREQ_SINGLE_BURST; -+ pNodeConfig.Init.Direction = DMA_PERIPH_TO_MEMORY; -+ pNodeConfig.Init.SrcInc = DMA_SINC_FIXED; -+ pNodeConfig.Init.DestInc = DMA_DINC_INCREMENTED; -+ pNodeConfig.Init.SrcDataWidth = DMA_SRC_DATAWIDTH_WORD; -+ pNodeConfig.Init.DestDataWidth = DMA_DEST_DATAWIDTH_WORD; -+ pNodeConfig.Init.SrcBurstLength = 1; -+ pNodeConfig.Init.DestBurstLength = 1; -+ pNodeConfig.Init.TransferAllocatedPort = DMA_SRC_ALLOCATED_PORT0|DMA_DEST_ALLOCATED_PORT0; -+ pNodeConfig.Init.TransferEventMode = DMA_TCEM_BLOCK_TRANSFER; -+ pNodeConfig.TriggerConfig.TriggerPolarity = DMA_TRIG_POLARITY_MASKED; -+ pNodeConfig.DataHandlingConfig.DataExchange = DMA_EXCHANGE_NONE; -+ pNodeConfig.DataHandlingConfig.DataAlignment = DMA_DATA_RIGHTALIGN_ZEROPADDED; -+ pNodeConfig.SrcAddress = 0; -+ pNodeConfig.DstAddress = 0; -+ pNodeConfig.DataSize = 0; -+ -+ /* Build Node1 Node */ -+ ret |= HAL_DMAEx_List_BuildNode(&pNodeConfig, &Node); -+ /* Insert Node1 to Queue */ -+ ret |= HAL_DMAEx_List_InsertNode_Tail(&Queue, &Node); -+ ret |= HAL_DMAEx_List_SetCircularMode(&Queue); -+ ret |= HAL_DMAEx_List_LinkQ(hdma, &Queue); -+ -+ return ret; -+} -+#endif /* !defined(CONFIG_SOC_SERIES_STM32U5X) */ -+ - static int stm32_dma_init(const struct device *dev) - { - struct video_stm32_dcmi_data *data = dev->data; -@@ -147,6 +237,30 @@ static int stm32_dma_init(const struct device *dev) - - /*** Configure the DMA ***/ - /* Set the parameters to be configured */ -+ -+#if defined(CONFIG_SOC_SERIES_STM32U5X) -+ hdma.Init.Request = GPDMA1_REQUEST_DCMI_PSSI; -+ hdma.Init.BlkHWRequest = DMA_BREQ_SINGLE_BURST; -+ hdma.Init.Direction = DMA_PERIPH_TO_MEMORY; -+ hdma.Init.SrcInc = DMA_SINC_FIXED; -+ hdma.Init.DestInc = DMA_DINC_INCREMENTED; -+ hdma.Init.SrcDataWidth = DMA_SRC_DATAWIDTH_WORD; -+ hdma.Init.DestDataWidth = DMA_DEST_DATAWIDTH_WORD; -+ -+ hdma.Init.Priority = DMA_LOW_PRIORITY_HIGH_WEIGHT; -+ hdma.Init.SrcBurstLength = 1; -+ hdma.Init.DestBurstLength = 1; -+ hdma.Init.TransferAllocatedPort = -+ DMA_SRC_ALLOCATED_PORT0 | DMA_DEST_ALLOCATED_PORT1; -+ hdma.Init.TransferEventMode = DMA_TCEM_BLOCK_TRANSFER; -+ hdma.Init.Mode = DMA_NORMAL; -+ -+ hdma.InitLinkedList.Priority = DMA_LOW_PRIORITY_LOW_WEIGHT; -+ hdma.InitLinkedList.LinkStepMode = DMA_LSM_FULL_EXECUTION; -+ hdma.InitLinkedList.LinkAllocatedPort = DMA_LINK_ALLOCATED_PORT0 | DMA_LINK_ALLOCATED_PORT1; -+ hdma.InitLinkedList.TransferEventMode = DMA_TCEM_LAST_LL_ITEM_TRANSFER; -+ hdma.InitLinkedList.LinkedListMode = DMA_LINKEDLIST_CIRCULAR; -+#else /* ! defined(CONFIG_SOC_SERIES_STM32U5X) */ - hdma.Init.Request = DMA_REQUEST_DCMI; - hdma.Init.Direction = DMA_PERIPH_TO_MEMORY; - hdma.Init.PeriphInc = DMA_PINC_DISABLE; -@@ -155,6 +269,8 @@ static int stm32_dma_init(const struct device *dev) - hdma.Init.MemDataAlignment = DMA_MDATAALIGN_WORD; - hdma.Init.Mode = DMA_CIRCULAR; - hdma.Init.Priority = DMA_PRIORITY_HIGH; -+#endif /* defined(CONFIG_SOC_SERIES_STM32U5X) */ -+ - #if defined(CONFIG_SOC_SERIES_STM32F7X) || defined(CONFIG_SOC_SERIES_STM32H7X) - hdma.Init.FIFOMode = DMA_FIFOMODE_DISABLE; - #endif -@@ -164,6 +280,9 @@ static int stm32_dma_init(const struct device *dev) - config->dma.channel); - #elif defined(CONFIG_SOC_SERIES_STM32L4X) - hdma.Instance = __LL_DMA_GET_CHANNEL_INSTANCE(config->dma.reg, config->dma.channel); -+#elif defined(CONFIG_SOC_SERIES_STM32U5X) -+ hdma.Instance = LL_DMA_GET_CHANNEL_INSTANCE( -+ config->dma.reg, config->dma.channel); - #endif - - /* Initialize DMA HAL */ -@@ -174,6 +293,20 @@ static int stm32_dma_init(const struct device *dev) - return -EIO; - } - -+#if defined(CONFIG_SOC_SERIES_STM32U5X) -+ if (HAL_DMAEx_List_Init(&hdma) != HAL_OK) { -+ LOG_ERR("HAL_DMAEx_List_Init Failed"); -+ return -EINVAL; -+ } -+ if (stm32_dma_list_init(&hdma) != HAL_OK) { -+ LOG_ERR("stm32_dma_list_init Failed"); -+ return -EINVAL; -+ } -+ if (HAL_DMA_ConfigChannelAttributes(&hdma, DMA_CHANNEL_NPRIV) != HAL_OK) { -+ LOG_ERR("HAL_DMA_ConfigChannelAttributes Failed"); -+ return -EINVAL; -+ } -+#endif - return 0; - } - -@@ -270,7 +403,13 @@ static int video_stm32_dcmi_set_stream(const struct device *dev, bool enable, - data->hdcmi.Instance->CR &= ~(DCMI_CR_FCRC_0 | DCMI_CR_FCRC_1); - data->hdcmi.Instance->CR |= STM32_DCMI_GET_CAPTURE_RATE(data->capture_rate); - -- err = HAL_DCMI_Start_DMA(&data->hdcmi, DCMI_MODE_CONTINUOUS, -+ if (data->fmt.pixelformat == VIDEO_PIX_FMT_JPEG) { -+ data->hdcmi.Instance->CR |= DCMI_CR_JPEG; -+ } -+ -+ uint32_t dcmi_mode = DCMI_MODE_CONTINUOUS; -+ -+ err = HAL_DCMI_Start_DMA(&data->hdcmi, dcmi_mode, - (uint32_t)data->vbuf->buffer, data->vbuf->bytesused / 4); - if (err != HAL_OK) { - LOG_ERR("Failed to start DCMI DMA"); -@@ -283,9 +422,13 @@ static int video_stm32_dcmi_set_stream(const struct device *dev, bool enable, - static int video_stm32_dcmi_enqueue(const struct device *dev, struct video_buffer *vbuf) - { - struct video_stm32_dcmi_data *data = dev->data; -- const uint32_t buffer_size = data->fmt.pitch * data->fmt.height; -+ uint32_t buffer_size = data->fmt.pitch * data->fmt.height; -+ -+ if (data->fmt.pixelformat == VIDEO_PIX_FMT_JPEG) { -+ buffer_size = vbuf->size; -+ } - -- if (buffer_size > vbuf->size) { -+ if (0 == buffer_size || buffer_size > vbuf->size) { - return -EINVAL; - } - -diff --git a/dts/arm/st/u5/stm32u5.dtsi b/dts/arm/st/u5/stm32u5.dtsi -index 1ea790dd980..b8499dbe37b 100644 ---- a/dts/arm/st/u5/stm32u5.dtsi -+++ b/dts/arm/st/u5/stm32u5.dtsi -@@ -887,6 +887,21 @@ - }; - }; - -+ #define GPDMA1_REQUEST_DCMI_PSSI 86U -+ #define DCMI_PSSI_IRQn 119 -+ -+ dcmi: dcmi@4202C000 { -+ compatible = "st,stm32-dcmi"; -+ reg = <0x4202C000 0x400>; -+ interrupts = ; -+ interrupt-names = "dcmi"; -+ clocks = <&rcc STM32_CLOCK(AHB2, 12U)>; -+ -+ dmas = <&gpdma1 5 GPDMA1_REQUEST_DCMI_PSSI (STM32_DMA_PERIPH_TO_MEMORY | STM32_DMA_PERIPH_NO_INC | -+ STM32_DMA_MEM_INC | STM32_DMA_PERIPH_32BITS | STM32_DMA_MEM_32BITS | -+ STM32_DMA_PRIORITY_HIGH) STM32_DMA_FIFO_1_4>; -+ status = "okay"; -+ }; - }; - - swj_port: swj_port { -diff --git a/samples/boards/st/uart/circular_dma/boards/nucleo_wba55cg.overlay b/samples/boards/st/uart/circular_dma/boards/nucleo_wba55cg.overlay -index 751836f8554..52382316dd1 100644 ---- a/samples/boards/st/uart/circular_dma/boards/nucleo_wba55cg.overlay -+++ b/samples/boards/st/uart/circular_dma/boards/nucleo_wba55cg.overlay -@@ -6,7 +6,7 @@ - - &usart1 { - dmas = <&gpdma1 0 12 (STM32_DMA_PERIPH_TX) -- &gpdma1 1 11 (STM32_DMA_MODE_CYCLIC | STM32_DMA_PERIPH_RX | STM32_DMA_MEM_8BITS)>; -+ &gpdma1 1 11 (STM32_DMA_MODE_CYCLIC | STM32_DMA_PERIPH_RX | STM32_DMA_MEM_8BITS)>; - dma-names = "tx", "rx"; - fifo-enable; - }; From dd2ce83fe599ecc69555e3a68a6f4174bafc4dad Mon Sep 17 00:00:00 2001 From: Roman Pustobaiev Date: Thu, 16 Oct 2025 11:16:23 +0200 Subject: [PATCH 04/18] Adding UVC sample to test opencv using host application Adding separate application gf_dkx to run on hardware Refactoring, cleanup --- host/CMakeLists.txt | 8 +- host/Src/main.cpp | 69 +++++-- u5/CMakeLists.txt | 2 + u5/app/CMakeLists.txt | 2 + u5/app/Kconfig | 29 --- u5/app/Kconfig.sample_msc | 38 ---- u5/app/Kconfig.sample_usbd | 54 ------ u5/app/boards/grinreflex_dk1.overlay | 19 +- u5/app/boards/grinreflex_dk2.overlay | 15 -- u5/app/prj.conf | 3 - u5/app/src/grinreflex_utils.cpp | 91 --------- u5/app/src/grinreflex_utils.hpp | 10 - .../vendor/grinreflex/Kconfig.grinreflex_dk1 | 37 +++- .../vendor/grinreflex/grinreflex_dk1.dts | 19 +- .../grinreflex/grinreflex_dk1_defconfig | 10 +- .../grinreflex_dk2/Kconfig.grinreflex_dk2 | 37 +++- .../vendor/grinreflex_dk2/grinreflex_dk2.dts | 15 ++ .../grinreflex_dk2/grinreflex_dk2_defconfig | 3 + u5/common/CMakeLists.txt | 25 +++ .../include/gf/lvgl_utils.hpp} | 10 +- u5/{app/src => common/include/gf}/uart.hpp | 0 u5/common/include/gf/utils.hpp | 12 ++ u5/{app/src => common/include/gf}/video.hpp | 0 u5/common/nema/CMakeLists.txt | 28 +++ u5/common/opencv/CMakeLists.txt | 28 +++ .../opencv/include/gf}/opencv_utils.hpp | 7 +- u5/{app => common/opencv}/src/cascades.h.in | 0 .../opencv}/src/opencv_utils.cpp | 53 ++++-- .../src/lvgl_utils.cpp} | 6 +- u5/{app => common}/src/uart.cpp | 2 +- u5/common/src/utils.cpp | 82 +++++++++ u5/{app => common}/src/video.cpp | 2 +- u5/drivers/CMakeLists.txt | 6 - u5/drivers/Kconfig | 1 - u5/drivers/jpeg/jpeg.c | 4 +- u5/drivers/sensor/CMakeLists.txt | 4 - u5/drivers/sensor/Kconfig | 6 - .../sensor/example_sensor/CMakeLists.txt | 5 - u5/drivers/sensor/example_sensor/Kconfig | 10 - .../sensor/example_sensor/example_sensor.c | 86 --------- u5/gf_dkx/CMakeLists.txt | 35 ++++ u5/gf_dkx/Kconfig | 58 ++++++ u5/gf_dkx/boards/grinreflex_dk1.conf | 13 ++ u5/gf_dkx/boards/grinreflex_dk1.overlay | 50 +++++ u5/gf_dkx/boards/grinreflex_dk2.conf | 4 + u5/gf_dkx/boards/grinreflex_dk2.overlay | 22 +++ u5/gf_dkx/lvgl.conf | 24 +++ u5/gf_dkx/prj.conf | 100 ++++++++++ .../src/config.hpp} | 2 +- u5/{app => gf_dkx}/src/grinreflex.h | 0 u5/gf_dkx/src/lvgl_video_app.cpp | 111 +++++++++++ u5/gf_dkx/src/main.c | 38 ++++ .../src/opencv_fd_app.cpp} | 84 ++++----- u5/uvc_gf_dk1/CMakeLists.txt | 22 +++ u5/uvc_gf_dk1/Kconfig | 8 + u5/uvc_gf_dk1/boards/grinreflex_dk1.conf | 0 u5/uvc_gf_dk1/boards/grinreflex_dk1.overlay | 22 +++ u5/uvc_gf_dk1/lvgl.conf | 24 +++ u5/uvc_gf_dk1/prj.conf | 24 +++ u5/uvc_gf_dk1/src/main.c | 172 ++++++++++++++++++ 60 files changed, 1179 insertions(+), 472 deletions(-) delete mode 100644 u5/app/Kconfig.sample_msc delete mode 100644 u5/app/Kconfig.sample_usbd delete mode 100644 u5/app/src/grinreflex_utils.cpp delete mode 100644 u5/app/src/grinreflex_utils.hpp create mode 100644 u5/common/CMakeLists.txt rename u5/{app/src/gfx_utils.hpp => common/include/gf/lvgl_utils.hpp} (58%) rename u5/{app/src => common/include/gf}/uart.hpp (100%) create mode 100644 u5/common/include/gf/utils.hpp rename u5/{app/src => common/include/gf}/video.hpp (100%) create mode 100644 u5/common/nema/CMakeLists.txt create mode 100644 u5/common/opencv/CMakeLists.txt rename u5/{app/src => common/opencv/include/gf}/opencv_utils.hpp (91%) rename u5/{app => common/opencv}/src/cascades.h.in (100%) rename u5/{app => common/opencv}/src/opencv_utils.cpp (76%) rename u5/{app/src/gfx_utils.cpp => common/src/lvgl_utils.cpp} (89%) rename u5/{app => common}/src/uart.cpp (99%) create mode 100644 u5/common/src/utils.cpp rename u5/{app => common}/src/video.cpp (99%) delete mode 100644 u5/drivers/sensor/CMakeLists.txt delete mode 100644 u5/drivers/sensor/Kconfig delete mode 100644 u5/drivers/sensor/example_sensor/CMakeLists.txt delete mode 100644 u5/drivers/sensor/example_sensor/Kconfig delete mode 100644 u5/drivers/sensor/example_sensor/example_sensor.c create mode 100644 u5/gf_dkx/CMakeLists.txt create mode 100644 u5/gf_dkx/Kconfig create mode 100644 u5/gf_dkx/boards/grinreflex_dk1.conf create mode 100644 u5/gf_dkx/boards/grinreflex_dk1.overlay create mode 100644 u5/gf_dkx/boards/grinreflex_dk2.conf create mode 100644 u5/gf_dkx/boards/grinreflex_dk2.overlay create mode 100644 u5/gf_dkx/lvgl.conf create mode 100644 u5/gf_dkx/prj.conf rename u5/{app/src/grinreflex_config.hpp => gf_dkx/src/config.hpp} (96%) rename u5/{app => gf_dkx}/src/grinreflex.h (100%) create mode 100644 u5/gf_dkx/src/lvgl_video_app.cpp create mode 100644 u5/gf_dkx/src/main.c rename u5/{app/src/grinreflex_v2.cpp => gf_dkx/src/opencv_fd_app.cpp} (74%) create mode 100644 u5/uvc_gf_dk1/CMakeLists.txt create mode 100644 u5/uvc_gf_dk1/Kconfig create mode 100644 u5/uvc_gf_dk1/boards/grinreflex_dk1.conf create mode 100644 u5/uvc_gf_dk1/boards/grinreflex_dk1.overlay create mode 100644 u5/uvc_gf_dk1/lvgl.conf create mode 100644 u5/uvc_gf_dk1/prj.conf create mode 100644 u5/uvc_gf_dk1/src/main.c diff --git a/host/CMakeLists.txt b/host/CMakeLists.txt index 9542c02..e10eaab 100644 --- a/host/CMakeLists.txt +++ b/host/CMakeLists.txt @@ -16,12 +16,12 @@ configure_file( @ONLY ) -set(OPENCV_UTILS_DIR_PATH "../u5/app/src") +set(OPENCV_UTILS_DIR_PATH "../u5/common/opencv") set(HOST_APP_SRCS Src/main.cpp - ${OPENCV_UTILS_DIR_PATH}/opencv_utils.cpp - ${OPENCV_UTILS_DIR_PATH}/opencv_utils.hpp + ${OPENCV_UTILS_DIR_PATH}/src/opencv_utils.cpp + ${OPENCV_UTILS_DIR_PATH}/include/gf/opencv_utils.hpp ) # Add executable @@ -31,6 +31,6 @@ add_executable(host_app ${HOST_APP_SRCS}) target_include_directories(host_app PRIVATE "${CMAKE_BINARY_DIR}/Include" PRIVATE ${OpenCV_INCLUDE_DIRS} - PRIVATE ${OPENCV_UTILS_DIR_PATH}) + PRIVATE ${OPENCV_UTILS_DIR_PATH}/include) target_link_libraries(host_app PRIVATE ${OpenCV_LIBS}) diff --git a/host/Src/main.cpp b/host/Src/main.cpp index ac42de1..ed8b64e 100644 --- a/host/Src/main.cpp +++ b/host/Src/main.cpp @@ -3,7 +3,7 @@ #include "HaarPath.hpp" -#include "opencv_utils.hpp" +#include "gf/opencv_utils.hpp" cv::String haarPath = HAAR_PATH; cv::String face_cascade_name = haarPath + "lbpcascades/lbpcascade_frontalface_improved.xml"; @@ -11,6 +11,19 @@ cv::String smile_cascade_name = haarPath + "haarcascades/haarcascade_smile.xml"; cv::CascadeClassifier face_cascade; cv::CascadeClassifier smile_cascade; cv::String window_name = "Capture - Face detection"; +cv::String window_name_2 = "Capture - Smile detection"; + + +bool checkFrameSizeSupported(cv::Mat &gray) { + if (gray.cols == 640 && gray.rows == 480) { + return true; + } + + if (gray.cols == 320 && gray.rows == 240) { + return true; + } + return false; +} int main(int, char**) { // open the first webcam plugged in the computer @@ -39,6 +52,8 @@ int main(int, char**) { // display the frame until you press a key std::cout << "Start capturing" << std::endl; + float gamma = 0.1; + std::map gammaScore; while (camera.read(frameOriginal)) { if (frameOriginal.empty()) @@ -51,15 +66,23 @@ int main(int, char**) { cv::cvtColor(frameOriginal, frameOriginalGray, cv::COLOR_BGR2GRAY); //cv::equalizeHist(frameOriginalGray, frameOriginalGray); + if (false == checkFrameSizeSupported(frameOriginalGray)) { + std::cerr << "Not supported frame size" << std::endl; + exit(-1); + } + cv::Mat frameCroppedGray; - cv::Rect crop; - crop.x = (frameOriginalGray.cols - 320) / 2; - crop.width = 320; - crop.y = (frameOriginalGray.rows - 240) / 2; - //crop.y = 240; - crop.height = 240; - frameCroppedGray = frameOriginalGray(crop); + if (frameOriginalGray.cols == 640 && frameOriginalGray.rows == 480) { + cv::Rect crop; + crop.width = 320; + crop.height = 240; + crop.x = (frameOriginalGray.cols - crop.width) / 2; + crop.y = (frameOriginalGray.rows - crop.height) / 2; + frameCroppedGray = frameOriginalGray(crop); + } else { + frameCroppedGray = frameOriginalGray; + } cv::Rect faceROIMax{}; faceROIMax.width = 64; @@ -69,16 +92,32 @@ int main(int, char**) { std::vector faces, smiles; - std::vector ROIs = detectFaceAndSmile( + cv::Mat smilePostProc = detectFaceAndSmile( face_cascade, smile_cascade, frameScreenGray, frameCroppedGray, faceROIMax, faces, - smiles + smiles, + gamma ); + if (!faces.empty() && !smiles.empty()) { + if (gammaScore.count(gamma)) { + gammaScore[gamma] = gammaScore[gamma] + 1; + } else { + gammaScore[gamma] = 1; + } + } + + if (!faces.empty() && smiles.empty()) { + gamma += 0.1; + if (gamma >= 1.0) { + gamma = 0.1; + } + } + for (const auto &ROI : faces) { rectangle(frameScreenGray, ROI, cv::Scalar(0, 0, 255), 1, 1, 0); } @@ -86,11 +125,19 @@ int main(int, char**) { rectangle(frameScreenGray, ROI, cv::Scalar(0, 0, 255), 1, 1, 0); } + if (smilePostProc.cols && smilePostProc.rows) { + cv::imshow(window_name_2, smilePostProc); + } cv::imshow(window_name, frameScreenGray); + int c = cv::waitKey(100); - if ((char)c == 27) { return 0; } + if ((char)c == 27) { break; } } + for (auto & [gamma, score] : gammaScore) { + std::cout << "gamma = " << gamma << ", score = " << score << std::endl; + } + return 0; } diff --git a/u5/CMakeLists.txt b/u5/CMakeLists.txt index a073494..0d43622 100644 --- a/u5/CMakeLists.txt +++ b/u5/CMakeLists.txt @@ -13,3 +13,5 @@ zephyr_include_directories(include) add_subdirectory(drivers) add_subdirectory(lib) +add_subdirectory(common) + diff --git a/u5/app/CMakeLists.txt b/u5/app/CMakeLists.txt index d3f5f6e..d1495a4 100644 --- a/u5/app/CMakeLists.txt +++ b/u5/app/CMakeLists.txt @@ -6,6 +6,8 @@ cmake_minimum_required(VERSION 3.13.1) +message(FATAL_ERROR This application is deprecated, please use uvc_gf_dk1 / gf_dkx / ...) + set(EXTRA_CONF_FILE ${CMAKE_CURRENT_SOURCE_DIR}/lvgl.conf; ) diff --git a/u5/app/Kconfig b/u5/app/Kconfig index 1ba699d..c004ca1 100644 --- a/u5/app/Kconfig +++ b/u5/app/Kconfig @@ -17,32 +17,3 @@ source "subsys/logging/Kconfig.template.log_config" source "Kconfig.zephyr" rsource "Kconfig.sample_usbd" rsource "Kconfig.sample_msc" - -menu "GrinReflex dk1 settings" - -config GRINREFLEX_JPEG_VIDEO - bool "Enable jpeg video input" - default y - depends on BOARD_GRINREFLEX_DK1 || BOARD_GRINREFLEX_DK2 - -config GRINREFLEX_JPEG_VIDEO_QUALITY - int "Set jpeg quality, lower value - higher quality" - default 63 - depends on BOARD_GRINREFLEX_DK1 || BOARD_GRINREFLEX_DK2 - -config GRINREFLEX_VIDEO_WIDTH - int "Width of a video frame" - default 160 - -config GRINREFLEX_VIDEO_HEIGHT - int "Height of a video frame" - default 120 - -if GRINREFLEX_JPEG_VIDEO -config LV_COLOR_16_SWAP - bool - default y -endif - - -endmenu \ No newline at end of file diff --git a/u5/app/Kconfig.sample_msc b/u5/app/Kconfig.sample_msc deleted file mode 100644 index 2da93ca..0000000 --- a/u5/app/Kconfig.sample_msc +++ /dev/null @@ -1,38 +0,0 @@ - - -menu "MSC sample options" - -choice - prompt "Storage and file system type used by the application" - default APP_MSC_STORAGE_FLASH_FATFS - help - Specify the type of storage and file system. - -config APP_MSC_STORAGE_FLASH_FATFS - bool "Use FLASH disk and FAT file system" - imply DISK_DRIVERS - imply DISK_ACCESS - imply FILE_SYSTEM - imply FAT_FILESYSTEM_ELM - imply FILE_SYSTEM_MKFS - imply FS_FATFS_FSTAB_AUTOMOUNT - -endchoice - -config MASS_STORAGE_DISK_NAME - default "NAND" if DISK_DRIVER_FLASH - -if DISK_DRIVER_FLASH - -config FLASH_MAP - default y - -config FLASH_PAGE_LAYOUT - default y - -config FLASH_LOG_LEVEL - default 3 - -endif # DISK_DRIVER_FLASH - -endmenu \ No newline at end of file diff --git a/u5/app/Kconfig.sample_usbd b/u5/app/Kconfig.sample_usbd deleted file mode 100644 index b1bdcad..0000000 --- a/u5/app/Kconfig.sample_usbd +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright (c) 2023 Nordic Semiconductor ASA -# SPDX-License-Identifier: Apache-2.0 - -# This file contains Kconfig options and defaults for configuring USB devices -# using the new experimental USB device support. The scope of these options is -# limited to USB samples in project tree, you cannot use them in your own -# application. - -menu "USB sample options" - depends on USB_DEVICE_STACK_NEXT - -config SAMPLE_USBD_MANUFACTURER - string "USB device sample manufacturer string" - default "Zephyr Project" - help - USB device sample manufacturer string. - -config SAMPLE_USBD_PRODUCT - string "USB device sample product string" - default "USBD sample" - help - USB device sample product stringa. - -config SAMPLE_USBD_PID - hex "USB device sample Product ID" - default 0x0001 - help - USB device sample Product ID. - -config SAMPLE_USBD_SELF_POWERED - bool "USB device sample Self-powered attribute" - default y - help - Set the Self-powered attribute in the sample configuration. - -config SAMPLE_USBD_REMOTE_WAKEUP - bool "USB device sample Remote Wakeup attribute" - help - Set the Remote Wakeup attribute in the sample configuration. - -config SAMPLE_USBD_MAX_POWER - int "USB device sample bMaxPower value" - default 125 - range 0 250 - help - bMaxPower value in the sample configuration in 2 mA units. - -config SAMPLE_USBD_20_EXTENSION_DESC - bool "Use default USB 2.0 Extension Descriptor" - depends on USBD_BOS_SUPPORT - help - Set bcdUSB value to 0201 and use default USB 2.0 Extension Descriptor. - -endmenu diff --git a/u5/app/boards/grinreflex_dk1.overlay b/u5/app/boards/grinreflex_dk1.overlay index 9c9dcc6..20a6118 100644 --- a/u5/app/boards/grinreflex_dk1.overlay +++ b/u5/app/boards/grinreflex_dk1.overlay @@ -9,10 +9,8 @@ / { chosen: chosen { - zephyr,nema = &nema0; zephyr,spi1 = &spi1; zephyr,usart2 = &usart2; - zephyr,jpeg = &jpeg0; }; fstab { @@ -33,19 +31,6 @@ disk-name = "NAND"; cache-size = <4096>; }; - - nema0: nema-gfx { - compatible = "st,nema-gfx"; - status = "okay"; - }; - - jpeg0: jpeg_hw { - compatible = "st,jpeg-hw"; - dmas = <&gpdma1 2 125 (STM32_DMA_PERIPH_RX | STM32_DMA_PERIPH_32BITS | STM32_DMA_MEM_32BITS) - &gpdma1 3 124 (STM32_DMA_PERIPH_TX | STM32_DMA_PERIPH_32BITS | STM32_DMA_MEM_32BITS)>; - dma-names = "tx", "rx"; - status = "okay"; - }; }; &usart2 { @@ -58,4 +43,8 @@ &gpdma1 { status = "okay"; +}; + +&dcmi { + jpeg-dev = <&jpeg0>; }; \ No newline at end of file diff --git a/u5/app/boards/grinreflex_dk2.overlay b/u5/app/boards/grinreflex_dk2.overlay index 75221be..62a9e81 100644 --- a/u5/app/boards/grinreflex_dk2.overlay +++ b/u5/app/boards/grinreflex_dk2.overlay @@ -9,22 +9,7 @@ / { chosen: chosen { - zephyr,nema = &nema0; zephyr,spi1 = &spi1; - zephyr,jpeg = &jpeg0; - }; - - nema0: nema-gfx { - compatible = "st,nema-gfx"; - status = "okay"; - }; - - jpeg0: jpeg_hw { - compatible = "st,jpeg-hw"; - dmas = <&gpdma1 2 125 (STM32_DMA_PERIPH_RX | STM32_DMA_PERIPH_32BITS | STM32_DMA_MEM_32BITS) - &gpdma1 3 124 (STM32_DMA_PERIPH_TX | STM32_DMA_PERIPH_32BITS | STM32_DMA_MEM_32BITS)>; - dma-names = "tx", "rx"; - status = "okay"; }; }; diff --git a/u5/app/prj.conf b/u5/app/prj.conf index d0d7494..4850df5 100644 --- a/u5/app/prj.conf +++ b/u5/app/prj.conf @@ -9,9 +9,6 @@ CONFIG_STD_CPP20=y CONFIG_MAIN_STACK_SIZE=65536 -CONFIG_DISPLAY=y -CONFIG_DISPLAY_LOG_LEVEL_ERR=y - CONFIG_LOG=y CONFIG_LOG_DEFAULT_LEVEL=3 CONFIG_SHELL=y diff --git a/u5/app/src/grinreflex_utils.cpp b/u5/app/src/grinreflex_utils.cpp deleted file mode 100644 index c38b6a0..0000000 --- a/u5/app/src/grinreflex_utils.cpp +++ /dev/null @@ -1,91 +0,0 @@ -#include "grinreflex_config.hpp" -#include "gfx_utils.hpp" - -#include "grinreflex_utils.hpp" - -lv_obj_t *allocLvROIRect(lv_obj_t *parent, uint32_t borderWidth) { - lv_obj_t *rect = lv_obj_create(parent); - lv_obj_set_pos(rect, 0, 0); - lv_obj_set_size(rect, 1, 1); - - // Unfilled, just a thick border - lv_obj_set_style_bg_opa(rect, LV_OPA_TRANSP, LV_PART_MAIN); - lv_obj_set_style_border_width(rect, borderWidth, LV_PART_MAIN); - lv_obj_set_style_border_opa(rect, LV_OPA_COVER, LV_PART_MAIN); - lv_obj_set_style_border_color(rect, lv_palette_main(LV_PALETTE_GREEN), LV_PART_MAIN); - lv_obj_set_style_radius(rect, 0, LV_PART_MAIN); // 0 = sharp corners - lv_obj_set_style_pad_all(rect, 0, LV_PART_MAIN); - lv_obj_set_style_outline_opa(rect, LV_OPA_TRANSP, LV_PART_MAIN); // no focus outline - lv_obj_set_style_shadow_opa(rect, LV_OPA_TRANSP, LV_PART_MAIN); // no shadow - lv_obj_move_foreground(rect); - lv_obj_add_flag(rect, LV_OBJ_FLAG_HIDDEN); - - return rect; -} - -void cropFullFrameToRoi(lv_obj_t *parent, uint8_t *fullFb, uint8_t *roiFb) { - Gfx::GfxBuffer src{}; - Gfx::GfxBuffer dst{}; - - uint32_t x = (FULL_FRAME_WIDTH - ROI_FRAME_WIDTH) / 2; - //uint32_t y = (FULL_FRAME_HEIGHT - ROI_FRAME_HEIGHT) / 2; -#if defined(FULL_FRAME_FLIP_Y) - uint32_t y = 0; -#else - uint32_t y = FULL_FRAME_HEIGHT - ROI_FRAME_HEIGHT; -#endif - - uint32_t fullFbStride = FULL_FRAME_WIDTH * LV_COLOR_FORMAT_GET_SIZE(FULL_FRAME_COLOR_FORMAT); - uint32_t fullFbOffset = (y * fullFbStride) + x; - uint32_t fullFbSize = ROI_FRAME_HEIGHT * fullFbStride; - - src.buf = fullFb + fullFbOffset; - src.width = ROI_FRAME_WIDTH; - src.height = ROI_FRAME_HEIGHT; - src.stride = fullFbStride; - src.cf = FULL_FRAME_COLOR_FORMAT; - src.size = fullFbSize - fullFbOffset; - - uint32_t roiFbStride = ROI_FRAME_WIDTH * LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8); - - dst.buf = roiFb; - dst.width = ROI_FRAME_WIDTH; - dst.height = ROI_FRAME_HEIGHT; - dst.stride = roiFbStride; - dst.cf = LV_COLOR_FORMAT_L8; - dst.size = ROI_FRAME_HEIGHT * roiFbStride; - - -#if defined(FULL_FRAME_FLIP_Y) - Gfx::fit(parent, src, dst, true); -#else - Gfx::fit(parent, src, dst); -#endif - //lv_task_handler(); -} - -void fitRoiFrameToThumbnail(lv_obj_t *parent, uint8_t *roiFb, uint8_t *tFb) { - Gfx::GfxBuffer src{}; - Gfx::GfxBuffer dst{}; - - uint32_t roiFbStride = ROI_FRAME_WIDTH * LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8); - - src.buf = roiFb; - src.width = ROI_FRAME_WIDTH; - src.height = ROI_FRAME_HEIGHT; - src.stride = roiFbStride; - src.cf = LV_COLOR_FORMAT_L8; - src.size = ROI_FRAME_HEIGHT * roiFbStride; - - uint32_t tnFbStride = THUMBNAIL_FRAME_WIDTH * LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8); - - dst.buf = tFb; - dst.width = THUMBNAIL_FRAME_WIDTH; - dst.height = THUMBNAIL_FRAME_HEIGHT; - dst.stride = tnFbStride; - dst.cf = LV_COLOR_FORMAT_L8; - dst.size = THUMBNAIL_FRAME_HEIGHT * tnFbStride; - - Gfx::fit(parent, src, dst); - //lv_task_handler(); -} diff --git a/u5/app/src/grinreflex_utils.hpp b/u5/app/src/grinreflex_utils.hpp deleted file mode 100644 index 0cba9ad..0000000 --- a/u5/app/src/grinreflex_utils.hpp +++ /dev/null @@ -1,10 +0,0 @@ - - -#pragma once - -#include - -lv_obj_t *allocLvROIRect(lv_obj_t *parent, uint32_t borderWidth); - -void cropFullFrameToRoi(lv_obj_t *parent, uint8_t *fullFb, uint8_t *roiFb); -void fitRoiFrameToThumbnail(lv_obj_t *parent, uint8_t *roiFb, uint8_t *tFb); diff --git a/u5/boards/vendor/grinreflex/Kconfig.grinreflex_dk1 b/u5/boards/vendor/grinreflex/Kconfig.grinreflex_dk1 index 58fd46e..15961c3 100644 --- a/u5/boards/vendor/grinreflex/Kconfig.grinreflex_dk1 +++ b/u5/boards/vendor/grinreflex/Kconfig.grinreflex_dk1 @@ -1,4 +1,39 @@ # Copyright (c) 2025 Grinreflex config BOARD_GRINREFLEX_DK1 - select SOC_STM32U5G9XX \ No newline at end of file + select SOC_STM32U5G9XX + +menu "GrinReflex Video settings" + +config GRINREFLEX_JPEG_VIDEO + bool "Enable jpeg video input" + default y + +config GRINREFLEX_JPEG_VIDEO_QUALITY + int "Set jpeg quality, lower value - higher quality" + default 63 + depends on GRINREFLEX_JPEG_VIDEO + +config GRINREFLEX_VIDEO_WIDTH + int "Width of a video frame" + default 160 + +config GRINREFLEX_VIDEO_HEIGHT + int "Height of a video frame" + default 120 + +config GRINREFLEX_VIDEO_RGB565_SWAP_BYTES + bool "Swap rgb565 bytes (e.g. in ov2640 sensor)" + default n + +endmenu + +menu "GrinReflex LVGL settings" + +if GRINREFLEX_JPEG_VIDEO +config LV_COLOR_16_SWAP + bool + default y +endif + +endmenu \ No newline at end of file diff --git a/u5/boards/vendor/grinreflex/grinreflex_dk1.dts b/u5/boards/vendor/grinreflex/grinreflex_dk1.dts index 98c2b8e..6f75181 100644 --- a/u5/boards/vendor/grinreflex/grinreflex_dk1.dts +++ b/u5/boards/vendor/grinreflex/grinreflex_dk1.dts @@ -21,6 +21,8 @@ zephyr,sram = &sram0; zephyr,flash = &flash0; zephyr,code-partition = &slot0_partition; + zephyr,nema = &nema0; + zephyr,jpeg = &jpeg0; zephyr,display = &st7735r_160x80; zephyr,camera = &zephyr_camera_dvp; }; @@ -101,6 +103,19 @@ <22 0 &gpioc 8 0>; /* DVP_D2 */ }; + nema0: nema-gfx { + compatible = "st,nema-gfx"; + status = "okay"; + }; + + jpeg0: jpeg_hw { + compatible = "st,jpeg-hw"; + dmas = <&gpdma1 2 125 (STM32_DMA_PERIPH_RX | STM32_DMA_PERIPH_32BITS | STM32_DMA_MEM_32BITS) + &gpdma1 3 124 (STM32_DMA_PERIPH_TX | STM32_DMA_PERIPH_32BITS | STM32_DMA_MEM_32BITS)>; + dma-names = "tx", "rx"; + status = "okay"; + }; + }; &clk_hsi48 { @@ -460,7 +475,3 @@ zephyr_camera_dvp: &dcmi { pinctrl-0 = <&rcc_mco_pa8>; pinctrl-names = "default"; }; - -&dcmi { - jpeg-dev = <&jpeg0>; -}; diff --git a/u5/boards/vendor/grinreflex/grinreflex_dk1_defconfig b/u5/boards/vendor/grinreflex/grinreflex_dk1_defconfig index 695db2c..3d05425 100644 --- a/u5/boards/vendor/grinreflex/grinreflex_dk1_defconfig +++ b/u5/boards/vendor/grinreflex/grinreflex_dk1_defconfig @@ -22,13 +22,7 @@ CONFIG_VIDEO_LOG_LEVEL_DBG=y CONFIG_VIDEO_LOG_LEVEL_INF=y -CONFIG_STM32_JPEG_RGB_FORMAT=3 -CONFIG_GRINREFLEX_JPEG_VIDEO=y -CONFIG_GRINREFLEX_JPEG_VIDEO_QUALITY=50 -CONFIG_GRINREFLEX_VIDEO_WIDTH=640 -CONFIG_GRINREFLEX_VIDEO_HEIGHT=480 - -#CONFIG_VIDEO_BUFFER_POOL_SZ_MAX=76800 -CONFIG_VIDEO_BUFFER_POOL_SZ_MAX=614400 +CONFIG_DISPLAY=y +CONFIG_DISPLAY_LOG_LEVEL_ERR=y #CONFIG_LV_COLOR_16_SWAP=y \ No newline at end of file diff --git a/u5/boards/vendor/grinreflex_dk2/Kconfig.grinreflex_dk2 b/u5/boards/vendor/grinreflex_dk2/Kconfig.grinreflex_dk2 index 2b19d20..6ba085b 100644 --- a/u5/boards/vendor/grinreflex_dk2/Kconfig.grinreflex_dk2 +++ b/u5/boards/vendor/grinreflex_dk2/Kconfig.grinreflex_dk2 @@ -1,4 +1,39 @@ # Copyright (c) 2025 Grinreflex config BOARD_GRINREFLEX_DK2 - select SOC_STM32U5G9XX \ No newline at end of file + select SOC_STM32U5G9XX + +menu "GrinReflex Video settings" + +config GRINREFLEX_JPEG_VIDEO + bool "Enable jpeg video input" + default y + +config GRINREFLEX_JPEG_VIDEO_QUALITY + int "Set jpeg quality, lower value - higher quality" + default 63 + depends on GRINREFLEX_JPEG_VIDEO + +config GRINREFLEX_VIDEO_WIDTH + int "Width of a video frame" + default 160 + +config GRINREFLEX_VIDEO_HEIGHT + int "Height of a video frame" + default 120 + +config GRINREFLEX_VIDEO_RGB565_SWAP_BYTES + bool "Swap rgb565 bytes (e.g. in ov2640 sensor)" + default n + +endmenu + +menu "GrinReflex LVGL settings" + +if GRINREFLEX_JPEG_VIDEO +config LV_COLOR_16_SWAP + bool + default y +endif + +endmenu \ No newline at end of file diff --git a/u5/boards/vendor/grinreflex_dk2/grinreflex_dk2.dts b/u5/boards/vendor/grinreflex_dk2/grinreflex_dk2.dts index c24fab1..d7eb7af 100644 --- a/u5/boards/vendor/grinreflex_dk2/grinreflex_dk2.dts +++ b/u5/boards/vendor/grinreflex_dk2/grinreflex_dk2.dts @@ -21,6 +21,8 @@ zephyr,sram = &sram0; zephyr,flash = &flash0; zephyr,code-partition = &slot0_partition; + zephyr,nema = &nema0; + zephyr,jpeg = &jpeg0; zephyr,display = &st7735r_160x80; zephyr,camera = &zephyr_camera_dvp; }; @@ -103,6 +105,19 @@ <22 0 &gpioc 8 0>; /* DVP_D2 */ }; + nema0: nema-gfx { + compatible = "st,nema-gfx"; + status = "okay"; + }; + + jpeg0: jpeg_hw { + compatible = "st,jpeg-hw"; + dmas = <&gpdma1 2 125 (STM32_DMA_PERIPH_RX | STM32_DMA_PERIPH_32BITS | STM32_DMA_MEM_32BITS) + &gpdma1 3 124 (STM32_DMA_PERIPH_TX | STM32_DMA_PERIPH_32BITS | STM32_DMA_MEM_32BITS)>; + dma-names = "tx", "rx"; + status = "okay"; + }; + }; &clk_hsi48 { diff --git a/u5/boards/vendor/grinreflex_dk2/grinreflex_dk2_defconfig b/u5/boards/vendor/grinreflex_dk2/grinreflex_dk2_defconfig index 90b8554..7be071a 100644 --- a/u5/boards/vendor/grinreflex_dk2/grinreflex_dk2_defconfig +++ b/u5/boards/vendor/grinreflex_dk2/grinreflex_dk2_defconfig @@ -31,4 +31,7 @@ CONFIG_GRINREFLEX_VIDEO_HEIGHT=480 #CONFIG_VIDEO_BUFFER_POOL_SZ_MAX=76800 CONFIG_VIDEO_BUFFER_POOL_SZ_MAX=153600 +CONFIG_DISPLAY=y +CONFIG_DISPLAY_LOG_LEVEL_ERR=y + #CONFIG_LV_COLOR_16_SWAP=y \ No newline at end of file diff --git a/u5/common/CMakeLists.txt b/u5/common/CMakeLists.txt new file mode 100644 index 0000000..3fa94c7 --- /dev/null +++ b/u5/common/CMakeLists.txt @@ -0,0 +1,25 @@ + + +if (CONFIG_OPENCV_LIB) +add_subdirectory(opencv) +endif() + +if (CONFIG_NEMA) +add_subdirectory(nema) +endif() + +target_include_directories(app PUBLIC include) + +target_sources(app PRIVATE + src/lvgl_utils.cpp + src/video.cpp + src/utils.cpp +) + +dt_has_chosen(USART2 PROPERTY "zephyr_usart2") + +if (USART2) +target_sources(app PRIVATE + src/uart.cpp +) +endif() diff --git a/u5/app/src/gfx_utils.hpp b/u5/common/include/gf/lvgl_utils.hpp similarity index 58% rename from u5/app/src/gfx_utils.hpp rename to u5/common/include/gf/lvgl_utils.hpp index 0963f20..e5cbdec 100644 --- a/u5/app/src/gfx_utils.hpp +++ b/u5/common/include/gf/lvgl_utils.hpp @@ -6,14 +6,14 @@ #include -namespace Gfx { +namespace lvgl { -struct GfxRect { - uint32_t x, y; +struct Size { + Size (uint32_t w, uint32_t h) : width(w), height(h) {} uint32_t width, height; }; -struct GfxBuffer { +struct Buffer { uint8_t *buf; uint32_t width, height, stride; size_t size; @@ -24,6 +24,6 @@ struct GfxBuffer { * @brief Fit source buffer onto destination * */ -void fit(lv_obj_t *canvas, const GfxBuffer &src, const GfxBuffer &dst, bool flip_y = false); +void fit(lv_obj_t *canvas, const Buffer &src, const Buffer &dst, bool flip_y = false); } // namespace Gfx diff --git a/u5/app/src/uart.hpp b/u5/common/include/gf/uart.hpp similarity index 100% rename from u5/app/src/uart.hpp rename to u5/common/include/gf/uart.hpp diff --git a/u5/common/include/gf/utils.hpp b/u5/common/include/gf/utils.hpp new file mode 100644 index 0000000..a76a3f7 --- /dev/null +++ b/u5/common/include/gf/utils.hpp @@ -0,0 +1,12 @@ + + +#pragma once + +#include + +lv_obj_t *allocLvROIRect(lv_obj_t *parent, uint32_t borderWidth); + +void cropFullFrameToRoi(lv_obj_t *parent, uint8_t *fullFb, uint8_t *roiFb, lvgl::Size fullSz, + lvgl::Size roiSz, uint32_t fullCf, uint32_t roiCf, bool flip_y = false); +void fitRoiFrameToThumbnail(lv_obj_t *parent, uint8_t *roiFb, uint8_t *tFb, lvgl::Size roiSz, + lvgl::Size tSz, uint32_t roiCf, uint32_t tCf); diff --git a/u5/app/src/video.hpp b/u5/common/include/gf/video.hpp similarity index 100% rename from u5/app/src/video.hpp rename to u5/common/include/gf/video.hpp diff --git a/u5/common/nema/CMakeLists.txt b/u5/common/nema/CMakeLists.txt new file mode 100644 index 0000000..b280c93 --- /dev/null +++ b/u5/common/nema/CMakeLists.txt @@ -0,0 +1,28 @@ +if (NOT CONFIG_FPU) + message(FATAL_ERROR "FPU must be enabled: CONFIG_FPU=y") +endif() + +set(LVGL_LIB_PATH ${CMAKE_CURRENT_LIST_DIR}/../../../modules/lib/gui/lvgl) +set(LVGL_NEMA_PATH ${LVGL_LIB_PATH}/libs/nema_gfx) +set(LVGL_NEMA_LIBA_PATH ${LVGL_NEMA_PATH}/lib/core/cortex_m33_revC/gcc) + + +zephyr_include_directories( + ${LVGL_NEMA_PATH}/include + ${LVGL_LIB_PATH}/src/draw/nema_gfx +) + +if (CONFIG_FP_SOFTABI) +target_link_libraries(app PRIVATE + ${LVGL_NEMA_LIBA_PATH}/libnemagfx.a + ${LVGL_NEMA_LIBA_PATH}/libnemagfx-wc16.a +) +else() +target_link_libraries(app PRIVATE + ${LVGL_NEMA_LIBA_PATH}/libnemagfx-float-abi-hard.a + ${LVGL_NEMA_LIBA_PATH}/libnemagfx-float-abi-hard-wc16.a +) +endif() # CONFIG_FP_SOFTABI + +file(GLOB_RECURSE SOURCES ${LVGL_LIB_PATH}/src/draw/nema_gfx/*.c) +zephyr_library_sources(${SOURCES}) \ No newline at end of file diff --git a/u5/common/opencv/CMakeLists.txt b/u5/common/opencv/CMakeLists.txt new file mode 100644 index 0000000..6ca12eb --- /dev/null +++ b/u5/common/opencv/CMakeLists.txt @@ -0,0 +1,28 @@ +# TODO: below two lines are unclear +set(NO_THREADSAFE_STATICS $) +zephyr_compile_options($<$:${NO_THREADSAFE_STATICS}>) + +zephyr_compile_options(-DOPENCV_DISABLE_THREAD_SUPPORT) +zephyr_compile_options(-DOPENCV_FLANN_USE_STD_RAND) + +target_sources(app PRIVATE + src/opencv_utils.cpp +) + +set(OPENCV_LBP_CASCADES ${OPENCV_INSTALL_DIR}/share/opencv4/lbpcascades) +set(OPENCV_HAAR_CASCADES ${OPENCV_INSTALL_DIR}/share/opencv4/haarcascades) + +set(OPENCV_CASCADE_FACE_PATH ${OPENCV_LBP_CASCADES}/lbpcascade_frontalface_improved.xml) +set(OPENCV_CASCADE_SMILE_PATH ${OPENCV_HAAR_CASCADES}/haarcascade_smile.xml) + +configure_file( + src/cascades.h.in + src/cascades.h + @ONLY +) + +target_include_directories(app PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/src") + +target_include_directories(app PRIVATE include) + +target_link_libraries(app PUBLIC opencv_lib) \ No newline at end of file diff --git a/u5/app/src/opencv_utils.hpp b/u5/common/opencv/include/gf/opencv_utils.hpp similarity index 91% rename from u5/app/src/opencv_utils.hpp rename to u5/common/opencv/include/gf/opencv_utils.hpp index 3fa8719..7656530 100644 --- a/u5/app/src/opencv_utils.hpp +++ b/u5/common/opencv/include/gf/opencv_utils.hpp @@ -22,16 +22,17 @@ bool loadCascade(cv::CascadeClassifier &cascade, const uint8_t *buffer, size_t l * @param faceROIMax maximum faceROI to be resized to if detected ROI is largen than this * @param faces vector with faces rects * @param smiles vector with sniles rects - * @return std::vector all ROI's ever considered during run + * @return cv::Mat smile ROI ever considered during run */ -std::vector detectFaceAndSmile( +cv::Mat detectFaceAndSmile( cv::CascadeClassifier &faceCascade, cv::CascadeClassifier &smileCascade, cv::Mat &thumbnailFrame, cv::Mat &fullFrame, cv::Rect &faceROIMax, std::vector &faces, - std::vector &smiles + std::vector &smiles, + float mouthGamma ); cv::Mat cv_preprocessForQR(const cv::Mat& bgr); diff --git a/u5/app/src/cascades.h.in b/u5/common/opencv/src/cascades.h.in similarity index 100% rename from u5/app/src/cascades.h.in rename to u5/common/opencv/src/cascades.h.in diff --git a/u5/app/src/opencv_utils.cpp b/u5/common/opencv/src/opencv_utils.cpp similarity index 76% rename from u5/app/src/opencv_utils.cpp rename to u5/common/opencv/src/opencv_utils.cpp index d76217e..044134b 100644 --- a/u5/app/src/opencv_utils.cpp +++ b/u5/common/opencv/src/opencv_utils.cpp @@ -7,7 +7,7 @@ #include #include -#include "opencv_utils.hpp" +#include "gf/opencv_utils.hpp" bool loadCascade(cv::CascadeClassifier &cascade, const char *path) { if (!cascade.load(path)) { @@ -53,41 +53,63 @@ cv::Mat cv_preprocessForQR(const cv::Mat& bgr) { return bin; } -std::vector detectFaceAndSmile( +static cv::Mat preprocessSmileROI(const cv::Mat& grayROI, float gamma) { + // 1) CLAHE for local contrast (great in dim light) + cv::Ptr clahe = cv::createCLAHE(5.0, cv::Size(4,4)); + cv::Mat claheROI, bfROI; + clahe->apply(grayROI, claheROI); + + // 2) Denoise but keep edges + cv::bilateralFilter(claheROI, bfROI, 3, 20, 50); + + // 3) Gamma lift (power-law) + bfROI.convertTo(bfROI, CV_32FC1, 1.0/255.0); + cv::pow(bfROI, gamma, bfROI); + bfROI.convertTo(bfROI, CV_8UC1, 255.0); + + // 4) Light unsharp mask to emphasize edges + cv::Mat blurImg; + cv::GaussianBlur(bfROI, blurImg, cv::Size(0,0), 1.0); + cv::addWeighted(bfROI, 1.5, blurImg, -0.5, 0, bfROI); + + return bfROI; +} + +cv::Mat detectFaceAndSmile( cv::CascadeClassifier &faceCascade, cv::CascadeClassifier &smileCascade, cv::Mat &thumbnailFrame, cv::Mat &fullFrame, cv::Rect &faceROIMax, std::vector &faceROIs, - std::vector &smileROIs) { - - std::vector outROIs; + std::vector &smileROIs, + float mouthGamma) { const double scaleFactor = 1.04; const int minNeighbors = 3; - const int flags = 0; + const int flags = cv::CASCADE_FIND_BIGGEST_OBJECT | cv::CASCADE_DO_ROUGH_SEARCH; + cv::Mat smilePostProc; + //cv::equalizeHist(thumbnailFrame, thumbnailFrame); faceCascade.detectMultiScale(thumbnailFrame, faceROIs, scaleFactor, minNeighbors, flags); for (auto & faceROI : faceROIs) { - outROIs.push_back(faceROI); try { cv::Rect faceROIFull = remapROI(faceROI, thumbnailFrame, fullFrame); /* Get lower half/third of face ROI + some gap where the smile is most likelly to be */ - const uint32_t offset = (faceROIFull.height * 2) / 3; - faceROIFull.y = faceROIFull.y + offset; + const uint32_t offset_y = (faceROIFull.height * 2) / 3; + + faceROIFull.y = faceROIFull.y + offset_y - 10; faceROIFull.height = faceROIFull.height - faceROIFull.height / 2; if (faceROIFull.height + faceROIFull.y > fullFrame.rows) { faceROIFull.height = fullFrame.rows - faceROIFull.y; } cv::Rect roiThumb = remapROI(faceROIFull, fullFrame, thumbnailFrame); - outROIs.push_back(roiThumb); float sx = 1.0f; float sy = 1.0f; @@ -104,10 +126,14 @@ std::vector detectFaceAndSmile( faceROIFrameResized = faceROIFrame; } - const double scaleFactor = 1.05; + const double scaleFactor = 1.1; const int minNeighbors = 3; - const int flags = 0; + const int flags = cv::CASCADE_FIND_BIGGEST_OBJECT | cv::CASCADE_DO_ROUGH_SEARCH; + + //cv::equalizeHist(faceROIFrameResized, faceROIFrameResized); + faceROIFrameResized = preprocessSmileROI(faceROIFrameResized, mouthGamma); + smilePostProc = faceROIFrameResized.clone(); smileCascade.detectMultiScale(faceROIFrameResized, smileROIs, scaleFactor, minNeighbors, flags); @@ -118,7 +144,6 @@ std::vector detectFaceAndSmile( cv::Rect smileROIRemapped = translateScaleROI(smileROI, 1.0f / sx, 1.0f / sy, faceROIFull.x, faceROIFull.y); smileROIRemapped = translateScaleROI(smileROIRemapped, ssx_inv, ssy_inv, 0.0f, 0.0f); - outROIs.push_back(smileROIRemapped); smileROI = smileROIRemapped; } @@ -129,7 +154,7 @@ std::vector detectFaceAndSmile( } } - return outROIs; + return smilePostProc; } cv::Rect remapROI(cv::Rect &ROI, cv::Mat &inFrame, cv::Mat &outFrame) { diff --git a/u5/app/src/gfx_utils.cpp b/u5/common/src/lvgl_utils.cpp similarity index 89% rename from u5/app/src/gfx_utils.cpp rename to u5/common/src/lvgl_utils.cpp index fb897f2..69e4fd5 100644 --- a/u5/app/src/gfx_utils.cpp +++ b/u5/common/src/lvgl_utils.cpp @@ -1,9 +1,9 @@ -#include "gfx_utils.hpp" +#include "gf/lvgl_utils.hpp" -namespace Gfx { +namespace lvgl { -void fit(lv_obj_t *canvas, const GfxBuffer &src, const GfxBuffer &dst, bool flip_y) { +void fit(lv_obj_t *canvas, const Buffer &src, const Buffer &dst, bool flip_y) { lv_image_dsc_t src_img = { .header = diff --git a/u5/app/src/uart.cpp b/u5/common/src/uart.cpp similarity index 99% rename from u5/app/src/uart.cpp rename to u5/common/src/uart.cpp index 20147b5..c4635fb 100644 --- a/u5/app/src/uart.cpp +++ b/u5/common/src/uart.cpp @@ -7,7 +7,7 @@ #include #include -#include "uart.hpp" +#include "gf/uart.hpp" LOG_MODULE_REGISTER(uart); diff --git a/u5/common/src/utils.cpp b/u5/common/src/utils.cpp new file mode 100644 index 0000000..89d56f5 --- /dev/null +++ b/u5/common/src/utils.cpp @@ -0,0 +1,82 @@ +#include "gf/utils.hpp" + +lv_obj_t *allocLvROIRect(lv_obj_t *parent, uint32_t borderWidth) { + lv_obj_t *rect = lv_obj_create(parent); + lv_obj_set_pos(rect, 0, 0); + lv_obj_set_size(rect, 1, 1); + + // Unfilled, just a thick border + lv_obj_set_style_bg_opa(rect, LV_OPA_TRANSP, LV_PART_MAIN); + lv_obj_set_style_border_width(rect, borderWidth, LV_PART_MAIN); + lv_obj_set_style_border_opa(rect, LV_OPA_COVER, LV_PART_MAIN); + lv_obj_set_style_border_color(rect, lv_palette_main(LV_PALETTE_GREEN), LV_PART_MAIN); + lv_obj_set_style_radius(rect, 0, LV_PART_MAIN); // 0 = sharp corners + lv_obj_set_style_pad_all(rect, 0, LV_PART_MAIN); + lv_obj_set_style_outline_opa(rect, LV_OPA_TRANSP, LV_PART_MAIN); // no focus outline + lv_obj_set_style_shadow_opa(rect, LV_OPA_TRANSP, LV_PART_MAIN); // no shadow + lv_obj_move_foreground(rect); + lv_obj_add_flag(rect, LV_OBJ_FLAG_HIDDEN); + + return rect; +} + +void cropFullFrameToRoi(lv_obj_t *parent, uint8_t *fullFb, uint8_t *roiFb, lvgl::Size fullSz, + lvgl::Size roiSz, uint32_t fullCf, uint32_t roiCf, bool flip_y) { + lvgl::Buffer src{}; + lvgl::Buffer dst{}; + + uint32_t x = (fullSz.width - roiSz.width) / 2; + //uint32_t y = (FULL_FRAME_HEIGHT - ROI_FRAME_HEIGHT) / 2; + uint32_t y = flip_y ? 0 : fullSz.height - roiSz.height; + + uint32_t fullFbStride = fullSz.width * LV_COLOR_FORMAT_GET_SIZE(fullCf); + uint32_t fullFbOffset = (y * fullFbStride) + x; + uint32_t fullFbSize = roiSz.height * fullFbStride; + + src.buf = fullFb + fullFbOffset; + src.width = roiSz.width; + src.height = roiSz.height; + src.stride = fullFbStride; + src.cf = fullCf; + src.size = fullFbSize - fullFbOffset; + + uint32_t roiFbStride = roiSz.width * LV_COLOR_FORMAT_GET_SIZE(roiCf); + + dst.buf = roiFb; + dst.width = roiSz.width; + dst.height = roiSz.height; + dst.stride = roiFbStride; + dst.cf = roiCf; + dst.size = roiSz.height * roiFbStride; + + + lvgl::fit(parent, src, dst, flip_y); + //lv_task_handler(); +} + +void fitRoiFrameToThumbnail(lv_obj_t *parent, uint8_t *roiFb, uint8_t *tFb, lvgl::Size roiSz, + lvgl::Size tSz, uint32_t roiCf, uint32_t tCf) { + lvgl::Buffer src{}; + lvgl::Buffer dst{}; + + uint32_t roiFbStride = roiSz.width * LV_COLOR_FORMAT_GET_SIZE(roiCf); + + src.buf = roiFb; + src.width = roiSz.width; + src.height = roiSz.height; + src.stride = roiFbStride; + src.cf = roiCf; + src.size = roiSz.height * roiFbStride; + + uint32_t tnFbStride = tSz.width * LV_COLOR_FORMAT_GET_SIZE(tCf); + + dst.buf = tFb; + dst.width = tSz.width; + dst.height = tSz.height; + dst.stride = tnFbStride; + dst.cf = tCf; + dst.size = tSz.height * tnFbStride; + + lvgl::fit(parent, src, dst); + //lv_task_handler(); +} diff --git a/u5/app/src/video.cpp b/u5/common/src/video.cpp similarity index 99% rename from u5/app/src/video.cpp rename to u5/common/src/video.cpp index 5e770ef..a683214 100644 --- a/u5/app/src/video.cpp +++ b/u5/common/src/video.cpp @@ -5,7 +5,7 @@ #include #include -#include "video.hpp" +#include "gf/video.hpp" #define LOG_LEVEL CONFIG_LOG_DEFAULT_LEVEL #include diff --git a/u5/drivers/CMakeLists.txt b/u5/drivers/CMakeLists.txt index a247a59..56f0b30 100644 --- a/u5/drivers/CMakeLists.txt +++ b/u5/drivers/CMakeLists.txt @@ -1,12 +1,6 @@ # Copyright (c) 2021 Nordic Semiconductor ASA # SPDX-License-Identifier: Apache-2.0 -# Out-of-tree drivers for custom classes -add_subdirectory_ifdef(CONFIG_BLINK blink) - -# Out-of-tree drivers for existing driver classes -add_subdirectory_ifdef(CONFIG_SENSOR sensor) - # Out-of-tree drivers for existing driver classes STM32 NEMA 2DGPU add_subdirectory_ifdef(CONFIG_NEMA nema) add_subdirectory_ifdef(CONFIG_STM32_JPEG jpeg) diff --git a/u5/drivers/Kconfig b/u5/drivers/Kconfig index 92f92b8..b821cc8 100644 --- a/u5/drivers/Kconfig +++ b/u5/drivers/Kconfig @@ -2,7 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 menu "Drivers" -rsource "sensor/Kconfig" rsource "nema/Kconfig" rsource "jpeg/Kconfig" endmenu diff --git a/u5/drivers/jpeg/jpeg.c b/u5/drivers/jpeg/jpeg.c index 71c87c7..eff7a50 100644 --- a/u5/drivers/jpeg/jpeg.c +++ b/u5/drivers/jpeg/jpeg.c @@ -120,10 +120,12 @@ void HAL_JPEG_ErrorCallback(JPEG_HandleTypeDef *hjpeg) { void HAL_JPEG_DecodeCpltCallback(JPEG_HandleTypeDef *hjpeg) { struct jpeg_hw_data *data = CONTAINER_OF(hjpeg, struct jpeg_hw_data, hjpeg); - atomic_set(&data->data_ready, 1); if (data->parent_dev && data->cplt_callback) { data->cplt_callback(data->parent_dev); } + atomic_set(&data->data_ready, 1); + + // LOG_INF("HAL_JPEG_DecodeCpltCallback"); } static void dma_callback(const struct device *dma_dev, void *arg, diff --git a/u5/drivers/sensor/CMakeLists.txt b/u5/drivers/sensor/CMakeLists.txt deleted file mode 100644 index 72ed549..0000000 --- a/u5/drivers/sensor/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (c) 2021 Nordic Semiconductor ASA -# SPDX-License-Identifier: Apache-2.0 - -add_subdirectory_ifdef(CONFIG_EXAMPLE_SENSOR example_sensor) diff --git a/u5/drivers/sensor/Kconfig b/u5/drivers/sensor/Kconfig deleted file mode 100644 index 07b6c17..0000000 --- a/u5/drivers/sensor/Kconfig +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2021 Nordic Semiconductor ASA -# SPDX-License-Identifier: Apache-2.0 - -if SENSOR -rsource "example_sensor/Kconfig" -endif # SENSOR diff --git a/u5/drivers/sensor/example_sensor/CMakeLists.txt b/u5/drivers/sensor/example_sensor/CMakeLists.txt deleted file mode 100644 index fcb0596..0000000 --- a/u5/drivers/sensor/example_sensor/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright (c) 2021 Nordic Semiconductor ASA -# SPDX-License-Identifier: Apache-2.0 - -zephyr_library() -zephyr_library_sources(example_sensor.c) diff --git a/u5/drivers/sensor/example_sensor/Kconfig b/u5/drivers/sensor/example_sensor/Kconfig deleted file mode 100644 index fe9a358..0000000 --- a/u5/drivers/sensor/example_sensor/Kconfig +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) 2021 Nordic Semiconductor ASA -# SPDX-License-Identifier: Apache-2.0 - -config EXAMPLE_SENSOR - bool "Example sensor" - default y - depends on DT_HAS_ZEPHYR_EXAMPLE_SENSOR_ENABLED - select GPIO - help - Enable example sensor diff --git a/u5/drivers/sensor/example_sensor/example_sensor.c b/u5/drivers/sensor/example_sensor/example_sensor.c deleted file mode 100644 index b300cfe..0000000 --- a/u5/drivers/sensor/example_sensor/example_sensor.c +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (c) 2021 Nordic Semiconductor ASA - * SPDX-License-Identifier: Apache-2.0 - */ - -#define DT_DRV_COMPAT zephyr_example_sensor - -#include -#include -#include - -#include -LOG_MODULE_REGISTER(example_sensor, CONFIG_SENSOR_LOG_LEVEL); - -struct example_sensor_data { - int state; -}; - -struct example_sensor_config { - struct gpio_dt_spec input; -}; - -static int example_sensor_sample_fetch(const struct device *dev, - enum sensor_channel chan) -{ - const struct example_sensor_config *config = dev->config; - struct example_sensor_data *data = dev->data; - - data->state = gpio_pin_get_dt(&config->input); - - return 0; -} - -static int example_sensor_channel_get(const struct device *dev, - enum sensor_channel chan, - struct sensor_value *val) -{ - struct example_sensor_data *data = dev->data; - - if (chan != SENSOR_CHAN_PROX) { - return -ENOTSUP; - } - - val->val1 = data->state; - - return 0; -} - -static DEVICE_API(sensor, example_sensor_api) = { - .sample_fetch = &example_sensor_sample_fetch, - .channel_get = &example_sensor_channel_get, -}; - -static int example_sensor_init(const struct device *dev) -{ - const struct example_sensor_config *config = dev->config; - - int ret; - - if (!device_is_ready(config->input.port)) { - LOG_ERR("Input GPIO not ready"); - return -ENODEV; - } - - ret = gpio_pin_configure_dt(&config->input, GPIO_INPUT); - if (ret < 0) { - LOG_ERR("Could not configure input GPIO (%d)", ret); - return ret; - } - - return 0; -} - -#define EXAMPLE_SENSOR_INIT(i) \ - static struct example_sensor_data example_sensor_data_##i; \ - \ - static const struct example_sensor_config example_sensor_config_##i = {\ - .input = GPIO_DT_SPEC_INST_GET(i, input_gpios), \ - }; \ - \ - DEVICE_DT_INST_DEFINE(i, example_sensor_init, NULL, \ - &example_sensor_data_##i, \ - &example_sensor_config_##i, POST_KERNEL, \ - CONFIG_SENSOR_INIT_PRIORITY, &example_sensor_api); - -DT_INST_FOREACH_STATUS_OKAY(EXAMPLE_SENSOR_INIT) diff --git a/u5/gf_dkx/CMakeLists.txt b/u5/gf_dkx/CMakeLists.txt new file mode 100644 index 0000000..820e360 --- /dev/null +++ b/u5/gf_dkx/CMakeLists.txt @@ -0,0 +1,35 @@ +#------------------------------------------------------------------------------- +# Zephyr Example Application +# +# Copyright (c) 2021 Nordic Semiconductor ASA +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.13.1) + +set(EXTRA_CONF_FILE + ${CMAKE_CURRENT_SOURCE_DIR}/lvgl.conf; +) + +find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) + +project(gf_dkx) + +target_include_directories(app PRIVATE + src +) + +target_sources(app PRIVATE + src/main.c +) + +include(${ZEPHYR_BASE}/samples/subsys/usb/common/common.cmake) + +if (CONFIG_OPENCV_LIB) +target_sources(app PRIVATE + src/opencv_fd_app.cpp +) +else() +target_sources(app PRIVATE + src/lvgl_video_app.cpp +) +endif() diff --git a/u5/gf_dkx/Kconfig b/u5/gf_dkx/Kconfig new file mode 100644 index 0000000..46f6b7b --- /dev/null +++ b/u5/gf_dkx/Kconfig @@ -0,0 +1,58 @@ +# Copyright (c) 2021 Nordic Semiconductor ASA +# SPDX-License-Identifier: Apache-2.0 +# +# This file is the application Kconfig entry point. All application Kconfig +# options can be defined here or included via other application Kconfig files. +# You can browse these options using the west targets menuconfig (terminal) or +# guiconfig (GUI). + +#menu "Zephyr" +#source "Kconfig.zephyr" +#endmenu + +module = grinreflex +module-str = grinreflex + +source "subsys/logging/Kconfig.template.log_config" +source "samples/subsys/usb/common/Kconfig.sample_usbd" +source "Kconfig.zephyr" + +menu "MSC sample options" + +choice + prompt "Storage and file system type used by the application" + default APP_MSC_STORAGE_NONE + help + Specify the type of storage and file system. + +config APP_MSC_STORAGE_FLASH_FATFS + bool "Use FLASH disk and FAT file system" + imply DISK_DRIVERS + imply DISK_ACCESS + imply FILE_SYSTEM + imply FAT_FILESYSTEM_ELM + imply FILE_SYSTEM_MKFS + imply FS_FATFS_FSTAB_AUTOMOUNT + +config APP_MSC_STORAGE_NONE + bool "Use nothing" + +endchoice + +config MASS_STORAGE_DISK_NAME + default "NAND" if DISK_DRIVER_FLASH + +if DISK_DRIVER_FLASH + +config FLASH_MAP + default y + +config FLASH_PAGE_LAYOUT + default y + +config FLASH_LOG_LEVEL + default 3 + +endif # DISK_DRIVER_FLASH + +endmenu \ No newline at end of file diff --git a/u5/gf_dkx/boards/grinreflex_dk1.conf b/u5/gf_dkx/boards/grinreflex_dk1.conf new file mode 100644 index 0000000..f23a10f --- /dev/null +++ b/u5/gf_dkx/boards/grinreflex_dk1.conf @@ -0,0 +1,13 @@ + +#CONFIG_USB_DEVICE_STACK_NEXT=y +#CONFIG_USBD_MSC_CLASS=y +#CONFIG_USBD_MSC_LUNS_PER_INSTANCE=3 +#CONFIG_USBD_LOG_LEVEL_WRN=y +#CONFIG_UDC_DRIVER_LOG_LEVEL_WRN=y + +#CONFIG_SAMPLE_USBD_PID=0x0008 +#CONFIG_SAMPLE_USBD_PRODUCT="USBD MSC sample" + +#CONFIG_FILE_SYSTEM_SHELL=y + +#CONFIG_USB_DEVICE_INITIALIZE_AT_BOOT=n diff --git a/u5/gf_dkx/boards/grinreflex_dk1.overlay b/u5/gf_dkx/boards/grinreflex_dk1.overlay new file mode 100644 index 0000000..20a6118 --- /dev/null +++ b/u5/gf_dkx/boards/grinreflex_dk1.overlay @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2024, Eve Redero + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include + +/ { + chosen: chosen { + zephyr,spi1 = &spi1; + zephyr,usart2 = &usart2; + }; + + fstab { + compatible = "zephyr,fstab"; + + ffs2: ffs2 { + compatible = "zephyr,fstab,fatfs"; + automount; + disk-access; + mount-point = "/NAND:"; + }; + }; + + flashdisk0: flash-disk { + compatible = "zephyr,flash-disk"; + flash-controller = <&mx66lm1g45>; + partition = <&extflash_partition>; + disk-name = "NAND"; + cache-size = <4096>; + }; +}; + +&usart2 { + dmas = <&gpdma1 0 27 (STM32_DMA_PERIPH_TX) + &gpdma1 1 26 (STM32_DMA_MODE_CYCLIC | STM32_DMA_PERIPH_RX | STM32_DMA_MEM_8BITS)>; + dma-names = "tx", "rx"; + current-speed = <921600>; + fifo-enable; +}; + +&gpdma1 { + status = "okay"; +}; + +&dcmi { + jpeg-dev = <&jpeg0>; +}; \ No newline at end of file diff --git a/u5/gf_dkx/boards/grinreflex_dk2.conf b/u5/gf_dkx/boards/grinreflex_dk2.conf new file mode 100644 index 0000000..a0f7d65 --- /dev/null +++ b/u5/gf_dkx/boards/grinreflex_dk2.conf @@ -0,0 +1,4 @@ +CONFIG_FAT_FILESYSTEM_ELM=n +CONFIG_POSIX_FILE_SYSTEM=n + +CONFIG_VIDEO_SHELL=y diff --git a/u5/gf_dkx/boards/grinreflex_dk2.overlay b/u5/gf_dkx/boards/grinreflex_dk2.overlay new file mode 100644 index 0000000..62a9e81 --- /dev/null +++ b/u5/gf_dkx/boards/grinreflex_dk2.overlay @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2024, Eve Redero + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include + +/ { + chosen: chosen { + zephyr,spi1 = &spi1; + }; +}; + +&gpdma1 { + status = "okay"; +}; + +&dcmi { + jpeg-dev = <&jpeg0>; +}; \ No newline at end of file diff --git a/u5/gf_dkx/lvgl.conf b/u5/gf_dkx/lvgl.conf new file mode 100644 index 0000000..c6a38ad --- /dev/null +++ b/u5/gf_dkx/lvgl.conf @@ -0,0 +1,24 @@ + + +CONFIG_LV_Z_MEM_POOL_SIZE=16384 +CONFIG_LV_Z_SHELL=n + +CONFIG_LVGL=y +CONFIG_LV_USE_LOG=y +CONFIG_LV_USE_LABEL=y +CONFIG_LV_USE_ARC=y +CONFIG_LV_USE_IMAGE=y +#CONFIG_LV_USE_MONKEY=y +CONFIG_LV_FONT_MONTSERRAT_14=y +#CONFIG_LV_Z_POINTER_INPUT=y + +CONFIG_LV_CACHE_DEF_SIZE=4096 +CONFIG_LV_IMAGE_HEADER_CACHE_DEF_CNT=1 +#CONFIG_LV_USE_TJPGD=y +#CONFIG_LV_USE_FS_MEMFS=y +#CONFIG_LV_USE_FS=y +CONFIG_LV_USE_BUILTIN_MALLOC=n + +#CONFIG_LV_USE_PERF_MONITOR=y +#CONFIG_LV_USE_DROPDOWN=y +#CONFIG_LV_USE_SYSMON=y \ No newline at end of file diff --git a/u5/gf_dkx/prj.conf b/u5/gf_dkx/prj.conf new file mode 100644 index 0000000..4edbfb1 --- /dev/null +++ b/u5/gf_dkx/prj.conf @@ -0,0 +1,100 @@ +# Copyright (c) 2021 Nordic Semiconductor ASA +# SPDX-License-Identifier: Apache-2.0 +# +# This is a Kconfig fragment which can be used to enable debug-related options +# in the application. See the README for more details. + +CONFIG_CPP=y +CONFIG_STD_CPP20=y +CONFIG_CPP_EXCEPTIONS=y +CONFIG_CPP_RTTI=y + +CONFIG_MAIN_STACK_SIZE=65536 + +# Ensure logs/printk come out before the crash +CONFIG_PRINTK=y +CONFIG_LOG=y +CONFIG_LOG_PRINTK=y +CONFIG_LOG_DEFAULT_LEVEL=3 +CONFIG_LOG_MODE_MINIMAL=y +CONFIG_LOG_MODE_DEFERRED=n # immediate logs +#CONFIG_LOG_MODE_IMMEDIATE=y + +CONFIG_SHELL=y + +CONFIG_INPUT=y + +CONFIG_REQUIRES_FULL_LIBC=y +CONFIG_GLIBCXX_LIBCPP=y + +CONFIG_NEWLIB_LIBC=y +CONFIG_PICOLIBC=n + +CONFIG_COMMON_LIBC_MALLOC=y +CONFIG_HEAP_MEM_POOL_SIZE=49152 +CONFIG_COMMON_LIBC_MALLOC_ARENA_SIZE=2097152 +#CONFIG_COMMON_LIBC_MALLOC_ARENA_SIZE=1048576 + +CONFIG_POSIX_API=y + +CONFIG_ASSERT=y +# Make asserts print file:line and message +CONFIG_ASSERT_VERBOSE=y + +CONFIG_STACK_SENTINEL=y +CONFIG_THREAD_STACK_INFO=y +CONFIG_INIT_STACKS=y # fill stacks, helps detect overflow + +CONFIG_DEBUG_OPTIMIZATIONS=y +CONFIG_SPEED_OPTIMIZATIONS=y + +#CONFIG_TFLITE_LIB=y +CONFIG_OPENCV_LIB=y +#CONFIG_QUIRC_LIB=y +CONFIG_NEMA=y + +CONFIG_FPU=y +CONFIG_FP_SOFTABI=y + +CONFIG_GPIO=y +#CONFIG_SPI=y +#CONFIG_SPI_LOG_LEVEL_DBG=y + +CONFIG_DMA=y +CONFIG_DMA_LOG_LEVEL_INF=y + +CONFIG_SERIAL=y +CONFIG_RING_BUFFER=y +#CONFIG_UART_LOG_LEVEL_DBG=y + +CONFIG_UART_ASYNC_API=y +CONFIG_UART_INTERRUPT_DRIVEN=y + +CONFIG_STM32_JPEG=y + +#CONFIG_DISPLAY_LOG_LEVEL_DBG=y + +#CONFIG_HW_STACK_PROTECTION=y + +# Better symbols for gdb +#CONFIG_DEBUG_INFO=y +# (Optional) minimal optimizations for nicer backtraces +# CONFIG_NO_OPTIMIZATIONS=y + +# Richer fault dump on ARM (registers, etc.) +CONFIG_FAULT_DUMP=2 + +CONFIG_VIDEO_SHELL=y + +CONFIG_STM32_JPEG_RGB_FORMAT=3 +CONFIG_GRINREFLEX_JPEG_VIDEO=y +CONFIG_GRINREFLEX_JPEG_VIDEO_QUALITY=10 +CONFIG_GRINREFLEX_VIDEO_WIDTH=640 +CONFIG_GRINREFLEX_VIDEO_HEIGHT=480 + +CONFIG_VIDEO_LOG_LEVEL_DBG=y +CONFIG_VIDEO_LOG_LEVEL_WRN=y +CONFIG_VIDEO_LOG_LEVEL_ERR=y + +#CONFIG_VIDEO_BUFFER_POOL_SZ_MAX=76800 +CONFIG_VIDEO_BUFFER_POOL_SZ_MAX=153600 \ No newline at end of file diff --git a/u5/app/src/grinreflex_config.hpp b/u5/gf_dkx/src/config.hpp similarity index 96% rename from u5/app/src/grinreflex_config.hpp rename to u5/gf_dkx/src/config.hpp index 5f64bac..f280d01 100644 --- a/u5/app/src/grinreflex_config.hpp +++ b/u5/gf_dkx/src/config.hpp @@ -33,4 +33,4 @@ #define THUMBNAIL_FRAME_WIDTH 80 #define THUMBNAIL_FRAME_HEIGHT 80 -//#define FULL_FRAME_FLIP_Y (1) \ No newline at end of file +#define FULL_FRAME_FLIP_Y (1) \ No newline at end of file diff --git a/u5/app/src/grinreflex.h b/u5/gf_dkx/src/grinreflex.h similarity index 100% rename from u5/app/src/grinreflex.h rename to u5/gf_dkx/src/grinreflex.h diff --git a/u5/gf_dkx/src/lvgl_video_app.cpp b/u5/gf_dkx/src/lvgl_video_app.cpp new file mode 100644 index 0000000..e2246be --- /dev/null +++ b/u5/gf_dkx/src/lvgl_video_app.cpp @@ -0,0 +1,111 @@ + +#include +#include + +#if defined(CONFIG_OPENCV_LIB) +#include "opencv2/opencv.hpp" +#include "opencv_utils.hpp" +#include "cascades.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#define LOG_LEVEL CONFIG_LOG_DEFAULT_LEVEL +#include +LOG_MODULE_REGISTER(grinreflex_app); + +#include + +#include "config.hpp" +#include "gf/utils.hpp" +#include "gf/video.hpp" +#include "gf/lvgl_utils.hpp" +#include "grinreflex.h" + +static const struct device *video_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_camera)); +static const struct device *display_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_display)); +const struct device *jpeg_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_jpeg)); + +static uint8_t fullFrameBuffer[FULL_FRAME_WIDTH * FULL_FRAME_HEIGHT * LV_COLOR_FORMAT_GET_SIZE(FULL_FRAME_COLOR_FORMAT)]; +static uint8_t roiFrameBuffer[ROI_FRAME_WIDTH * ROI_FRAME_HEIGHT * LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8)]; +static uint8_t thumbnailFrameBuffer[THUMBNAIL_FRAME_WIDTH * THUMBNAIL_FRAME_HEIGHT * LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8)]; + +static lv_obj_t *screen_canvas; +static lv_obj_t *dummy_canvas; + +int init() +{ + if (!device_is_ready(display_dev)) { + LOG_ERR("Display device not ready, aborting test"); + return -ENODEV; + } + + if (!device_is_ready(video_dev)) { + LOG_ERR("Video device not ready, aborting test"); + return -ENODEV; + } + + if (!device_is_ready(jpeg_dev)) { + printf("%s JPEG device not ready", jpeg_dev->name); + return -ENODEV; + } + + Video::setup(); + + dummy_canvas = lv_canvas_create(NULL); + screen_canvas = lv_canvas_create(lv_screen_active()); + lv_canvas_fill_bg(screen_canvas, lv_color_hex3(0xccc), LV_OPA_COVER); + lv_obj_center(screen_canvas); + + display_blanking_off(display_dev); + + return 0; +} + +int loop() +{ + + struct video_buffer vbuf{}; + struct video_buffer *vbuf_ptr = &vbuf; + struct jpeg_out_prop jpeg_prop; + + vbuf_ptr->type = VIDEO_BUF_TYPE_OUTPUT; + + int err = video_dequeue(video_dev, &vbuf_ptr, K_FOREVER); + if (err) { + LOG_ERR("Unable to dequeue video buf"); + return 0; + } + + //jpeg_hw_decode(jpeg_dev, (uint8_t *)vbuf_ptr->buffer, vbuf_ptr->bytesused, jpeg_frame_buffer); + + jpeg_hw_poll(jpeg_dev, 0, &jpeg_prop); + + //printf("JPEG done w = %d, h = %d, color = %d chroma = %d\n", jpeg_prop.width, + // jpeg_prop.height, jpeg_prop.color_space, jpeg_prop.chroma); + + jpeg_color_convert_helper(jpeg_dev, &jpeg_prop, vbuf_ptr->buffer, fullFrameBuffer); + + cropFullFrameToRoi(dummy_canvas, fullFrameBuffer, roiFrameBuffer); + fitRoiFrameToThumbnail(screen_canvas, roiFrameBuffer, thumbnailFrameBuffer); + lv_task_handler(); + + err = video_enqueue(video_dev, vbuf_ptr); + if (err) { + LOG_ERR("Unable to requeue video buf"); + } + + k_msleep(50); + + return 0; +} \ No newline at end of file diff --git a/u5/gf_dkx/src/main.c b/u5/gf_dkx/src/main.c new file mode 100644 index 0000000..672166a --- /dev/null +++ b/u5/gf_dkx/src/main.c @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2024 Charles Dias + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include + +#define CONFIG_VIDEO_WIDTH 160 +#define CONFIG_VIDEO_HEIGHT 120 + +#include "grinreflex.h" + +LOG_MODULE_REGISTER(main, CONFIG_LOG_DEFAULT_LEVEL); + +#if !DT_HAS_CHOSEN(zephyr_camera) +#error No camera chosen in devicetree. Missing "--shield" or "--snippet video-sw-generator" flag? +#endif + +#if !DT_HAS_CHOSEN(zephyr_display) +#error No display chosen in devicetree. Missing "--shield" flag? +#endif + +int main(void) +{ + init(); + + while (1) + { + loop(); + } +} diff --git a/u5/app/src/grinreflex_v2.cpp b/u5/gf_dkx/src/opencv_fd_app.cpp similarity index 74% rename from u5/app/src/grinreflex_v2.cpp rename to u5/gf_dkx/src/opencv_fd_app.cpp index da69665..08fc73e 100644 --- a/u5/app/src/grinreflex_v2.cpp +++ b/u5/gf_dkx/src/opencv_fd_app.cpp @@ -4,7 +4,7 @@ #if defined(CONFIG_OPENCV_LIB) #include "opencv2/opencv.hpp" -#include "opencv_utils.hpp" +#include "gf/opencv_utils.hpp" #include "cascades.h" #endif @@ -24,19 +24,21 @@ #include LOG_MODULE_REGISTER(grinreflex_app); -#include "video.hpp" -#include "gfx_utils.hpp" +#include "gf/video.hpp" +#include "gf/lvgl_utils.hpp" #include -#include "grinreflex_config.hpp" -#include "grinreflex_utils.hpp" +#include "config.hpp" +#include "gf/utils.hpp" #include "grinreflex.h" static const struct device *video_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_camera)); static const struct device *display_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_display)); const struct device *jpeg_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_jpeg)); +#if DT_NODE_EXISTS(DT_ALIAS(led0)) static const struct gpio_dt_spec led = GPIO_DT_SPEC_GET(DT_ALIAS(led0), gpios); +#endif static uint8_t fullFrameBuffer[FULL_FRAME_WIDTH * FULL_FRAME_HEIGHT * LV_COLOR_FORMAT_GET_SIZE(FULL_FRAME_COLOR_FORMAT)]; static uint8_t roiFrameBuffer[ROI_FRAME_WIDTH * ROI_FRAME_HEIGHT * LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8)]; @@ -46,12 +48,13 @@ static lv_obj_t *screen_canvas; static lv_obj_t *dummy_canvas; static lv_obj_t *screen; -#if defined(CONFIG_OPENCV_LIB) static lv_obj_t *ROIRectFace; static lv_obj_t *ROIRectSmile; static cv::CascadeClassifier faceCascade; static cv::CascadeClassifier smileCascade; -#endif /* defined(CONFIG_OPENCV_LIB) */ +float constexpr smileROIGammaMax = 0.7f; +float constexpr smileROIGammaMin = 0.1f; +float smileROIGamma = smileROIGammaMin; int init() { @@ -70,6 +73,7 @@ int init() return -ENODEV; } +#if DT_NODE_EXISTS(DT_ALIAS(led0)) if (!gpio_is_ready_dt(&led)) { return -ENODEV; } @@ -77,8 +81,7 @@ int init() if (gpio_pin_configure_dt(&led, GPIO_OUTPUT_ACTIVE) < 0) { return -ENODEV; } - -#if defined(CONFIG_OPENCV_LIB) +#endif cv::setNumThreads(1); cv::setUseOptimized(false); @@ -92,8 +95,6 @@ int init() return -3; } -#endif /* defined(CONFIG_OPENCV_LIB) */ - Video::setup(); dummy_canvas = lv_canvas_create(NULL); @@ -105,17 +106,14 @@ int init() screen = lv_img_create(lv_scr_act()); -#if defined(CONFIG_OPENCV_LIB) ROIRectFace = allocLvROIRect(lv_scr_act(), 4); ROIRectSmile = allocLvROIRect(lv_scr_act(), 2); -#endif return 0; } int loop() { - struct video_buffer vbuf{}; struct video_buffer *vbuf_ptr = &vbuf; struct jpeg_out_prop jpeg_prop; @@ -137,21 +135,15 @@ int loop() jpeg_color_convert_helper(jpeg_dev, &jpeg_prop, vbuf_ptr->buffer, fullFrameBuffer); -#if !defined(CONFIG_OPENCV_LIB) + cropFullFrameToRoi(dummy_canvas, fullFrameBuffer, roiFrameBuffer, + lvgl::Size(FULL_FRAME_WIDTH, FULL_FRAME_HEIGHT), + lvgl::Size(ROI_FRAME_WIDTH, ROI_FRAME_HEIGHT), + FULL_FRAME_COLOR_FORMAT, LV_COLOR_FORMAT_L8, FULL_FRAME_FLIP_Y); - cropFullFrameToRoi(dummy_canvas, fullFrameBuffer, roiFrameBuffer); - fitRoiFrameToThumbnail(screen_canvas, roiFrameBuffer, thumbnailFrameBuffer); - lv_task_handler(); - - err = video_enqueue(video_dev, vbuf_ptr); - if (err) { - LOG_ERR("Unable to requeue video buf"); - return 0; - } -#else /* !defined(CONFIG_OPENCV_LIB) */ - - cropFullFrameToRoi(dummy_canvas, fullFrameBuffer, roiFrameBuffer); - fitRoiFrameToThumbnail(screen_canvas, roiFrameBuffer, thumbnailFrameBuffer); + fitRoiFrameToThumbnail(screen_canvas, roiFrameBuffer, thumbnailFrameBuffer, + lvgl::Size(ROI_FRAME_WIDTH, ROI_FRAME_HEIGHT), + lvgl::Size(THUMBNAIL_FRAME_WIDTH, THUMBNAIL_FRAME_HEIGHT), + LV_COLOR_FORMAT_L8, LV_COLOR_FORMAT_L8); lv_task_handler(); // Return buffer back, we don't need it anymore, thank you @@ -171,39 +163,49 @@ int loop() std::vector faces; std::vector smiles; - std::vector objects = detectFaceAndSmile( + detectFaceAndSmile( faceCascade, smileCascade, matGraySmall, matGrayFull, faceROIMax, faces, - smiles + smiles, + smileROIGamma ); + if (!faces.empty() && smiles.empty()) { + smileROIGamma += 0.1f; + if (smileROIGamma > smileROIGammaMax) { + smileROIGamma = smileROIGammaMin; + } + } + cv::Rect face, smile; if (faces.size()) { lv_obj_remove_flag(ROIRectFace, LV_OBJ_FLAG_HIDDEN); - auto object = faces[0]; - lv_obj_align_to(ROIRectFace, screen_canvas, LV_ALIGN_TOP_LEFT, object.x, object.y); - lv_obj_set_size(ROIRectFace, object.width, object.height); + face = faces[0]; + lv_obj_align_to(ROIRectFace, screen_canvas, LV_ALIGN_TOP_LEFT, face.x, face.y); + lv_obj_set_size(ROIRectFace, face.width, face.height); } else { lv_obj_add_flag(ROIRectFace, LV_OBJ_FLAG_HIDDEN); } if (smiles.size()) { - auto object = smiles[0]; - lv_obj_align_to(ROIRectSmile, screen_canvas, LV_ALIGN_TOP_LEFT, object.x, object.y); - lv_obj_set_size(ROIRectSmile, object.width, object.height); - - lv_obj_remove_flag(ROIRectSmile, LV_OBJ_FLAG_HIDDEN); - - gpio_pin_set_dt(&led, 0); + smile = smiles[0]; + if ((smile & face).area() != 0) { + lv_obj_align_to(ROIRectSmile, screen_canvas, LV_ALIGN_TOP_LEFT, smile.x, smile.y); + lv_obj_set_size(ROIRectSmile, smile.width, smile.height); + + lv_obj_remove_flag(ROIRectSmile, LV_OBJ_FLAG_HIDDEN); +#if DT_NODE_EXISTS(DT_ALIAS(led0)) + gpio_pin_set_dt(&led, 0); +#endif + } } else { lv_obj_add_flag(ROIRectSmile, LV_OBJ_FLAG_HIDDEN); } -#endif /* !defined(CONFIG_OPENCV_LIB) */ return 0; } \ No newline at end of file diff --git a/u5/uvc_gf_dk1/CMakeLists.txt b/u5/uvc_gf_dk1/CMakeLists.txt new file mode 100644 index 0000000..261c167 --- /dev/null +++ b/u5/uvc_gf_dk1/CMakeLists.txt @@ -0,0 +1,22 @@ +cmake_minimum_required(VERSION 3.13.1) + +set(EXTRA_CONF_FILE + ${CMAKE_CURRENT_SOURCE_DIR}/lvgl.conf; +) + +find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) + +project(uvc_gf_dk1) +include(${ZEPHYR_BASE}/samples/subsys/usb/common/common.cmake) + +target_include_directories(app PRIVATE + src +) + +target_sources(app PRIVATE + src/main.c +) + +if (NOT CONFIG_BOARD_GRINREFLEX_DK1) +message(FATAL "only CONFIG_BOARD_GRINREFLEX_DK1 is supported") +endif() \ No newline at end of file diff --git a/u5/uvc_gf_dk1/Kconfig b/u5/uvc_gf_dk1/Kconfig new file mode 100644 index 0000000..4c1c2f8 --- /dev/null +++ b/u5/uvc_gf_dk1/Kconfig @@ -0,0 +1,8 @@ + + +module = grinreflex +module-str = grinreflex + +source "subsys/logging/Kconfig.template.log_config" +source "samples/subsys/usb/common/Kconfig.sample_usbd" +source "Kconfig.zephyr" \ No newline at end of file diff --git a/u5/uvc_gf_dk1/boards/grinreflex_dk1.conf b/u5/uvc_gf_dk1/boards/grinreflex_dk1.conf new file mode 100644 index 0000000..e69de29 diff --git a/u5/uvc_gf_dk1/boards/grinreflex_dk1.overlay b/u5/uvc_gf_dk1/boards/grinreflex_dk1.overlay new file mode 100644 index 0000000..dafcdc0 --- /dev/null +++ b/u5/uvc_gf_dk1/boards/grinreflex_dk1.overlay @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2024, Eve Redero + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include + +/ { + chosen: chosen { + zephyr,spi1 = &spi1; + zephyr,usart2 = &usart2; + }; + uvc: uvc { + compatible = "zephyr,uvc-device"; + }; +}; + +&gpdma1 { + status = "okay"; +}; \ No newline at end of file diff --git a/u5/uvc_gf_dk1/lvgl.conf b/u5/uvc_gf_dk1/lvgl.conf new file mode 100644 index 0000000..c6a38ad --- /dev/null +++ b/u5/uvc_gf_dk1/lvgl.conf @@ -0,0 +1,24 @@ + + +CONFIG_LV_Z_MEM_POOL_SIZE=16384 +CONFIG_LV_Z_SHELL=n + +CONFIG_LVGL=y +CONFIG_LV_USE_LOG=y +CONFIG_LV_USE_LABEL=y +CONFIG_LV_USE_ARC=y +CONFIG_LV_USE_IMAGE=y +#CONFIG_LV_USE_MONKEY=y +CONFIG_LV_FONT_MONTSERRAT_14=y +#CONFIG_LV_Z_POINTER_INPUT=y + +CONFIG_LV_CACHE_DEF_SIZE=4096 +CONFIG_LV_IMAGE_HEADER_CACHE_DEF_CNT=1 +#CONFIG_LV_USE_TJPGD=y +#CONFIG_LV_USE_FS_MEMFS=y +#CONFIG_LV_USE_FS=y +CONFIG_LV_USE_BUILTIN_MALLOC=n + +#CONFIG_LV_USE_PERF_MONITOR=y +#CONFIG_LV_USE_DROPDOWN=y +#CONFIG_LV_USE_SYSMON=y \ No newline at end of file diff --git a/u5/uvc_gf_dk1/prj.conf b/u5/uvc_gf_dk1/prj.conf new file mode 100644 index 0000000..d4b01f6 --- /dev/null +++ b/u5/uvc_gf_dk1/prj.conf @@ -0,0 +1,24 @@ + + +CONFIG_LOG=y +CONFIG_POLL=y +CONFIG_SAMPLE_USBD_PID=0x0011 +CONFIG_SAMPLE_USBD_PRODUCT="UVC sample" +CONFIG_UDC_BUF_POOL_SIZE=2048 +CONFIG_UDC_DRIVER_LOG_LEVEL_WRN=y +CONFIG_USBD_LOG_LEVEL_WRN=y +CONFIG_USBD_VIDEO_CLASS=y +CONFIG_USBD_VIDEO_LOG_LEVEL_WRN=y +CONFIG_USB_DEVICE_STACK_NEXT=y + +CONFIG_GRINREFLEX_JPEG_VIDEO=n +CONFIG_GRINREFLEX_VIDEO_RGB565_SWAP_BYTES=y +CONFIG_GRINREFLEX_VIDEO_WIDTH=640 +CONFIG_GRINREFLEX_VIDEO_HEIGHT=480 + +CONFIG_VIDEO_BUFFER_POOL_NUM_MAX=2 +CONFIG_VIDEO_BUFFER_POOL_SZ_MAX=1228800 +CONFIG_VIDEO_LOG_LEVEL_WRN=y + +CONFIG_SHELL=y +CONFIG_VIDEO_SHELL=y \ No newline at end of file diff --git a/u5/uvc_gf_dk1/src/main.c b/u5/uvc_gf_dk1/src/main.c new file mode 100644 index 0000000..fb7caa6 --- /dev/null +++ b/u5/uvc_gf_dk1/src/main.c @@ -0,0 +1,172 @@ + +/* + * Copyright (c) 2025 tinyVision.ai Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include + +#include + +#include +#include +#include +#include +#include + +LOG_MODULE_REGISTER(uvc_sample, LOG_LEVEL_INF); + +const struct device *const uvc_dev = DEVICE_DT_GET(DT_NODELABEL(uvc)); +const struct device *const video_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_camera)); + +int main(void) +{ + struct usbd_context *sample_usbd; + struct video_buffer *vbuf; + struct video_format fmt = {0}; + struct video_caps caps; + struct k_poll_signal sig; + struct k_poll_event evt[1]; + k_timeout_t timeout = K_FOREVER; + size_t bsize; + int ret; + + if (!device_is_ready(video_dev)) { + LOG_ERR("video source %s failed to initialize", video_dev->name); + return -ENODEV; + } + + caps.type = VIDEO_BUF_TYPE_OUTPUT; + + if (video_get_caps(video_dev, &caps)) { + LOG_ERR("Unable to retrieve video capabilities"); + return 0; + } + + /* Must be done before initializing USB */ + uvc_set_video_dev(uvc_dev, video_dev); + + sample_usbd = sample_usbd_init_device(NULL); + if (sample_usbd == NULL) { + return -ENODEV; + } + + ret = usbd_enable(sample_usbd); + if (ret != 0) { + return ret; + } + + LOG_INF("Waiting the host to select the video format"); + + /* Get the video format once it is selected by the host */ + while (true) { + fmt.type = VIDEO_BUF_TYPE_INPUT; + + ret = video_get_format(uvc_dev, &fmt); + if (ret == 0) { + break; + } + if (ret != -EAGAIN) { + LOG_ERR("Failed to get the video format"); + return ret; + } + + k_sleep(K_MSEC(10)); + } + + LOG_INF("The host selected format '%s' %ux%u, preparing %u buffers of %u bytes", + VIDEO_FOURCC_TO_STR(fmt.pixelformat), fmt.width, fmt.height, + CONFIG_VIDEO_BUFFER_POOL_NUM_MAX, fmt.pitch * fmt.height); + + /* Size to allocate for each buffer */ + if (caps.min_line_count == LINE_COUNT_HEIGHT) { + bsize = fmt.pitch * fmt.height; + } else { + bsize = fmt.pitch * caps.min_line_count; + } + + for (int i = 0; i < CONFIG_VIDEO_BUFFER_POOL_NUM_MAX; i++) { + vbuf = video_buffer_alloc(bsize, K_NO_WAIT); + if (vbuf == NULL) { + LOG_ERR("Could not allocate the video buffer"); + return -ENOMEM; + } + + vbuf->type = VIDEO_BUF_TYPE_OUTPUT; + + ret = video_enqueue(video_dev, vbuf); + if (ret != 0) { + LOG_ERR("Could not enqueue video buffer"); + return ret; + } + } + + LOG_DBG("Preparing signaling for %s input/output", video_dev->name); + + k_poll_signal_init(&sig); + k_poll_event_init(&evt[0], K_POLL_TYPE_SIGNAL, K_POLL_MODE_NOTIFY_ONLY, &sig); + + ret = video_set_signal(video_dev, &sig); + if (ret != 0) { + LOG_WRN("Failed to setup the signal on %s output endpoint", video_dev->name); + timeout = K_MSEC(1); + } + + ret = video_set_signal(uvc_dev, &sig); + if (ret != 0) { + LOG_ERR("Failed to setup the signal on %s input endpoint", uvc_dev->name); + return ret; + } + + LOG_INF("Starting the video transfer"); + + ret = video_stream_start(video_dev, VIDEO_BUF_TYPE_OUTPUT); + if (ret != 0) { + LOG_ERR("Failed to start %s", video_dev->name); + return ret; + } + + while (true) { + ret = k_poll(evt, ARRAY_SIZE(evt), timeout); + if (ret != 0 && ret != -EAGAIN) { + LOG_ERR("Poll exited with status %d", ret); + return ret; + } + + vbuf = &(struct video_buffer){.type = VIDEO_BUF_TYPE_OUTPUT}; + + if (video_dequeue(video_dev, &vbuf, K_NO_WAIT) == 0) { + LOG_DBG("Dequeued %p from %s, enqueueing to %s", + (void *)vbuf, video_dev->name, uvc_dev->name); + + vbuf->type = VIDEO_BUF_TYPE_INPUT; + + ret = video_enqueue(uvc_dev, vbuf); + if (ret != 0) { + LOG_ERR("Could not enqueue video buffer to %s", uvc_dev->name); + return ret; + } + } + + vbuf = &(struct video_buffer){.type = VIDEO_BUF_TYPE_INPUT}; + + if (video_dequeue(uvc_dev, &vbuf, K_NO_WAIT) == 0) { + LOG_DBG("Dequeued %p from %s, enqueueing to %s", + (void *)vbuf, uvc_dev->name, video_dev->name); + + vbuf->type = VIDEO_BUF_TYPE_OUTPUT; + + ret = video_enqueue(video_dev, vbuf); + if (ret != 0) { + LOG_ERR("Could not enqueue video buffer to %s", video_dev->name); + return ret; + } + } + + k_poll_signal_reset(&sig); + } + + return 0; +} From 4883a0ced9a696ee1c68b00fa1e959165e4519c1 Mon Sep 17 00:00:00 2001 From: Roman Pustobaiev Date: Fri, 17 Oct 2025 14:19:45 +0200 Subject: [PATCH 05/18] More improvements Adding opencv pipeline to preprocess mouth area Found best possible configuration for smile detection Code formatting, cleanup --- host/CMakeLists.txt | 2 + host/Src/main.cpp | 123 +++++-- u5/common/include/gf/lvgl_utils.hpp | 7 +- u5/common/include/gf/utils.hpp | 14 +- u5/common/include/gf/video.hpp | 5 +- u5/common/opencv/CMakeLists.txt | 1 + .../opencv/include/gf/opencv_pipeline.hpp | 106 ++++++ u5/common/opencv/include/gf/opencv_utils.hpp | 115 ++++-- u5/common/opencv/src/opencv_pipeline.cpp | 102 ++++++ u5/common/opencv/src/opencv_utils.cpp | 333 +++++++++++------- u5/common/src/lvgl_utils.cpp | 4 +- u5/common/src/utils.cpp | 142 ++++---- u5/common/src/video.cpp | 319 +++++++++-------- u5/gf_dkx/prj.conf | 12 +- u5/gf_dkx/src/config.hpp | 3 +- u5/gf_dkx/src/lvgl_video_app.cpp | 113 +++--- u5/gf_dkx/src/main.c | 18 +- u5/gf_dkx/src/opencv_fd_app.cpp | 278 ++++++++------- 18 files changed, 1045 insertions(+), 652 deletions(-) create mode 100644 u5/common/opencv/include/gf/opencv_pipeline.hpp create mode 100644 u5/common/opencv/src/opencv_pipeline.cpp diff --git a/host/CMakeLists.txt b/host/CMakeLists.txt index e10eaab..cd89b70 100644 --- a/host/CMakeLists.txt +++ b/host/CMakeLists.txt @@ -21,7 +21,9 @@ set(OPENCV_UTILS_DIR_PATH "../u5/common/opencv") set(HOST_APP_SRCS Src/main.cpp ${OPENCV_UTILS_DIR_PATH}/src/opencv_utils.cpp + ${OPENCV_UTILS_DIR_PATH}/src/opencv_pipeline.cpp ${OPENCV_UTILS_DIR_PATH}/include/gf/opencv_utils.hpp + ${OPENCV_UTILS_DIR_PATH}/include/gf/opencv_pipeline.hpp ) # Add executable diff --git a/host/Src/main.cpp b/host/Src/main.cpp index ed8b64e..a526080 100644 --- a/host/Src/main.cpp +++ b/host/Src/main.cpp @@ -11,8 +11,9 @@ cv::String smile_cascade_name = haarPath + "haarcascades/haarcascade_smile.xml"; cv::CascadeClassifier face_cascade; cv::CascadeClassifier smile_cascade; cv::String window_name = "Capture - Face detection"; -cv::String window_name_2 = "Capture - Smile detection"; +cv::String window_name_smile = "Smile"; +using namespace gf_cv; bool checkFrameSizeSupported(cv::Mat &gray) { if (gray.cols == 640 && gray.rows == 480) { @@ -48,12 +49,12 @@ int main(int, char**) { return -1; }; - cv::namedWindow(window_name, 2); + cv::namedWindow(window_name, cv::WND_PROP_ASPECT_RATIO); // display the frame until you press a key std::cout << "Start capturing" << std::endl; - float gamma = 0.1; - std::map gammaScore; + ROIKernelParamsSweep smileROIKernelParams; + std::map smileROIKernelParamsScore; while (camera.read(frameOriginal)) { if (frameOriginal.empty()) @@ -86,56 +87,110 @@ int main(int, char**) { cv::Rect faceROIMax{}; faceROIMax.width = 64; - faceROIMax.height = 64; + faceROIMax.height = 40; cv::resize(frameCroppedGray, frameScreenGray, cv::Size(80, 80), 0.0f, 0.0f, cv::INTER_NEAREST ); std::vector faces, smiles; - cv::Mat smilePostProc = detectFaceAndSmile( + smileROIKernelParams.clahe.clipLimit = Range{4.0, 4.0, 1.0}; + smileROIKernelParams.clahe.tileSize = Range{4, 4, 1}; + smileROIKernelParams.biFilter.d = Range{3, 3, 1}; + smileROIKernelParams.biFilter.sigmaColor = Range{10.0, 10.0, 1.0}; + smileROIKernelParams.biFilter.sigmaSpace = Range{50.0, 50.0, 1.0}; + smileROIKernelParams.gamma.gamma = Range{0.44, 0.50, 0.02}; + smileROIKernelParams.blur.sigmaX = Range{0.82, 0.87, 0.01}; + + auto rects = detectFaceAndSmile( face_cascade, smile_cascade, frameScreenGray, frameCroppedGray, faceROIMax, - faces, - smiles, - gamma + smileROIKernelParams ); - if (!faces.empty() && !smiles.empty()) { - if (gammaScore.count(gamma)) { - gammaScore[gamma] = gammaScore[gamma] + 1; - } else { - gammaScore[gamma] = 1; - } + for (const auto &rect : rects) { + rectangle(frameScreenGray, rect, cv::Scalar(0, 0, 255), 1, 1, 0); } - if (!faces.empty() && smiles.empty()) { - gamma += 0.1; - if (gamma >= 1.0) { - gamma = 0.1; - } - } + cv::imshow(window_name, frameScreenGray); - for (const auto &ROI : faces) { - rectangle(frameScreenGray, ROI, cv::Scalar(0, 0, 255), 1, 1, 0); - } - for (const auto &ROI : smiles) { - rectangle(frameScreenGray, ROI, cv::Scalar(0, 0, 255), 1, 1, 0); - } + int c = cv::waitKey(1); + if ((char)c == 27) { break; } + + continue; - if (smilePostProc.cols && smilePostProc.rows) { - cv::imshow(window_name_2, smilePostProc); +#if 0 + if (faces.empty()) { + continue; } - cv::imshow(window_name, frameScreenGray); - int c = cv::waitKey(100); - if ((char)c == 27) { break; } + const double scaleFactor = 1.1; + const int minNeighbors = 3; + const int flags = cv::CASCADE_FIND_BIGGEST_OBJECT | cv::CASCADE_DO_ROUGH_SEARCH; + + std::vector claheClipLimit_V({4.0}); + std::vector claheTileSize_V({4}); + std::vector biFilterD_V({3}); + std::vector biFilterSigmaColor_V({20, 22, 24, 26, 28, 20}); + std::vector biFilterSigmaSpace_V({50}); + std::vector gammaGamma_V({0.45, 0.47, 0.49, 0.50, 0.52, 0.54, 0.56, 0.58, 0.60, 0.62, 0.65}); + std::vector blurSigmaX_V({0.85, 0.86, 0.87, 0.88, 0.89, 0.90, 0.91, 0.92, 0.93, 0.94, 0.95}); + + for (auto claheClipLimit : claheClipLimit_V) { + for (auto claheTileSize : claheTileSize_V) { + for (auto biFilterD : biFilterD_V) { + for (auto biFilterSigmaColor : biFilterSigmaColor_V) { + for (auto biFilterSigmaSpace : biFilterSigmaSpace_V) { + for (auto gammaGamma : gammaGamma_V) { + for (auto blurSigmaX : blurSigmaX_V) { + smileROIKernelParams.clahe.clipLimit = claheClipLimit; + smileROIKernelParams.clahe.tileSize = claheTileSize; + smileROIKernelParams.biFilter.d = biFilterD; + smileROIKernelParams.biFilter.sigmaColor = biFilterSigmaColor; + smileROIKernelParams.biFilter.sigmaSpace = biFilterSigmaSpace; + smileROIKernelParams.gamma.gamma = gammaGamma; + smileROIKernelParams.blur.sigmaX = blurSigmaX; + + smileROIMat = preprocessSmileROI(smileROIMat, smileROIKernelParams); + std::vector smiles; + smile_cascade.detectMultiScale(smileROIMat, smiles, scaleFactor, minNeighbors, + flags); + + if (!smiles.empty()) { + std::string key; + key += "clahe.clipLimit=" + std::to_string(smileROIKernelParams.clahe.clipLimit) + ":"; + key += "clahe.tileSize=" + std::to_string(smileROIKernelParams.clahe.tileSize) + ":"; + key += "biFilter.d=" + std::to_string(smileROIKernelParams.biFilter.d) + ":"; + key += "biFilter.sigmaColor=" + std::to_string(smileROIKernelParams.biFilter.sigmaColor) + ":"; + key += "biFilter.sigmaSpace=" + std::to_string(smileROIKernelParams.biFilter.sigmaSpace) + ":"; + key += "gamma.gamma=" + std::to_string(smileROIKernelParams.gamma.gamma) + ":"; + key += "blur.sigmaX=" + std::to_string(smileROIKernelParams.blur.sigmaX); + + auto it = smileROIKernelParamsScore.try_emplace(key, 1); + if (!it.second) { + auto &node = *it.first; + node.second += 1; + } + + //cv::imshow(window_name_2, smileROIMat); + //cv::waitKey(100); + break; + } + + } + } + } + } + } + } + } +#endif } - for (auto & [gamma, score] : gammaScore) { - std::cout << "gamma = " << gamma << ", score = " << score << std::endl; + for (auto & [key, score] : smileROIKernelParamsScore) { + std::cout << "key = " << key << ", score = " << score << std::endl; } return 0; diff --git a/u5/common/include/gf/lvgl_utils.hpp b/u5/common/include/gf/lvgl_utils.hpp index e5cbdec..d61f79b 100644 --- a/u5/common/include/gf/lvgl_utils.hpp +++ b/u5/common/include/gf/lvgl_utils.hpp @@ -9,7 +9,7 @@ namespace lvgl { struct Size { - Size (uint32_t w, uint32_t h) : width(w), height(h) {} + Size(uint32_t w, uint32_t h) : width(w), height(h) {} uint32_t width, height; }; @@ -24,6 +24,7 @@ struct Buffer { * @brief Fit source buffer onto destination * */ -void fit(lv_obj_t *canvas, const Buffer &src, const Buffer &dst, bool flip_y = false); +void fit(lv_obj_t *canvas, const Buffer &src, const Buffer &dst, + bool flip_y = false); -} // namespace Gfx +} // namespace lvgl diff --git a/u5/common/include/gf/utils.hpp b/u5/common/include/gf/utils.hpp index a76a3f7..0c677c6 100644 --- a/u5/common/include/gf/utils.hpp +++ b/u5/common/include/gf/utils.hpp @@ -4,9 +4,11 @@ #include -lv_obj_t *allocLvROIRect(lv_obj_t *parent, uint32_t borderWidth); - -void cropFullFrameToRoi(lv_obj_t *parent, uint8_t *fullFb, uint8_t *roiFb, lvgl::Size fullSz, - lvgl::Size roiSz, uint32_t fullCf, uint32_t roiCf, bool flip_y = false); -void fitRoiFrameToThumbnail(lv_obj_t *parent, uint8_t *roiFb, uint8_t *tFb, lvgl::Size roiSz, - lvgl::Size tSz, uint32_t roiCf, uint32_t tCf); +lv_obj_t *allocLvROIRect(lv_obj_t *parent, uint32_t borderWidth, lv_color_t c); + +void cropFullFrameToRoi(lv_obj_t *parent, uint8_t *fullFb, uint8_t *roiFb, + lvgl::Size fullSz, lvgl::Size roiSz, uint32_t fullCf, + uint32_t roiCf, bool flip_y = false); +void fitRoiFrameToThumbnail(lv_obj_t *parent, uint8_t *roiFb, uint8_t *tFb, + lvgl::Size roiSz, lvgl::Size tSz, uint32_t roiCf, + uint32_t tCf); diff --git a/u5/common/include/gf/video.hpp b/u5/common/include/gf/video.hpp index 001ea03..73ed23e 100644 --- a/u5/common/include/gf/video.hpp +++ b/u5/common/include/gf/video.hpp @@ -1,11 +1,10 @@ #pragma once -#include #include +#include -namespace Video -{ +namespace Video { void setup(); diff --git a/u5/common/opencv/CMakeLists.txt b/u5/common/opencv/CMakeLists.txt index 6ca12eb..88322b6 100644 --- a/u5/common/opencv/CMakeLists.txt +++ b/u5/common/opencv/CMakeLists.txt @@ -7,6 +7,7 @@ zephyr_compile_options(-DOPENCV_FLANN_USE_STD_RAND) target_sources(app PRIVATE src/opencv_utils.cpp + src/opencv_pipeline.cpp ) set(OPENCV_LBP_CASCADES ${OPENCV_INSTALL_DIR}/share/opencv4/lbpcascades) diff --git a/u5/common/opencv/include/gf/opencv_pipeline.hpp b/u5/common/opencv/include/gf/opencv_pipeline.hpp new file mode 100644 index 0000000..9e3892a --- /dev/null +++ b/u5/common/opencv/include/gf/opencv_pipeline.hpp @@ -0,0 +1,106 @@ + +#include "opencv_utils.hpp" + +#pragma once + +namespace gf_cv { + +class Stage { +public: + explicit Stage() = default; + virtual ~Stage() = default; + + struct sentinel {}; + + struct iterator { + Stage &stage; + + cv::Mat operator*() { return stage.get(); } + iterator &operator++() { + stage.next(); + return *this; + } + bool operator!=(sentinel) { return stage.hasNext(); } + }; + + iterator begin() { return {*this}; } + sentinel end() const { return {}; } + + virtual cv::Mat get() = 0; + virtual void next() = 0; + virtual bool hasNext() = 0; +}; + +class ClaheStage : public Stage { +public: + explicit ClaheStage(cv::Mat &in, Range &clipLimit, + Range &tileSize); + ~ClaheStage() = default; + + cv::Mat get() override; + void next() override; + bool hasNext() override; + +private: + cv::Mat ∈ + Range &clipLimit; + Range &tileSize; + + Range::iterator clipLimitIt; + Range::iterator tileSizeIt; +}; + +class BilateralStage : public Stage { +public: + explicit BilateralStage(cv::Mat &in, Range &d, Range &sigmaColor, + Range &sigmaSpace); + ~BilateralStage() = default; + + cv::Mat get() override; + void next() override; + bool hasNext() override; + +private: + cv::Mat ∈ + Range &d; + Range &sigmaColor; + Range &sigmaSpace; + + Range::iterator dIt; + Range::iterator sigmaColorIt; + Range::iterator sigmaSpaceIt; +}; + +class GammaStage : public Stage { +public: + explicit GammaStage(cv::Mat &in, Range &gamma); + ~GammaStage() = default; + + cv::Mat get() override; + void next() override; + bool hasNext() override; + +private: + cv::Mat ∈ + Range γ + + Range::iterator gammaIt; +}; + +class BlurStage : public Stage { +public: + explicit BlurStage(cv::Mat &in, Range &sigmaX); + ~BlurStage() = default; + + cv::Mat get() override; + void next() override; + bool hasNext() override; + +private: + cv::Mat ∈ + Range &sigmaX; + + Range::iterator sigmaXIt; +}; + +} // namespace gf_cv \ No newline at end of file diff --git a/u5/common/opencv/include/gf/opencv_utils.hpp b/u5/common/opencv/include/gf/opencv_utils.hpp index 7656530..9df19a0 100644 --- a/u5/common/opencv/include/gf/opencv_utils.hpp +++ b/u5/common/opencv/include/gf/opencv_utils.hpp @@ -1,45 +1,88 @@ #pragma once +#include + #include "opencv2/opencv.hpp" -#define OCV_SC_FACE_DETECT 0 -#define OCV_SC_QRCODE 1 -#if defined(CONFIG_QUIRC_LIB) -#define OCV_SC_QRCODE_QUIRC 2 -#endif +namespace gf_cv { bool loadCascade(cv::CascadeClassifier &cascade, const char *path); bool loadCascade(cv::CascadeClassifier &cascade, cv::FileStorage &fs); -bool loadCascade(cv::CascadeClassifier &cascade, const uint8_t *buffer, size_t len); +bool loadCascade(cv::CascadeClassifier &cascade, const uint8_t *buffer, + size_t len); + +template struct Range { + static_assert(std::is_arithmetic::value, + "struct Range: T must be arithmetic"); + + T start, stop, step; + + struct sentinel {}; + + struct iterator { + T value, stop, step; + + T operator*() const { return value; }; + iterator &operator++() { + value += step; + return *this; + } + bool operator!=(sentinel) { return value <= stop; } + bool operator==(sentinel) { return value > stop; } + }; + + iterator begin() const { + assert(step > T(0) && "Range: step must be > 0"); + return {start, stop, step}; + } + + sentinel end() const { return {}; } +}; + +struct ROIKernelParamsSweep { + struct Clahe { + Range clipLimit; + Range tileSize; + } clahe; + struct BiFilter { + Range d; + Range sigmaColor; + Range sigmaSpace; + } biFilter; + struct Gamma { + Range gamma; + } gamma; + struct Blur { + Range sigmaX; + } blur; +}; + /** - * @brief - * + * @brief + * * @param faceCascade * @param smileCascade - * @param thumbnailFrame frame to detect face in, thumbnail, small enough to save memory - * @param fullFrame frame to detect face in, roi is cropped from to have finer frained area - * @param faceROIMax maximum faceROI to be resized to if detected ROI is largen than this - * @param faces vector with faces rects - * @param smiles vector with sniles rects - * @return cv::Mat smile ROI ever considered during run + * @param thumbnailFrame frame to detect face in, thumbnail, small enough to + * save memory + * @param fullFrame frame to detect face in, roi is cropped from to have finer + * grained area + * @param faceROIMax maximum faceROI to be resized to if detected ROI is larger + * than this + * @param roIKernelParamsSweep + * @return vector of cv::Rects for detected objects */ -cv::Mat detectFaceAndSmile( - cv::CascadeClassifier &faceCascade, - cv::CascadeClassifier &smileCascade, - cv::Mat &thumbnailFrame, - cv::Mat &fullFrame, - cv::Rect &faceROIMax, - std::vector &faces, - std::vector &smiles, - float mouthGamma -); - -cv::Mat cv_preprocessForQR(const cv::Mat& bgr); +std::vector +detectFaceAndSmile(cv::CascadeClassifier &faceCascade, + cv::CascadeClassifier &smileCascade, cv::Mat &thumbnailFrame, + cv::Mat &fullFrame, cv::Rect &faceROIMax, + ROIKernelParamsSweep &roIKernelParamsSweep); + +cv::Mat cv_preprocessForQR(const cv::Mat &bgr); /** * @brief Remaps ROI that was defined withing inFrame to the outFrame - * + * * @param ROI original ROI rect * @param inFrame input frame * @param outFrame output frame @@ -49,13 +92,15 @@ cv::Rect remapROI(cv::Rect &ROI, cv::Mat &inFrame, cv::Mat &outFrame); /** * @brief Translate and scale ROI using homogenous transformations - * - * @param ROI - * @param sx - * @param sy - * @param tx - * @param ty - * @return cv::Rect + * + * @param ROI + * @param sx + * @param sy + * @param tx + * @param ty + * @return cv::Rect */ -cv::Rect translateScaleROI(cv::Rect &ROI, float sx, float sy, float tx, float ty); +cv::Rect translateScaleROI(cv::Rect &ROI, float sx, float sy, float tx, + float ty); +} /* namespace gf_cv */ \ No newline at end of file diff --git a/u5/common/opencv/src/opencv_pipeline.cpp b/u5/common/opencv/src/opencv_pipeline.cpp new file mode 100644 index 0000000..0876df0 --- /dev/null +++ b/u5/common/opencv/src/opencv_pipeline.cpp @@ -0,0 +1,102 @@ + +#include "gf/opencv_pipeline.hpp" + +namespace gf_cv { + +ClaheStage::ClaheStage(cv::Mat &in, Range &clipLimit, + Range &tileSize) + : in(in), clipLimit(clipLimit), tileSize(tileSize) { + + clipLimitIt = clipLimit.begin(); + tileSizeIt = tileSize.begin(); +} + +cv::Mat ClaheStage::get() { + cv::Ptr clahe = + cv::createCLAHE(*clipLimitIt, cv::Size(*tileSizeIt, *tileSizeIt)); + cv::Mat out; + clahe->apply(in, out); + return out; +} + +void ClaheStage::next() { + clipLimitIt = ++clipLimitIt; + if (clipLimitIt == clipLimit.end()) { + clipLimitIt = clipLimit.begin(); + tileSizeIt = ++tileSizeIt; + } +} + +bool ClaheStage::hasNext() { + return (clipLimitIt != clipLimit.end()) && (tileSizeIt != tileSize.end()); +} + +BilateralStage::BilateralStage(cv::Mat &in, Range &d, + Range &sigmaColor, + Range &sigmaSpace) + : in(in), d(d), sigmaColor(sigmaColor), sigmaSpace(sigmaSpace) { + + dIt = d.begin(); + sigmaColorIt = sigmaColor.begin(); + sigmaSpaceIt = sigmaSpace.begin(); +} + +cv::Mat BilateralStage::get() { + cv::Mat out; + cv::bilateralFilter(in, out, *dIt, *sigmaColorIt, *sigmaSpaceIt); + return out; +} + +void BilateralStage::next() { + dIt = ++dIt; + if (dIt == d.end()) { + dIt = d.begin(); + sigmaColorIt = ++sigmaColorIt; + } + if (sigmaColorIt == sigmaColor.end()) { + sigmaColorIt = sigmaColor.begin(); + sigmaSpaceIt = ++sigmaSpaceIt; + } +} + +bool BilateralStage::hasNext() { + return (dIt != d.end()) && (sigmaColorIt != sigmaColor.end()) && + (sigmaSpaceIt != sigmaSpace.end()); +} + +GammaStage::GammaStage(cv::Mat &in, Range &gamma) + : in(in), gamma(gamma) { + gammaIt = gamma.begin(); +} + +cv::Mat GammaStage::get() { + cv::Mat out = in.clone(); + out.convertTo(out, CV_32FC1, 1.0 / 255.0); + cv::pow(out, *gammaIt, out); + out.convertTo(out, CV_8UC1, 255.0); + return out; +} + +void GammaStage::next() { gammaIt = ++gammaIt; } + +bool GammaStage::hasNext() { return (gammaIt != gamma.end()); } + +BlurStage::BlurStage(cv::Mat &in, Range &sigmaX) + : in(in), sigmaX(sigmaX) { + sigmaXIt = sigmaX.begin(); +} + +cv::Mat BlurStage::get() { + cv::Mat blurFrame; + cv::Mat out = in.clone(); + cv::GaussianBlur(out, blurFrame, cv::Size(0, 0), *sigmaXIt); + // cv::Laplacian(bfROI, blurImg, -1, 1, 1, 0, cv::BORDER_DEFAULT); + cv::addWeighted(out, 1.5, blurFrame, -0.5, 0, out); + return out; +} + +void BlurStage::next() { sigmaXIt = ++sigmaXIt; } + +bool BlurStage::hasNext() { return (sigmaXIt != sigmaX.end()); } + +} // namespace gf_cv \ No newline at end of file diff --git a/u5/common/opencv/src/opencv_utils.cpp b/u5/common/opencv/src/opencv_utils.cpp index 044134b..b258622 100644 --- a/u5/common/opencv/src/opencv_utils.cpp +++ b/u5/common/opencv/src/opencv_utils.cpp @@ -7,189 +7,254 @@ #include #include +#include "gf/opencv_pipeline.hpp" #include "gf/opencv_utils.hpp" +namespace gf_cv { + bool loadCascade(cv::CascadeClassifier &cascade, const char *path) { - if (!cascade.load(path)) { - std::cerr << "Cannot load cascade: " << path << "\n"; - return false; - } - return true; + if (!cascade.load(path)) { + std::cerr << "Cannot load cascade: " << path << "\n"; + return false; + } + return true; } bool loadCascade(cv::CascadeClassifier &cascade, cv::FileStorage &fs) { - if (!cascade.read(fs.getFirstTopLevelNode())) { - std::cerr << "Cannot load cascade\n"; - return false; - } - return true; + if (!cascade.read(fs.getFirstTopLevelNode())) { + std::cerr << "Cannot load cascade\n"; + return false; + } + return true; } -bool loadCascade(cv::CascadeClassifier &cascade, const uint8_t *buffer, size_t len) { - cv::FileStorage fs(std::string((const char *)buffer, len), cv::FileStorage::READ | cv::FileStorage::MEMORY); - return loadCascade(cascade, fs); +bool loadCascade(cv::CascadeClassifier &cascade, const uint8_t *buffer, + size_t len) { + cv::FileStorage fs(std::string((const char *)buffer, len), + cv::FileStorage::READ | cv::FileStorage::MEMORY); + return loadCascade(cascade, fs); } -void detectObjects(cv::CascadeClassifier &cascade, cv::Mat &gray, std::vector &objects) { - const double scaleFactor = 1.1; - const int minNeighbors = 3; - const int flags = 0; +void detectObjects(cv::CascadeClassifier &cascade, cv::Mat &gray, + std::vector &objects) { + const double scaleFactor = 1.1; + const int minNeighbors = 3; + const int flags = 0; - cascade.detectMultiScale(gray, objects, scaleFactor, minNeighbors, - flags); + cascade.detectMultiScale(gray, objects, scaleFactor, minNeighbors, flags); } -void detectFaces(cv::CascadeClassifier &cascade, cv::Mat &gray, std::vector &faces) { - detectObjects(cascade, gray, faces); +void detectFaces(cv::CascadeClassifier &cascade, cv::Mat &gray, + std::vector &faces) { + detectObjects(cascade, gray, faces); } -cv::Mat cv_preprocessForQR(const cv::Mat& bgr) { - cv::Mat gray, up, eq, bin; - if (bgr.channels() == 3) cv::cvtColor(bgr, gray, cv::COLOR_BGR2GRAY); - else gray = bgr; - - cv::equalizeHist(gray, eq); - cv::adaptiveThreshold(eq, bin, 255, cv::ADAPTIVE_THRESH_MEAN_C, cv::THRESH_BINARY, 31, 5); - return bin; +cv::Mat cv_preprocessForQR(const cv::Mat &bgr) { + cv::Mat gray, up, eq, bin; + if (bgr.channels() == 3) + cv::cvtColor(bgr, gray, cv::COLOR_BGR2GRAY); + else + gray = bgr; + + cv::equalizeHist(gray, eq); + cv::adaptiveThreshold(eq, bin, 255, cv::ADAPTIVE_THRESH_MEAN_C, + cv::THRESH_BINARY, 31, 5); + return bin; } -static cv::Mat preprocessSmileROI(const cv::Mat& grayROI, float gamma) { - // 1) CLAHE for local contrast (great in dim light) - cv::Ptr clahe = cv::createCLAHE(5.0, cv::Size(4,4)); - cv::Mat claheROI, bfROI; - clahe->apply(grayROI, claheROI); - - // 2) Denoise but keep edges - cv::bilateralFilter(claheROI, bfROI, 3, 20, 50); +static bool detectSmile(cv::CascadeClassifier &cascade, cv::Mat &in, + std::vector &rects) { - // 3) Gamma lift (power-law) - bfROI.convertTo(bfROI, CV_32FC1, 1.0/255.0); - cv::pow(bfROI, gamma, bfROI); - bfROI.convertTo(bfROI, CV_8UC1, 255.0); + const double scaleFactor = 1.1; + const int minNeighbors = 3; + const int flags = + cv::CASCADE_FIND_BIGGEST_OBJECT | cv::CASCADE_DO_ROUGH_SEARCH; - // 4) Light unsharp mask to emphasize edges - cv::Mat blurImg; - cv::GaussianBlur(bfROI, blurImg, cv::Size(0,0), 1.0); - cv::addWeighted(bfROI, 1.5, blurImg, -0.5, 0, bfROI); - - return bfROI; + cascade.detectMultiScale(in, rects, scaleFactor, minNeighbors, flags); + return !rects.empty(); } -cv::Mat detectFaceAndSmile( - cv::CascadeClassifier &faceCascade, - cv::CascadeClassifier &smileCascade, - cv::Mat &thumbnailFrame, - cv::Mat &fullFrame, - cv::Rect &faceROIMax, - std::vector &faceROIs, - std::vector &smileROIs, - float mouthGamma) { - - const double scaleFactor = 1.04; - const int minNeighbors = 3; - const int flags = cv::CASCADE_FIND_BIGGEST_OBJECT | cv::CASCADE_DO_ROUGH_SEARCH; - cv::Mat smilePostProc; +static bool sweepBlur(cv::CascadeClassifier &cascade, cv::Mat &in, + std::vector &rects, + ROIKernelParamsSweep ¶ms) { - //cv::equalizeHist(thumbnailFrame, thumbnailFrame); - faceCascade.detectMultiScale(thumbnailFrame, faceROIs, scaleFactor, minNeighbors, - flags); + BlurStage blurStage = BlurStage(in, params.blur.sigmaX); - for (auto & faceROI : faceROIs) { - - try { + for (auto blur : blurStage) { + if (detectSmile(cascade, blur, rects)) { + return true; + } + } - cv::Rect faceROIFull = remapROI(faceROI, thumbnailFrame, fullFrame); + return false; +} - /* Get lower half/third of face ROI + some gap where the smile is most likelly to be */ - const uint32_t offset_y = (faceROIFull.height * 2) / 3; +static bool sweepGamma(cv::CascadeClassifier &cascade, cv::Mat &in, + std::vector &rects, + ROIKernelParamsSweep ¶ms) { - faceROIFull.y = faceROIFull.y + offset_y - 10; - faceROIFull.height = faceROIFull.height - faceROIFull.height / 2; - if (faceROIFull.height + faceROIFull.y > fullFrame.rows) { - faceROIFull.height = fullFrame.rows - faceROIFull.y; - } + GammaStage gammaStage = GammaStage(in, params.gamma.gamma); - cv::Rect roiThumb = remapROI(faceROIFull, fullFrame, thumbnailFrame); + for (auto gamma : gammaStage) { + if (sweepBlur(cascade, gamma, rects, params)) { + return true; + } + } + return false; +} - float sx = 1.0f; - float sy = 1.0f; +static bool sweepBilateral(cv::CascadeClassifier &cascade, cv::Mat &in, + std::vector &rects, + ROIKernelParamsSweep ¶ms) { - cv::Mat faceROIFrame = fullFrame(faceROIFull); - cv::Mat faceROIFrameResized; + BilateralStage bilateralStage = + BilateralStage(in, params.biFilter.d, params.biFilter.sigmaColor, + params.biFilter.sigmaSpace); - // Get area with minimum number of pixels - if (faceROIMax.width * faceROIMax.height < faceROIFull.width * faceROIFull.height) { - sx = (float)faceROIMax.width / (float)faceROIFull.width; - sy = (float)faceROIMax.height / (float)faceROIFull.height; - cv::resize(faceROIFrame, faceROIFrameResized, cv::Size(), sx, sy, cv::INTER_LINEAR); - } else { - faceROIFrameResized = faceROIFrame; - } + for (auto bilateral : bilateralStage) { + if (sweepGamma(cascade, bilateral, rects, params)) { + return true; + } + } + return false; +} - const double scaleFactor = 1.1; - const int minNeighbors = 3; - const int flags = cv::CASCADE_FIND_BIGGEST_OBJECT | cv::CASCADE_DO_ROUGH_SEARCH; +static bool sweepClahe(cv::CascadeClassifier &cascade, cv::Mat &in, + std::vector &rects, + ROIKernelParamsSweep ¶ms) { - //cv::equalizeHist(faceROIFrameResized, faceROIFrameResized); - faceROIFrameResized = preprocessSmileROI(faceROIFrameResized, mouthGamma); + ClaheStage claheStage = + ClaheStage(in, params.clahe.clipLimit, params.clahe.tileSize); - smilePostProc = faceROIFrameResized.clone(); - smileCascade.detectMultiScale(faceROIFrameResized, smileROIs, scaleFactor, minNeighbors, - flags); + for (auto clahe : claheStage) { + if (sweepBilateral(cascade, clahe, rects, params)) { + return true; + } + } - float ssx_inv = (float)thumbnailFrame.cols / (float)fullFrame.cols; - float ssy_inv = (float)thumbnailFrame.rows / (float)fullFrame.rows; + return false; +} - for (auto & smileROI : smileROIs) { - cv::Rect smileROIRemapped = translateScaleROI(smileROI, 1.0f / sx, 1.0f / sy, faceROIFull.x, faceROIFull.y); - smileROIRemapped = translateScaleROI(smileROIRemapped, ssx_inv, ssy_inv, 0.0f, 0.0f); +static void detectSmile(cv::CascadeClassifier &cascade, cv::Mat &in, + std::vector &rects, + ROIKernelParamsSweep ¶ms) { - smileROI = smileROIRemapped; - } + sweepClahe(cascade, in, rects, params); +} - } catch (cv::Exception &ce) { - std::cerr << "Exception: " << ce.what() << std::endl; - } catch ( ... ) { - std::cerr << "Unknown Exception"; - } +std::vector +detectFaceAndSmile(cv::CascadeClassifier &faceCascade, + cv::CascadeClassifier &smileCascade, cv::Mat &thumbnailFrame, + cv::Mat &fullFrame, cv::Rect &faceROIMax, + ROIKernelParamsSweep &roIKernelParamsSweep) { + + std::vector rects; + + const double scaleFactor = 1.04; + const int minNeighbors = 3; + const int flags = + cv::CASCADE_FIND_BIGGEST_OBJECT | cv::CASCADE_DO_ROUGH_SEARCH; + + // cv::equalizeHist(thumbnailFrame, thumbnailFrame); + std::vector faceRects; + faceCascade.detectMultiScale(thumbnailFrame, faceRects, scaleFactor, + minNeighbors, flags); + + for (auto &faceRect : faceRects) { + + rects.push_back(faceRect); + + try { + + cv::Rect faceRectFull = remapROI(faceRect, thumbnailFrame, fullFrame); + + /* Get lower half/third of face ROI + some gap where the smile is most + * likelly to be */ + const uint32_t offset_y = (faceRectFull.height * 2) / 3; + + faceRectFull.y = faceRectFull.y + offset_y - 10; + faceRectFull.height = faceRectFull.height - faceRectFull.height / 2; + if (faceRectFull.height + faceRectFull.y > fullFrame.rows) { + faceRectFull.height = fullFrame.rows - faceRectFull.y; + } + + cv::Rect roiThumb = remapROI(faceRectFull, fullFrame, thumbnailFrame); + + float sx = 1.0f; + float sy = 1.0f; + + cv::Mat faceROIFrame = fullFrame(faceRectFull); + cv::Mat faceROIFrameResized; + + // Get area with minimum number of pixels + if (faceROIMax.width * faceROIMax.height < + faceRectFull.width * faceRectFull.height) { + sx = (float)faceROIMax.width / (float)faceRectFull.width; + sy = (float)faceROIMax.height / (float)faceRectFull.height; + cv::resize(faceROIFrame, faceROIFrameResized, cv::Size(), sx, sy, + cv::INTER_LINEAR); + } else { + faceROIFrameResized = faceROIFrame; + } + + std::vector smileRects; + detectSmile(smileCascade, faceROIFrameResized, smileRects, + roIKernelParamsSweep); + // detectSmile(smileCascade, faceROIFrameResized, smileRects); + + float ssx_inv = (float)thumbnailFrame.cols / (float)fullFrame.cols; + float ssy_inv = (float)thumbnailFrame.rows / (float)fullFrame.rows; + + for (auto &smileRect : smileRects) { + cv::Rect smileROIRemapped = translateScaleROI( + smileRect, 1.0f / sx, 1.0f / sy, faceRectFull.x, faceRectFull.y); + smileROIRemapped = + translateScaleROI(smileROIRemapped, ssx_inv, ssy_inv, 0.0f, 0.0f); + rects.push_back(smileROIRemapped); + } + + } catch (cv::Exception &ce) { + std::cerr << "Exception: " << ce.what() << std::endl; + } catch (...) { + std::cerr << "Unknown Exception"; } + } - return smilePostProc; + return rects; } cv::Rect remapROI(cv::Rect &ROI, cv::Mat &inFrame, cv::Mat &outFrame) { - float sx = (float)outFrame.cols / (float)inFrame.cols; - float sy = (float)outFrame.rows / (float)inFrame.rows; - return translateScaleROI(ROI, sx, sy, 0.0f, 0.0f); + float sx = (float)outFrame.cols / (float)inFrame.cols; + float sy = (float)outFrame.rows / (float)inFrame.rows; + return translateScaleROI(ROI, sx, sy, 0.0f, 0.0f); } -cv::Rect translateScaleROI(cv::Rect &ROI, float sx, float sy, float tx, float ty) { - cv::Rect OutROI; +cv::Rect translateScaleROI(cv::Rect &ROI, float sx, float sy, float tx, + float ty) { + cv::Rect OutROI; - float inROIVecData[6] = { - (float)ROI.x, (float)(ROI.x + ROI.width), - (float)ROI.y, (float)(ROI.y + ROI.height), - 1.0f, 1.0f, - }; + float inROIVecData[6] = { + (float)ROI.x, (float)(ROI.x + ROI.width), + (float)ROI.y, (float)(ROI.y + ROI.height), + 1.0f, 1.0f, + }; - float scaleVecData[9] = { - sx, 0.0f, tx, - 0.0f, sy, ty, - 0.0f, 0.0f, 1.0f - }; + float scaleVecData[9] = {sx, 0.0f, tx, 0.0f, sy, ty, 0.0f, 0.0f, 1.0f}; - cv::Mat scaleMat(3, 3, CV_32FC1, &scaleVecData[0]); - cv::Mat inROIMat(3, 2, CV_32FC1, &inROIVecData[0]); + cv::Mat scaleMat(3, 3, CV_32FC1, &scaleVecData[0]); + cv::Mat inROIMat(3, 2, CV_32FC1, &inROIVecData[0]); - float outROIVecData[6]{}; - cv::Mat outROIMat(3, 2, CV_32FC1, &outROIVecData[0]); + float outROIVecData[6]{}; + cv::Mat outROIMat(3, 2, CV_32FC1, &outROIVecData[0]); - cv::gemm(scaleMat, inROIMat, 1.0f, cv::Mat(), 1.0f, outROIMat); + cv::gemm(scaleMat, inROIMat, 1.0f, cv::Mat(), 1.0f, outROIMat); - OutROI.x = outROIVecData[0]; - OutROI.y = outROIVecData[2]; - OutROI.width = outROIVecData[1] - outROIVecData[0]; - OutROI.height = outROIVecData[3] - outROIVecData[2]; + OutROI.x = outROIVecData[0]; + OutROI.y = outROIVecData[2]; + OutROI.width = outROIVecData[1] - outROIVecData[0]; + OutROI.height = outROIVecData[3] - outROIVecData[2]; - return OutROI; + return OutROI; } + +} /* namespace gf_cv */ \ No newline at end of file diff --git a/u5/common/src/lvgl_utils.cpp b/u5/common/src/lvgl_utils.cpp index 69e4fd5..3578a94 100644 --- a/u5/common/src/lvgl_utils.cpp +++ b/u5/common/src/lvgl_utils.cpp @@ -33,7 +33,7 @@ void fit(lv_obj_t *canvas, const Buffer &src, const Buffer &dst, bool flip_y) { if (flip_y) { img_dsc.rotation = 1800; - img_dsc.pivot.x = src.width / 2; + img_dsc.pivot.x = src.width / 2; img_dsc.pivot.y = src.height / 2; } @@ -44,4 +44,4 @@ void fit(lv_obj_t *canvas, const Buffer &src, const Buffer &dst, bool flip_y) { lv_canvas_finish_layer(canvas, &layer); } -} // namespace Gfx \ No newline at end of file +} // namespace lvgl \ No newline at end of file diff --git a/u5/common/src/utils.cpp b/u5/common/src/utils.cpp index 89d56f5..415b688 100644 --- a/u5/common/src/utils.cpp +++ b/u5/common/src/utils.cpp @@ -1,82 +1,84 @@ #include "gf/utils.hpp" -lv_obj_t *allocLvROIRect(lv_obj_t *parent, uint32_t borderWidth) { - lv_obj_t *rect = lv_obj_create(parent); - lv_obj_set_pos(rect, 0, 0); - lv_obj_set_size(rect, 1, 1); - - // Unfilled, just a thick border - lv_obj_set_style_bg_opa(rect, LV_OPA_TRANSP, LV_PART_MAIN); - lv_obj_set_style_border_width(rect, borderWidth, LV_PART_MAIN); - lv_obj_set_style_border_opa(rect, LV_OPA_COVER, LV_PART_MAIN); - lv_obj_set_style_border_color(rect, lv_palette_main(LV_PALETTE_GREEN), LV_PART_MAIN); - lv_obj_set_style_radius(rect, 0, LV_PART_MAIN); // 0 = sharp corners - lv_obj_set_style_pad_all(rect, 0, LV_PART_MAIN); - lv_obj_set_style_outline_opa(rect, LV_OPA_TRANSP, LV_PART_MAIN); // no focus outline - lv_obj_set_style_shadow_opa(rect, LV_OPA_TRANSP, LV_PART_MAIN); // no shadow - lv_obj_move_foreground(rect); - lv_obj_add_flag(rect, LV_OBJ_FLAG_HIDDEN); - - return rect; +lv_obj_t *allocLvROIRect(lv_obj_t *parent, uint32_t borderWidth, lv_color_t c) { + lv_obj_t *rect = lv_obj_create(parent); + lv_obj_set_pos(rect, 0, 0); + lv_obj_set_size(rect, 1, 1); + + // Unfilled, just a thick border + lv_obj_set_style_bg_opa(rect, LV_OPA_TRANSP, LV_PART_MAIN); + lv_obj_set_style_border_width(rect, borderWidth, LV_PART_MAIN); + lv_obj_set_style_border_opa(rect, LV_OPA_COVER, LV_PART_MAIN); + lv_obj_set_style_border_color(rect, c, LV_PART_MAIN); + lv_obj_set_style_radius(rect, 0, LV_PART_MAIN); // 0 = sharp corners + lv_obj_set_style_pad_all(rect, 0, LV_PART_MAIN); + lv_obj_set_style_outline_opa(rect, LV_OPA_TRANSP, + LV_PART_MAIN); // no focus outline + lv_obj_set_style_shadow_opa(rect, LV_OPA_TRANSP, LV_PART_MAIN); // no shadow + lv_obj_move_foreground(rect); + lv_obj_add_flag(rect, LV_OBJ_FLAG_HIDDEN); + + return rect; } -void cropFullFrameToRoi(lv_obj_t *parent, uint8_t *fullFb, uint8_t *roiFb, lvgl::Size fullSz, - lvgl::Size roiSz, uint32_t fullCf, uint32_t roiCf, bool flip_y) { - lvgl::Buffer src{}; - lvgl::Buffer dst{}; - - uint32_t x = (fullSz.width - roiSz.width) / 2; - //uint32_t y = (FULL_FRAME_HEIGHT - ROI_FRAME_HEIGHT) / 2; - uint32_t y = flip_y ? 0 : fullSz.height - roiSz.height; - - uint32_t fullFbStride = fullSz.width * LV_COLOR_FORMAT_GET_SIZE(fullCf); - uint32_t fullFbOffset = (y * fullFbStride) + x; - uint32_t fullFbSize = roiSz.height * fullFbStride; - - src.buf = fullFb + fullFbOffset; - src.width = roiSz.width; - src.height = roiSz.height; - src.stride = fullFbStride; - src.cf = fullCf; - src.size = fullFbSize - fullFbOffset; - - uint32_t roiFbStride = roiSz.width * LV_COLOR_FORMAT_GET_SIZE(roiCf); - - dst.buf = roiFb; - dst.width = roiSz.width; - dst.height = roiSz.height; - dst.stride = roiFbStride; - dst.cf = roiCf; - dst.size = roiSz.height * roiFbStride; - - - lvgl::fit(parent, src, dst, flip_y); - //lv_task_handler(); +void cropFullFrameToRoi(lv_obj_t *parent, uint8_t *fullFb, uint8_t *roiFb, + lvgl::Size fullSz, lvgl::Size roiSz, uint32_t fullCf, + uint32_t roiCf, bool flip_y) { + lvgl::Buffer src{}; + lvgl::Buffer dst{}; + + uint32_t x = (fullSz.width - roiSz.width) / 2; + // uint32_t y = (FULL_FRAME_HEIGHT - ROI_FRAME_HEIGHT) / 2; + uint32_t y = flip_y ? 0 : fullSz.height - roiSz.height; + + uint32_t fullFbStride = fullSz.width * LV_COLOR_FORMAT_GET_SIZE(fullCf); + uint32_t fullFbOffset = (y * fullFbStride) + x; + uint32_t fullFbSize = roiSz.height * fullFbStride; + + src.buf = fullFb + fullFbOffset; + src.width = roiSz.width; + src.height = roiSz.height; + src.stride = fullFbStride; + src.cf = fullCf; + src.size = fullFbSize - fullFbOffset; + + uint32_t roiFbStride = roiSz.width * LV_COLOR_FORMAT_GET_SIZE(roiCf); + + dst.buf = roiFb; + dst.width = roiSz.width; + dst.height = roiSz.height; + dst.stride = roiFbStride; + dst.cf = roiCf; + dst.size = roiSz.height * roiFbStride; + + lvgl::fit(parent, src, dst, flip_y); + // lv_task_handler(); } -void fitRoiFrameToThumbnail(lv_obj_t *parent, uint8_t *roiFb, uint8_t *tFb, lvgl::Size roiSz, - lvgl::Size tSz, uint32_t roiCf, uint32_t tCf) { - lvgl::Buffer src{}; - lvgl::Buffer dst{}; +void fitRoiFrameToThumbnail(lv_obj_t *parent, uint8_t *roiFb, uint8_t *tFb, + lvgl::Size roiSz, lvgl::Size tSz, uint32_t roiCf, + uint32_t tCf) { + lvgl::Buffer src{}; + lvgl::Buffer dst{}; - uint32_t roiFbStride = roiSz.width * LV_COLOR_FORMAT_GET_SIZE(roiCf); + uint32_t roiFbStride = roiSz.width * LV_COLOR_FORMAT_GET_SIZE(roiCf); - src.buf = roiFb; - src.width = roiSz.width; - src.height = roiSz.height; - src.stride = roiFbStride; - src.cf = roiCf; - src.size = roiSz.height * roiFbStride; + src.buf = roiFb; + src.width = roiSz.width; + src.height = roiSz.height; + src.stride = roiFbStride; + src.cf = roiCf; + src.size = roiSz.height * roiFbStride; - uint32_t tnFbStride = tSz.width * LV_COLOR_FORMAT_GET_SIZE(tCf); + uint32_t tnFbStride = tSz.width * LV_COLOR_FORMAT_GET_SIZE(tCf); - dst.buf = tFb; - dst.width = tSz.width; - dst.height = tSz.height; - dst.stride = tnFbStride; - dst.cf = tCf; - dst.size = tSz.height * tnFbStride; + dst.buf = tFb; + dst.width = tSz.width; + dst.height = tSz.height; + dst.stride = tnFbStride; + dst.cf = tCf; + dst.size = tSz.height * tnFbStride; - lvgl::fit(parent, src, dst); - //lv_task_handler(); + lvgl::fit(parent, src, dst); + // lv_task_handler(); } diff --git a/u5/common/src/video.cpp b/u5/common/src/video.cpp index a683214..2f708b8 100644 --- a/u5/common/src/video.cpp +++ b/u5/common/src/video.cpp @@ -11,189 +11,188 @@ #include LOG_MODULE_REGISTER(video); -namespace Video -{ +namespace Video { static const struct device *video_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_camera)); static struct video_buffer *buffers[2]; #if defined(CONFIG_GRINREFLEX_JPEG_VIDEO) -static struct video_buffer second_buffer{}; +static struct video_buffer second_buffer {}; #endif -void setup() -{ - struct video_format fmt; - struct video_caps caps; - enum video_buf_type type = VIDEO_BUF_TYPE_OUTPUT; - struct video_selection sel = { - .type = VIDEO_BUF_TYPE_OUTPUT, - }; - size_t bsize; - int i = 0; - int err; - - video_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_camera)); - if (!device_is_ready(video_dev)) { - LOG_ERR("%s device is not ready", video_dev->name); - return; - } - - LOG_INF("- Device name: %s", video_dev->name); - - /* Get capabilities */ - caps.type = type; - if (video_get_caps(video_dev, &caps)) { - LOG_ERR("Unable to retrieve video capabilities"); - return; - } - - LOG_INF("- Capabilities:"); - while (caps.format_caps[i].pixelformat) { - const struct video_format_cap *fcap = &caps.format_caps[i]; - /* four %c to string */ - LOG_INF(" %c%c%c%c width [%u; %u; %u] height [%u; %u; %u]", - (char)fcap->pixelformat, (char)(fcap->pixelformat >> 8), - (char)(fcap->pixelformat >> 16), (char)(fcap->pixelformat >> 24), - fcap->width_min, fcap->width_max, fcap->width_step, fcap->height_min, - fcap->height_max, fcap->height_step); - i++; - } - - /* Get default/native format */ - fmt.type = type; - if (video_get_format(video_dev, &fmt)) { - LOG_ERR("Unable to retrieve video format"); - return; - } - - /* Set the crop setting if necessary */ +void setup() { + struct video_format fmt; + struct video_caps caps; + enum video_buf_type type = VIDEO_BUF_TYPE_OUTPUT; + struct video_selection sel = { + .type = VIDEO_BUF_TYPE_OUTPUT, + }; + size_t bsize; + int i = 0; + int err; + + video_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_camera)); + if (!device_is_ready(video_dev)) { + LOG_ERR("%s device is not ready", video_dev->name); + return; + } + + LOG_INF("- Device name: %s", video_dev->name); + + /* Get capabilities */ + caps.type = type; + if (video_get_caps(video_dev, &caps)) { + LOG_ERR("Unable to retrieve video capabilities"); + return; + } + + LOG_INF("- Capabilities:"); + while (caps.format_caps[i].pixelformat) { + const struct video_format_cap *fcap = &caps.format_caps[i]; + /* four %c to string */ + LOG_INF(" %c%c%c%c width [%u; %u; %u] height [%u; %u; %u]", + (char)fcap->pixelformat, (char)(fcap->pixelformat >> 8), + (char)(fcap->pixelformat >> 16), (char)(fcap->pixelformat >> 24), + fcap->width_min, fcap->width_max, fcap->width_step, + fcap->height_min, fcap->height_max, fcap->height_step); + i++; + } + + /* Get default/native format */ + fmt.type = type; + if (video_get_format(video_dev, &fmt)) { + LOG_ERR("Unable to retrieve video format"); + return; + } + + /* Set the crop setting if necessary */ #if CONFIG_VIDEO_SOURCE_CROP_WIDTH && CONFIG_VIDEO_SOURCE_CROP_HEIGHT - sel.target = VIDEO_SEL_TGT_CROP; - sel.rect.left = CONFIG_VIDEO_SOURCE_CROP_LEFT; - sel.rect.top = CONFIG_VIDEO_SOURCE_CROP_TOP; - sel.rect.width = CONFIG_VIDEO_SOURCE_CROP_WIDTH; - sel.rect.height = CONFIG_VIDEO_SOURCE_CROP_HEIGHT; - if (video_set_selection(video_dev, &sel)) { - LOG_ERR("Unable to set selection crop"); - return; - } - LOG_INF("Selection crop set to (%u,%u)/%ux%u", - sel.rect.left, sel.rect.top, sel.rect.width, sel.rect.height); + sel.target = VIDEO_SEL_TGT_CROP; + sel.rect.left = CONFIG_VIDEO_SOURCE_CROP_LEFT; + sel.rect.top = CONFIG_VIDEO_SOURCE_CROP_TOP; + sel.rect.width = CONFIG_VIDEO_SOURCE_CROP_WIDTH; + sel.rect.height = CONFIG_VIDEO_SOURCE_CROP_HEIGHT; + if (video_set_selection(video_dev, &sel)) { + LOG_ERR("Unable to set selection crop"); + return; + } + LOG_INF("Selection crop set to (%u,%u)/%ux%u", sel.rect.left, sel.rect.top, + sel.rect.width, sel.rect.height); #endif - /* Set format */ - fmt.width = CONFIG_GRINREFLEX_VIDEO_WIDTH; - fmt.height = CONFIG_GRINREFLEX_VIDEO_HEIGHT; + /* Set format */ + fmt.width = CONFIG_GRINREFLEX_VIDEO_WIDTH; + fmt.height = CONFIG_GRINREFLEX_VIDEO_HEIGHT; #if defined(CONFIG_GRINREFLEX_JPEG_VIDEO) - fmt.pixelformat = VIDEO_PIX_FMT_JPEG; + fmt.pixelformat = VIDEO_PIX_FMT_JPEG; #else - fmt.pixelformat = VIDEO_PIX_FMT_RGB565; + fmt.pixelformat = VIDEO_PIX_FMT_RGB565; #endif - /* - * Check (if possible) if targeted size is same as crop - * and if compose is necessary - */ - sel.target = VIDEO_SEL_TGT_CROP; - err = video_get_selection(video_dev, &sel); - if (err < 0 && err != -ENOSYS) { - LOG_ERR("Unable to get selection crop"); - return; - } - - if (err == 0 && (sel.rect.width != fmt.width || sel.rect.height != fmt.height)) { - sel.target = VIDEO_SEL_TGT_COMPOSE; - sel.rect.left = 0; - sel.rect.top = 0; - sel.rect.width = fmt.width; - sel.rect.height = fmt.height; - err = video_set_selection(video_dev, &sel); - if (err < 0 && err != -ENOSYS) { - LOG_ERR("Unable to set selection compose"); - return; - } - } - - if (video_set_format(video_dev, &fmt)) { - LOG_ERR("Unable to set up video format"); - return; - } - - LOG_INF("- Format: %c%c%c%c %ux%u %u", (char)fmt.pixelformat, (char)(fmt.pixelformat >> 8), - (char)(fmt.pixelformat >> 16), (char)(fmt.pixelformat >> 24), fmt.width, fmt.height, - fmt.pitch); - - if (caps.min_line_count != LINE_COUNT_HEIGHT) { - LOG_ERR("Partial framebuffers not supported by this sample"); - return; - } - /* Size to allocate for each buffer */ - bsize = fmt.pitch * fmt.height; + /* + * Check (if possible) if targeted size is same as crop + * and if compose is necessary + */ + sel.target = VIDEO_SEL_TGT_CROP; + err = video_get_selection(video_dev, &sel); + if (err < 0 && err != -ENOSYS) { + LOG_ERR("Unable to get selection crop"); + return; + } + + if (err == 0 && + (sel.rect.width != fmt.width || sel.rect.height != fmt.height)) { + sel.target = VIDEO_SEL_TGT_COMPOSE; + sel.rect.left = 0; + sel.rect.top = 0; + sel.rect.width = fmt.width; + sel.rect.height = fmt.height; + err = video_set_selection(video_dev, &sel); + if (err < 0 && err != -ENOSYS) { + LOG_ERR("Unable to set selection compose"); + return; + } + } + + if (video_set_format(video_dev, &fmt)) { + LOG_ERR("Unable to set up video format"); + return; + } + + LOG_INF("- Format: %c%c%c%c %ux%u %u", (char)fmt.pixelformat, + (char)(fmt.pixelformat >> 8), (char)(fmt.pixelformat >> 16), + (char)(fmt.pixelformat >> 24), fmt.width, fmt.height, fmt.pitch); + + if (caps.min_line_count != LINE_COUNT_HEIGHT) { + LOG_ERR("Partial framebuffers not supported by this sample"); + return; + } + /* Size to allocate for each buffer */ + bsize = fmt.pitch * fmt.height; #if defined(CONFIG_GRINREFLEX_JPEG_VIDEO) - bsize = CONFIG_VIDEO_BUFFER_POOL_SZ_MAX; - - buffers[0] = video_buffer_aligned_alloc(bsize, CONFIG_VIDEO_BUFFER_POOL_ALIGN, - K_FOREVER); - if (buffers[0] == NULL) { - LOG_ERR("Unable to alloc video buffer"); - return; - } - buffers[0]->type = type; - video_enqueue(video_dev, buffers[0]); - - buffers[1] = &second_buffer; - - bsize = CONFIG_GRINREFLEX_VIDEO_WIDTH * CONFIG_GRINREFLEX_VIDEO_HEIGHT * 2; - buffers[1]->type = type; - buffers[1]->buffer = new uint8_t[bsize]; - buffers[1]->size = bsize; - - if (buffers[1]->buffer == NULL) { - LOG_ERR("Unable to alloc video buffer"); - return; - } - - video_enqueue(video_dev, buffers[1]); -#else /* !defined(CONFIG_GRINREFLEX_JPEG_VIDEO) */ - /* Alloc video buffers and enqueue for capture */ - for (i = 0; i < ARRAY_SIZE(buffers); i++) { - buffers[i] = video_buffer_aligned_alloc(bsize, CONFIG_VIDEO_BUFFER_POOL_ALIGN, - K_FOREVER); - if (buffers[i] == NULL) { - LOG_ERR("Unable to alloc video buffer"); - return; - } - buffers[i]->type = type; - video_enqueue(video_dev, buffers[i]); - } + bsize = CONFIG_VIDEO_BUFFER_POOL_SZ_MAX; + + buffers[0] = video_buffer_aligned_alloc(bsize, CONFIG_VIDEO_BUFFER_POOL_ALIGN, + K_FOREVER); + if (buffers[0] == NULL) { + LOG_ERR("Unable to alloc video buffer"); + return; + } + buffers[0]->type = type; + video_enqueue(video_dev, buffers[0]); + + buffers[1] = &second_buffer; + + bsize = CONFIG_GRINREFLEX_VIDEO_WIDTH * CONFIG_GRINREFLEX_VIDEO_HEIGHT * 2; + buffers[1]->type = type; + buffers[1]->buffer = new uint8_t[bsize]; + buffers[1]->size = bsize; + + if (buffers[1]->buffer == NULL) { + LOG_ERR("Unable to alloc video buffer"); + return; + } + + video_enqueue(video_dev, buffers[1]); +#else /* !defined(CONFIG_GRINREFLEX_JPEG_VIDEO) */ + /* Alloc video buffers and enqueue for capture */ + for (i = 0; i < ARRAY_SIZE(buffers); i++) { + buffers[i] = video_buffer_aligned_alloc( + bsize, CONFIG_VIDEO_BUFFER_POOL_ALIGN, K_FOREVER); + if (buffers[i] == NULL) { + LOG_ERR("Unable to alloc video buffer"); + return; + } + buffers[i]->type = type; + video_enqueue(video_dev, buffers[i]); + } #endif /* defined(CONFIG_GRINREFLEX_JPEG_VIDEO) */ - /* Set controls */ - struct video_control ctrl = {.id = VIDEO_CID_HFLIP, .val = 1}; + /* Set controls */ + struct video_control ctrl = {.id = VIDEO_CID_HFLIP, .val = 1}; - if (IS_ENABLED(CONFIG_VIDEO_HFLIP)) { - video_set_ctrl(video_dev, &ctrl); - } + if (IS_ENABLED(CONFIG_VIDEO_HFLIP)) { + video_set_ctrl(video_dev, &ctrl); + } - if (IS_ENABLED(CONFIG_VIDEO_VFLIP)) { - ctrl.id = VIDEO_CID_VFLIP; - video_set_ctrl(video_dev, &ctrl); - } + if (IS_ENABLED(CONFIG_VIDEO_VFLIP)) { + ctrl.id = VIDEO_CID_VFLIP; + video_set_ctrl(video_dev, &ctrl); + } #if defined(CONFIG_GRINREFLEX_JPEG_VIDEO) - ctrl.id = VIDEO_CID_JPEG_COMPRESSION_QUALITY; - ctrl.val = CONFIG_GRINREFLEX_JPEG_VIDEO_QUALITY; - video_set_ctrl(video_dev, &ctrl); + ctrl.id = VIDEO_CID_JPEG_COMPRESSION_QUALITY; + ctrl.val = CONFIG_GRINREFLEX_JPEG_VIDEO_QUALITY; + video_set_ctrl(video_dev, &ctrl); #endif - /* Start video capture */ - if (video_stream_start(video_dev, type)) { - LOG_ERR("Unable to start capture (interface)"); - return; - } + /* Start video capture */ + if (video_stream_start(video_dev, type)) { + LOG_ERR("Unable to start capture (interface)"); + return; + } - LOG_INF("- Capture started"); + LOG_INF("- Capture started"); } -} \ No newline at end of file +} // namespace Video \ No newline at end of file diff --git a/u5/gf_dkx/prj.conf b/u5/gf_dkx/prj.conf index 4edbfb1..c0937b5 100644 --- a/u5/gf_dkx/prj.conf +++ b/u5/gf_dkx/prj.conf @@ -41,11 +41,11 @@ CONFIG_ASSERT=y # Make asserts print file:line and message CONFIG_ASSERT_VERBOSE=y -CONFIG_STACK_SENTINEL=y -CONFIG_THREAD_STACK_INFO=y -CONFIG_INIT_STACKS=y # fill stacks, helps detect overflow +#CONFIG_STACK_SENTINEL=y +#CONFIG_THREAD_STACK_INFO=y +#CONFIG_INIT_STACKS=y # fill stacks, helps detect overflow -CONFIG_DEBUG_OPTIMIZATIONS=y +#CONFIG_DEBUG_OPTIMIZATIONS=y CONFIG_SPEED_OPTIMIZATIONS=y #CONFIG_TFLITE_LIB=y @@ -88,11 +88,11 @@ CONFIG_VIDEO_SHELL=y CONFIG_STM32_JPEG_RGB_FORMAT=3 CONFIG_GRINREFLEX_JPEG_VIDEO=y -CONFIG_GRINREFLEX_JPEG_VIDEO_QUALITY=10 +CONFIG_GRINREFLEX_JPEG_VIDEO_QUALITY=50 CONFIG_GRINREFLEX_VIDEO_WIDTH=640 CONFIG_GRINREFLEX_VIDEO_HEIGHT=480 -CONFIG_VIDEO_LOG_LEVEL_DBG=y +#CONFIG_VIDEO_LOG_LEVEL_DBG=y CONFIG_VIDEO_LOG_LEVEL_WRN=y CONFIG_VIDEO_LOG_LEVEL_ERR=y diff --git a/u5/gf_dkx/src/config.hpp b/u5/gf_dkx/src/config.hpp index f280d01..fcf2c0c 100644 --- a/u5/gf_dkx/src/config.hpp +++ b/u5/gf_dkx/src/config.hpp @@ -2,7 +2,6 @@ #pragma once - #if !defined(CONFIG_GRINREFLEX_JPEG_VIDEO) #error "Not supported yet" #endif @@ -33,4 +32,4 @@ #define THUMBNAIL_FRAME_WIDTH 80 #define THUMBNAIL_FRAME_HEIGHT 80 -#define FULL_FRAME_FLIP_Y (1) \ No newline at end of file +#define FULL_FRAME_FLIP_Y (0) \ No newline at end of file diff --git a/u5/gf_dkx/src/lvgl_video_app.cpp b/u5/gf_dkx/src/lvgl_video_app.cpp index e2246be..0333c88 100644 --- a/u5/gf_dkx/src/lvgl_video_app.cpp +++ b/u5/gf_dkx/src/lvgl_video_app.cpp @@ -3,10 +3,10 @@ #include #if defined(CONFIG_OPENCV_LIB) +#include "cascades.h" #include "opencv2/opencv.hpp" #include "opencv_utils.hpp" -#include "cascades.h" -#endif +#endif #include #include @@ -27,85 +27,92 @@ LOG_MODULE_REGISTER(grinreflex_app); #include #include "config.hpp" +#include "gf/lvgl_utils.hpp" #include "gf/utils.hpp" #include "gf/video.hpp" -#include "gf/lvgl_utils.hpp" #include "grinreflex.h" static const struct device *video_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_camera)); -static const struct device *display_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_display)); +static const struct device *display_dev = + DEVICE_DT_GET(DT_CHOSEN(zephyr_display)); const struct device *jpeg_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_jpeg)); -static uint8_t fullFrameBuffer[FULL_FRAME_WIDTH * FULL_FRAME_HEIGHT * LV_COLOR_FORMAT_GET_SIZE(FULL_FRAME_COLOR_FORMAT)]; -static uint8_t roiFrameBuffer[ROI_FRAME_WIDTH * ROI_FRAME_HEIGHT * LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8)]; -static uint8_t thumbnailFrameBuffer[THUMBNAIL_FRAME_WIDTH * THUMBNAIL_FRAME_HEIGHT * LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8)]; +static uint8_t + fullFrameBuffer[FULL_FRAME_WIDTH * FULL_FRAME_HEIGHT * + LV_COLOR_FORMAT_GET_SIZE(FULL_FRAME_COLOR_FORMAT)]; +static uint8_t roiFrameBuffer[ROI_FRAME_WIDTH * ROI_FRAME_HEIGHT * + LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8)]; +static uint8_t + thumbnailFrameBuffer[THUMBNAIL_FRAME_WIDTH * THUMBNAIL_FRAME_HEIGHT * + LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8)]; static lv_obj_t *screen_canvas; static lv_obj_t *dummy_canvas; -int init() -{ - if (!device_is_ready(display_dev)) { - LOG_ERR("Display device not ready, aborting test"); - return -ENODEV; - } +int init() { + if (!device_is_ready(display_dev)) { + LOG_ERR("Display device not ready, aborting test"); + return -ENODEV; + } - if (!device_is_ready(video_dev)) { - LOG_ERR("Video device not ready, aborting test"); - return -ENODEV; - } + if (!device_is_ready(video_dev)) { + LOG_ERR("Video device not ready, aborting test"); + return -ENODEV; + } - if (!device_is_ready(jpeg_dev)) { - printf("%s JPEG device not ready", jpeg_dev->name); - return -ENODEV; - } + if (!device_is_ready(jpeg_dev)) { + printf("%s JPEG device not ready", jpeg_dev->name); + return -ENODEV; + } - Video::setup(); + Video::setup(); - dummy_canvas = lv_canvas_create(NULL); - screen_canvas = lv_canvas_create(lv_screen_active()); - lv_canvas_fill_bg(screen_canvas, lv_color_hex3(0xccc), LV_OPA_COVER); - lv_obj_center(screen_canvas); + dummy_canvas = lv_canvas_create(NULL); + screen_canvas = lv_canvas_create(lv_screen_active()); + lv_canvas_fill_bg(screen_canvas, lv_color_hex3(0xccc), LV_OPA_COVER); + lv_obj_center(screen_canvas); - display_blanking_off(display_dev); + display_blanking_off(display_dev); - return 0; + return 0; } -int loop() -{ +int loop() { - struct video_buffer vbuf{}; - struct video_buffer *vbuf_ptr = &vbuf; - struct jpeg_out_prop jpeg_prop; + struct video_buffer vbuf {}; + struct video_buffer *vbuf_ptr = &vbuf; + struct jpeg_out_prop jpeg_prop; - vbuf_ptr->type = VIDEO_BUF_TYPE_OUTPUT; + vbuf_ptr->type = VIDEO_BUF_TYPE_OUTPUT; - int err = video_dequeue(video_dev, &vbuf_ptr, K_FOREVER); - if (err) { - LOG_ERR("Unable to dequeue video buf"); - return 0; - } + int err = video_dequeue(video_dev, &vbuf_ptr, K_FOREVER); + if (err) { + LOG_ERR("Unable to dequeue video buf"); + return 0; + } - //jpeg_hw_decode(jpeg_dev, (uint8_t *)vbuf_ptr->buffer, vbuf_ptr->bytesused, jpeg_frame_buffer); + // jpeg_hw_decode(jpeg_dev, (uint8_t *)vbuf_ptr->buffer, vbuf_ptr->bytesused, + // jpeg_frame_buffer); - jpeg_hw_poll(jpeg_dev, 0, &jpeg_prop); + jpeg_hw_poll(jpeg_dev, 0, &jpeg_prop); - //printf("JPEG done w = %d, h = %d, color = %d chroma = %d\n", jpeg_prop.width, - // jpeg_prop.height, jpeg_prop.color_space, jpeg_prop.chroma); + // printf("JPEG done w = %d, h = %d, color = %d chroma = %d\n", + // jpeg_prop.width, + // jpeg_prop.height, jpeg_prop.color_space, jpeg_prop.chroma); - jpeg_color_convert_helper(jpeg_dev, &jpeg_prop, vbuf_ptr->buffer, fullFrameBuffer); + jpeg_color_convert_helper(jpeg_dev, &jpeg_prop, vbuf_ptr->buffer, + fullFrameBuffer); - cropFullFrameToRoi(dummy_canvas, fullFrameBuffer, roiFrameBuffer); - fitRoiFrameToThumbnail(screen_canvas, roiFrameBuffer, thumbnailFrameBuffer); - lv_task_handler(); + cropFullFrameToRoi(dummy_canvas, fullFrameBuffer, roiFrameBuffer); + fitRoiFrameToThumbnail(screen_canvas, roiFrameBuffer, thumbnailFrameBuffer); + lv_task_handler(); - err = video_enqueue(video_dev, vbuf_ptr); - if (err) { - LOG_ERR("Unable to requeue video buf"); - } + err = video_enqueue(video_dev, vbuf_ptr); + if (err) { + LOG_ERR("Unable to requeue video buf"); + } - k_msleep(50); + k_msleep(50); - return 0; + return 0; } \ No newline at end of file diff --git a/u5/gf_dkx/src/main.c b/u5/gf_dkx/src/main.c index 672166a..6d65f01 100644 --- a/u5/gf_dkx/src/main.c +++ b/u5/gf_dkx/src/main.c @@ -4,13 +4,13 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include +#include #include #include -#include #include +#include +#include #include -#include #define CONFIG_VIDEO_WIDTH 160 #define CONFIG_VIDEO_HEIGHT 120 @@ -27,12 +27,10 @@ LOG_MODULE_REGISTER(main, CONFIG_LOG_DEFAULT_LEVEL); #error No display chosen in devicetree. Missing "--shield" flag? #endif -int main(void) -{ - init(); +int main(void) { + init(); - while (1) - { - loop(); - } + while (1) { + loop(); + } } diff --git a/u5/gf_dkx/src/opencv_fd_app.cpp b/u5/gf_dkx/src/opencv_fd_app.cpp index 08fc73e..3cd1c3d 100644 --- a/u5/gf_dkx/src/opencv_fd_app.cpp +++ b/u5/gf_dkx/src/opencv_fd_app.cpp @@ -3,10 +3,10 @@ #include #if defined(CONFIG_OPENCV_LIB) -#include "opencv2/opencv.hpp" -#include "gf/opencv_utils.hpp" #include "cascades.h" -#endif +#include "gf/opencv_utils.hpp" +#include "opencv2/opencv.hpp" +#endif #include #include @@ -24,8 +24,8 @@ #include LOG_MODULE_REGISTER(grinreflex_app); -#include "gf/video.hpp" #include "gf/lvgl_utils.hpp" +#include "gf/video.hpp" #include @@ -33,16 +33,24 @@ LOG_MODULE_REGISTER(grinreflex_app); #include "gf/utils.hpp" #include "grinreflex.h" +using namespace gf_cv; + static const struct device *video_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_camera)); -static const struct device *display_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_display)); +static const struct device *display_dev = + DEVICE_DT_GET(DT_CHOSEN(zephyr_display)); const struct device *jpeg_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_jpeg)); #if DT_NODE_EXISTS(DT_ALIAS(led0)) static const struct gpio_dt_spec led = GPIO_DT_SPEC_GET(DT_ALIAS(led0), gpios); #endif -static uint8_t fullFrameBuffer[FULL_FRAME_WIDTH * FULL_FRAME_HEIGHT * LV_COLOR_FORMAT_GET_SIZE(FULL_FRAME_COLOR_FORMAT)]; -static uint8_t roiFrameBuffer[ROI_FRAME_WIDTH * ROI_FRAME_HEIGHT * LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8)]; -static uint8_t thumbnailFrameBuffer[THUMBNAIL_FRAME_WIDTH * THUMBNAIL_FRAME_HEIGHT * LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8)]; +static uint8_t + fullFrameBuffer[FULL_FRAME_WIDTH * FULL_FRAME_HEIGHT * + LV_COLOR_FORMAT_GET_SIZE(FULL_FRAME_COLOR_FORMAT)]; +static uint8_t roiFrameBuffer[ROI_FRAME_WIDTH * ROI_FRAME_HEIGHT * + LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8)]; +static uint8_t + thumbnailFrameBuffer[THUMBNAIL_FRAME_WIDTH * THUMBNAIL_FRAME_HEIGHT * + LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8)]; static lv_obj_t *screen_canvas; static lv_obj_t *dummy_canvas; @@ -52,160 +60,162 @@ static lv_obj_t *ROIRectFace; static lv_obj_t *ROIRectSmile; static cv::CascadeClassifier faceCascade; static cv::CascadeClassifier smileCascade; -float constexpr smileROIGammaMax = 0.7f; -float constexpr smileROIGammaMin = 0.1f; -float smileROIGamma = smileROIGammaMin; - -int init() -{ - if (!device_is_ready(display_dev)) { - LOG_ERR("Display device not ready, aborting test"); - return -ENODEV; - } - - if (!device_is_ready(video_dev)) { - LOG_ERR("Video device not ready, aborting test"); - return -ENODEV; - } - - if (!device_is_ready(jpeg_dev)) { - printf("%s JPEG device not ready", jpeg_dev->name); - return -ENODEV; - } -#if DT_NODE_EXISTS(DT_ALIAS(led0)) - if (!gpio_is_ready_dt(&led)) { - return -ENODEV; - } - - if (gpio_pin_configure_dt(&led, GPIO_OUTPUT_ACTIVE) < 0) { - return -ENODEV; - } -#endif +ROIKernelParamsSweep smileROIKernelParams; - cv::setNumThreads(1); - cv::setUseOptimized(false); +int init() { + if (!device_is_ready(display_dev)) { + LOG_ERR("Display device not ready, aborting test"); + return -ENODEV; + } + if (!device_is_ready(video_dev)) { + LOG_ERR("Video device not ready, aborting test"); + return -ENODEV; + } - if (false == loadCascade(faceCascade, cascade_face_data(), cascade_face_len())) { - return -3; - } + if (!device_is_ready(jpeg_dev)) { + printf("%s JPEG device not ready", jpeg_dev->name); + return -ENODEV; + } - if (false == loadCascade(smileCascade, cascade_smile_data(), cascade_smile_len())) { - return -3; - } +#if DT_NODE_EXISTS(DT_ALIAS(led0)) + if (!gpio_is_ready_dt(&led)) { + return -ENODEV; + } - Video::setup(); + if (gpio_pin_configure_dt(&led, GPIO_OUTPUT_ACTIVE) < 0) { + return -ENODEV; + } +#endif - dummy_canvas = lv_canvas_create(NULL); - screen_canvas = lv_canvas_create(lv_screen_active()); - lv_canvas_fill_bg(screen_canvas, lv_color_hex3(0xccc), LV_OPA_COVER); - lv_obj_center(screen_canvas); + cv::setNumThreads(1); + cv::setUseOptimized(false); - display_blanking_off(display_dev); + if (false == + loadCascade(faceCascade, cascade_face_data(), cascade_face_len())) { + return -3; + } - screen = lv_img_create(lv_scr_act()); + if (false == + loadCascade(smileCascade, cascade_smile_data(), cascade_smile_len())) { + return -3; + } - ROIRectFace = allocLvROIRect(lv_scr_act(), 4); - ROIRectSmile = allocLvROIRect(lv_scr_act(), 2); + Video::setup(); - return 0; -} + dummy_canvas = lv_canvas_create(NULL); + screen_canvas = lv_canvas_create(lv_screen_active()); + lv_canvas_fill_bg(screen_canvas, lv_color_hex3(0xccc), LV_OPA_COVER); + lv_obj_center(screen_canvas); -int loop() -{ - struct video_buffer vbuf{}; - struct video_buffer *vbuf_ptr = &vbuf; - struct jpeg_out_prop jpeg_prop; + display_blanking_off(display_dev); - vbuf_ptr->type = VIDEO_BUF_TYPE_OUTPUT; + screen = lv_img_create(lv_scr_act()); - int err = video_dequeue(video_dev, &vbuf_ptr, K_FOREVER); - if (err) { - LOG_ERR("Unable to dequeue video buf"); - return 0; - } + ROIRectFace = + allocLvROIRect(lv_scr_act(), 4, lv_palette_main(LV_PALETTE_GREEN)); + ROIRectSmile = + allocLvROIRect(lv_scr_act(), 4, lv_palette_main(LV_PALETTE_RED)); - //jpeg_hw_decode(jpeg_dev, (uint8_t *)vbuf_ptr->buffer, vbuf_ptr->bytesused, jpeg_frame_buffer); + smileROIKernelParams.clahe.clipLimit = Range(4.0, 4.0, 1.0); + smileROIKernelParams.clahe.tileSize = Range(4, 4, 1); + smileROIKernelParams.biFilter.d = Range(3, 3, 1); + smileROIKernelParams.biFilter.sigmaColor = Range(20.0, 20.0, 1.0); + smileROIKernelParams.biFilter.sigmaSpace = Range(50.0, 50.0, 1.0); + smileROIKernelParams.gamma.gamma = Range(0.44, 0.50, 0.1); + smileROIKernelParams.blur.sigmaX = Range(0.75, 0.85, 0.05); - jpeg_hw_poll(jpeg_dev, 0, &jpeg_prop); + return 0; +} - //printf("JPEG done w = %d, h = %d, color = %d chroma = %d\n", jpeg_prop.width, - // jpeg_prop.height, jpeg_prop.color_space, jpeg_prop.chroma); +int loop() { + struct video_buffer vbuf {}; + struct video_buffer *vbuf_ptr = &vbuf; + struct jpeg_out_prop jpeg_prop; - jpeg_color_convert_helper(jpeg_dev, &jpeg_prop, vbuf_ptr->buffer, fullFrameBuffer); + vbuf_ptr->type = VIDEO_BUF_TYPE_OUTPUT; - cropFullFrameToRoi(dummy_canvas, fullFrameBuffer, roiFrameBuffer, - lvgl::Size(FULL_FRAME_WIDTH, FULL_FRAME_HEIGHT), - lvgl::Size(ROI_FRAME_WIDTH, ROI_FRAME_HEIGHT), - FULL_FRAME_COLOR_FORMAT, LV_COLOR_FORMAT_L8, FULL_FRAME_FLIP_Y); + int err = video_dequeue(video_dev, &vbuf_ptr, K_FOREVER); + if (err) { + LOG_ERR("Unable to dequeue video buf"); + return 0; + } - fitRoiFrameToThumbnail(screen_canvas, roiFrameBuffer, thumbnailFrameBuffer, - lvgl::Size(ROI_FRAME_WIDTH, ROI_FRAME_HEIGHT), - lvgl::Size(THUMBNAIL_FRAME_WIDTH, THUMBNAIL_FRAME_HEIGHT), - LV_COLOR_FORMAT_L8, LV_COLOR_FORMAT_L8); - lv_task_handler(); + // jpeg_hw_decode(jpeg_dev, (uint8_t *)vbuf_ptr->buffer, vbuf_ptr->bytesused, + // jpeg_frame_buffer); - // Return buffer back, we don't need it anymore, thank you - err = video_enqueue(video_dev, vbuf_ptr); - if (err) { - LOG_ERR("Unable to requeue video buf"); - return 0; - } + jpeg_hw_poll(jpeg_dev, 0, &jpeg_prop); - cv::Mat matGraySmall(THUMBNAIL_FRAME_HEIGHT, THUMBNAIL_FRAME_WIDTH, CV_8UC1, thumbnailFrameBuffer); - cv::Mat matGrayFull(ROI_FRAME_HEIGHT, ROI_FRAME_WIDTH, CV_8UC1, roiFrameBuffer); - - cv::Rect faceROIMax{}; - faceROIMax.width = 64; - faceROIMax.height = 64; - - std::vector faces; - std::vector smiles; - - detectFaceAndSmile( - faceCascade, - smileCascade, - matGraySmall, - matGrayFull, - faceROIMax, - faces, - smiles, - smileROIGamma - ); - - if (!faces.empty() && smiles.empty()) { - smileROIGamma += 0.1f; - if (smileROIGamma > smileROIGammaMax) { - smileROIGamma = smileROIGammaMin; - } - } - cv::Rect face, smile; - if (faces.size()) { - lv_obj_remove_flag(ROIRectFace, LV_OBJ_FLAG_HIDDEN); + // printf("JPEG done w = %d, h = %d, color = %d chroma = %d\n", + // jpeg_prop.width, + // jpeg_prop.height, jpeg_prop.color_space, jpeg_prop.chroma); - face = faces[0]; - lv_obj_align_to(ROIRectFace, screen_canvas, LV_ALIGN_TOP_LEFT, face.x, face.y); - lv_obj_set_size(ROIRectFace, face.width, face.height); + jpeg_color_convert_helper(jpeg_dev, &jpeg_prop, vbuf_ptr->buffer, + fullFrameBuffer); - } else { - lv_obj_add_flag(ROIRectFace, LV_OBJ_FLAG_HIDDEN); - } + cropFullFrameToRoi(dummy_canvas, fullFrameBuffer, roiFrameBuffer, + lvgl::Size(FULL_FRAME_WIDTH, FULL_FRAME_HEIGHT), + lvgl::Size(ROI_FRAME_WIDTH, ROI_FRAME_HEIGHT), + FULL_FRAME_COLOR_FORMAT, LV_COLOR_FORMAT_L8, + FULL_FRAME_FLIP_Y); - if (smiles.size()) { - smile = smiles[0]; - if ((smile & face).area() != 0) { - lv_obj_align_to(ROIRectSmile, screen_canvas, LV_ALIGN_TOP_LEFT, smile.x, smile.y); - lv_obj_set_size(ROIRectSmile, smile.width, smile.height); + fitRoiFrameToThumbnail( + screen_canvas, roiFrameBuffer, thumbnailFrameBuffer, + lvgl::Size(ROI_FRAME_WIDTH, ROI_FRAME_HEIGHT), + lvgl::Size(THUMBNAIL_FRAME_WIDTH, THUMBNAIL_FRAME_HEIGHT), + LV_COLOR_FORMAT_L8, LV_COLOR_FORMAT_L8); + lv_task_handler(); - lv_obj_remove_flag(ROIRectSmile, LV_OBJ_FLAG_HIDDEN); + // Return buffer back, we don't need it anymore, thank you + err = video_enqueue(video_dev, vbuf_ptr); + if (err) { + LOG_ERR("Unable to requeue video buf"); + return 0; + } + + cv::Mat matGraySmall(THUMBNAIL_FRAME_HEIGHT, THUMBNAIL_FRAME_WIDTH, CV_8UC1, + thumbnailFrameBuffer); + cv::Mat matGrayFull(ROI_FRAME_HEIGHT, ROI_FRAME_WIDTH, CV_8UC1, + roiFrameBuffer); + + // This is the best configuration I could find, 64 x 30 works even better + // But that may miss smile area + cv::Rect faceROIMax(0, 0, 64, 40); + + cv::equalizeHist(matGraySmall, matGraySmall); + auto rects = + detectFaceAndSmile(faceCascade, smileCascade, matGraySmall, matGrayFull, + faceROIMax, smileROIKernelParams); + + cv::Rect face, smile; + if (rects.size()) { + lv_obj_remove_flag(ROIRectFace, LV_OBJ_FLAG_HIDDEN); + + face = rects[0]; + lv_obj_align_to(ROIRectFace, screen_canvas, LV_ALIGN_TOP_LEFT, face.x, + face.y); + lv_obj_set_size(ROIRectFace, face.width, face.height); + + } else { + lv_obj_add_flag(ROIRectFace, LV_OBJ_FLAG_HIDDEN); + } + + if (rects.size() > 1) { + smile = rects[1]; + if ((smile & face).area() != 0) { + lv_obj_align_to(ROIRectSmile, screen_canvas, LV_ALIGN_TOP_LEFT, smile.x, + smile.y); + lv_obj_set_size(ROIRectSmile, smile.width, smile.height); + + lv_obj_remove_flag(ROIRectSmile, LV_OBJ_FLAG_HIDDEN); #if DT_NODE_EXISTS(DT_ALIAS(led0)) - gpio_pin_set_dt(&led, 0); + gpio_pin_set_dt(&led, 0); #endif - } - } else { - lv_obj_add_flag(ROIRectSmile, LV_OBJ_FLAG_HIDDEN); } + } else { + lv_obj_add_flag(ROIRectSmile, LV_OBJ_FLAG_HIDDEN); + } - return 0; + return 0; } \ No newline at end of file From ab3b93a4a6acaaf1e53c63b3bbbbc23abf17f8a5 Mon Sep 17 00:00:00 2001 From: Roman Pustobaiev Date: Fri, 17 Oct 2025 18:45:54 +0200 Subject: [PATCH 06/18] Adding opencv pipeline for mouth area Tweaking libc alloc buffer Adding properly aligned second video buffer (jpeg format only) --- host/Src/main.cpp | 217 +++++++++--------- .../opencv/include/gf/opencv_pipeline.hpp | 65 +++++- u5/common/opencv/include/gf/opencv_utils.hpp | 58 +---- u5/common/opencv/src/opencv_pipeline.cpp | 36 ++- u5/common/opencv/src/opencv_utils.cpp | 84 +------ u5/common/src/video.cpp | 7 +- u5/drivers/jpeg/jpeg.c | 38 ++- u5/gf_dkx/prj.conf | 8 +- u5/gf_dkx/src/lvgl_video_app.cpp | 13 +- u5/gf_dkx/src/opencv_fd_app.cpp | 34 +-- 10 files changed, 269 insertions(+), 291 deletions(-) diff --git a/host/Src/main.cpp b/host/Src/main.cpp index a526080..86b65aa 100644 --- a/host/Src/main.cpp +++ b/host/Src/main.cpp @@ -1,12 +1,13 @@ -#include #include +#include #include "HaarPath.hpp" #include "gf/opencv_utils.hpp" cv::String haarPath = HAAR_PATH; -cv::String face_cascade_name = haarPath + "lbpcascades/lbpcascade_frontalface_improved.xml"; +cv::String face_cascade_name = + haarPath + "lbpcascades/lbpcascade_frontalface_improved.xml"; cv::String smile_cascade_name = haarPath + "haarcascades/haarcascade_smile.xml"; cv::CascadeClassifier face_cascade; cv::CascadeClassifier smile_cascade; @@ -16,110 +17,116 @@ cv::String window_name_smile = "Smile"; using namespace gf_cv; bool checkFrameSizeSupported(cv::Mat &gray) { - if (gray.cols == 640 && gray.rows == 480) { - return true; - } - - if (gray.cols == 320 && gray.rows == 240) { - return true; - } - return false; + if (gray.cols == 640 && gray.rows == 480) { + return true; + } + + if (gray.cols == 320 && gray.rows == 240) { + return true; + } + return false; } -int main(int, char**) { - // open the first webcam plugged in the computer - cv::VideoCapture camera(0); // in linux check $ ls /dev/video0 - if (!camera.isOpened()) { - std::cerr << "ERROR: Could not open camera" << std::endl; - return 1; +int main(int, char **) { + // open the first webcam plugged in the computer + cv::VideoCapture camera(0); // in linux check $ ls /dev/video0 + if (!camera.isOpened()) { + std::cerr << "ERROR: Could not open camera" << std::endl; + return 1; + } + + // array to hold image + cv::Mat frameOriginal; + cv::Mat frameScreenGray; + + if (!face_cascade.load(face_cascade_name)) { + printf("--(!)Error loading face cascade\n"); + return -1; + }; + if (!smile_cascade.load(smile_cascade_name)) { + printf("--(!)Error loading eyes cascade\n"); + return -1; + }; + + cv::namedWindow(window_name, cv::WND_PROP_ASPECT_RATIO); + // display the frame until you press a key + + std::cout << "Start capturing" << std::endl; + std::map smileROIKernelParamsScore; + while (camera.read(frameOriginal)) { + + if (frameOriginal.empty()) { + printf(" --(!) No captured frame -- Break!"); + break; } - // array to hold image - cv::Mat frameOriginal; - cv::Mat frameScreenGray; - - if (!face_cascade.load(face_cascade_name)) - { - printf("--(!)Error loading face cascade\n"); - return -1; - }; - if (!smile_cascade.load(smile_cascade_name)) - { - printf("--(!)Error loading eyes cascade\n"); - return -1; - }; - - cv::namedWindow(window_name, cv::WND_PROP_ASPECT_RATIO); - // display the frame until you press a key - - std::cout << "Start capturing" << std::endl; - ROIKernelParamsSweep smileROIKernelParams; - std::map smileROIKernelParamsScore; - while (camera.read(frameOriginal)) { - - if (frameOriginal.empty()) - { - printf(" --(!) No captured frame -- Break!"); - break; - } + cv::Mat frameOriginalGray; + cv::cvtColor(frameOriginal, frameOriginalGray, cv::COLOR_BGR2GRAY); + // cv::equalizeHist(frameOriginalGray, frameOriginalGray); - cv::Mat frameOriginalGray; - cv::cvtColor(frameOriginal, frameOriginalGray, cv::COLOR_BGR2GRAY); - //cv::equalizeHist(frameOriginalGray, frameOriginalGray); + if (false == checkFrameSizeSupported(frameOriginalGray)) { + std::cerr << "Not supported frame size" << std::endl; + exit(-1); + } - if (false == checkFrameSizeSupported(frameOriginalGray)) { - std::cerr << "Not supported frame size" << std::endl; - exit(-1); - } + cv::Mat frameCroppedGray; + + if (frameOriginalGray.cols == 640 && frameOriginalGray.rows == 480) { + cv::Rect crop; + crop.width = 320; + crop.height = 240; + crop.x = (frameOriginalGray.cols - crop.width) / 2; + crop.y = (frameOriginalGray.rows - crop.height) / 2; + frameCroppedGray = frameOriginalGray(crop); + } else { + frameCroppedGray = frameOriginalGray; + } - cv::Mat frameCroppedGray; - - if (frameOriginalGray.cols == 640 && frameOriginalGray.rows == 480) { - cv::Rect crop; - crop.width = 320; - crop.height = 240; - crop.x = (frameOriginalGray.cols - crop.width) / 2; - crop.y = (frameOriginalGray.rows - crop.height) / 2; - frameCroppedGray = frameOriginalGray(crop); - } else { - frameCroppedGray = frameOriginalGray; - } + cv::Rect faceROIMax{}; + faceROIMax.width = 64; + faceROIMax.height = 40; - cv::Rect faceROIMax{}; - faceROIMax.width = 64; - faceROIMax.height = 40; - - cv::resize(frameCroppedGray, frameScreenGray, cv::Size(80, 80), 0.0f, 0.0f, cv::INTER_NEAREST ); - - std::vector faces, smiles; - - smileROIKernelParams.clahe.clipLimit = Range{4.0, 4.0, 1.0}; - smileROIKernelParams.clahe.tileSize = Range{4, 4, 1}; - smileROIKernelParams.biFilter.d = Range{3, 3, 1}; - smileROIKernelParams.biFilter.sigmaColor = Range{10.0, 10.0, 1.0}; - smileROIKernelParams.biFilter.sigmaSpace = Range{50.0, 50.0, 1.0}; - smileROIKernelParams.gamma.gamma = Range{0.44, 0.50, 0.02}; - smileROIKernelParams.blur.sigmaX = Range{0.82, 0.87, 0.01}; - - auto rects = detectFaceAndSmile( - face_cascade, - smile_cascade, - frameScreenGray, - frameCroppedGray, - faceROIMax, - smileROIKernelParams - ); - - for (const auto &rect : rects) { - rectangle(frameScreenGray, rect, cv::Scalar(0, 0, 255), 1, 1, 0); - } + cv::resize(frameCroppedGray, frameScreenGray, cv::Size(80, 80), 0.0f, 0.0f, + cv::INTER_NEAREST); + + std::vector faces, smiles; + + auto sigmaX = Range{0.82, 0.87, 0.01}; + BlurStage blurStage = BlurStage(sigmaX); - cv::imshow(window_name, frameScreenGray); + auto gamma = Range{0.44, 0.50, 0.02}; + GammaStage gammaStage = GammaStage(gamma); - int c = cv::waitKey(1); - if ((char)c == 27) { break; } + auto d = Range{3, 3, 1}; + auto sigmaColor = Range{10.0, 10.0, 1.0}; + auto sigmaSpace = Range{50.0, 50.0, 1.0}; + BilateralStage bilateralStage = BilateralStage(d, sigmaColor, sigmaSpace); - continue; + auto clipLimit = Range{4.0, 4.0, 1.0}; + auto tileSize = Range{4, 4, 1}; + ClaheStage claheStage = ClaheStage(clipLimit, tileSize); + + claheStage.setNextStage(&bilateralStage); + bilateralStage.setNextStage(&gammaStage); + gammaStage.setNextStage(nullptr); + // blurStage.setNextStage(nullptr); + + auto rects = + detectFaceAndSmile(face_cascade, smile_cascade, frameScreenGray, + frameCroppedGray, faceROIMax, claheStage); + + for (const auto &rect : rects) { + rectangle(frameScreenGray, rect, cv::Scalar(0, 0, 255), 1, 1, 0); + } + + cv::imshow(window_name, frameScreenGray); + + int c = cv::waitKey(1); + if ((char)c == 27) { + break; + } + + continue; #if 0 if (faces.empty()) { @@ -187,17 +194,19 @@ int main(int, char**) { } } #endif - } + } - for (auto & [key, score] : smileROIKernelParamsScore) { - std::cout << "key = " << key << ", score = " << score << std::endl; - } + for (auto &[key, score] : smileROIKernelParamsScore) { + std::cout << "key = " << key << ", score = " << score << std::endl; + } - return 0; + return 0; } -// CMD to generate executable: -// g++ webcam_opencv.cpp -o webcam_demo -I/usr/include/opencv4 -lopencv_core -lopencv_videoio -lopencv_highgui +// CMD to generate executable: +// g++ webcam_opencv.cpp -o webcam_demo -I/usr/include/opencv4 -lopencv_core +// -lopencv_videoio -lopencv_highgui -// Note: check your opencv hpp files - for many users it is at /usr/local/include/opencv4 -// Add more packages during compilation from the list obtained by $ pkg-config --cflags --libs opencv4 +// Note: check your opencv hpp files - for many users it is at +// /usr/local/include/opencv4 Add more packages during compilation from the list +// obtained by $ pkg-config --cflags --libs opencv4 diff --git a/u5/common/opencv/include/gf/opencv_pipeline.hpp b/u5/common/opencv/include/gf/opencv_pipeline.hpp index 9e3892a..976b477 100644 --- a/u5/common/opencv/include/gf/opencv_pipeline.hpp +++ b/u5/common/opencv/include/gf/opencv_pipeline.hpp @@ -1,12 +1,43 @@ -#include "opencv_utils.hpp" +#include + +#include "opencv2/opencv.hpp" #pragma once namespace gf_cv { +template struct Range { + static_assert(std::is_arithmetic::value, + "struct Range: T must be arithmetic"); + + T start, stop, step; + + struct sentinel {}; + + struct iterator { + T value, stop, step; + + T operator*() const { return value; }; + iterator &operator++() { + value += step; + return *this; + } + bool operator!=(sentinel) { return value <= stop; } + bool operator==(sentinel) { return value > stop; } + }; + + iterator begin() const { + assert(step > T(0) && "Range: step must be > 0"); + return {start, stop, step}; + } + + sentinel end() const { return {}; } +}; + class Stage { public: + using InvokeCallback = std::function; explicit Stage() = default; virtual ~Stage() = default; @@ -29,12 +60,30 @@ class Stage { virtual cv::Mat get() = 0; virtual void next() = 0; virtual bool hasNext() = 0; + + Stage &setNextStage(Stage *nextStage); + bool invoke(cv::Mat &in, InvokeCallback callback); + void setInput(cv::Mat &in); + +protected: + Stage *nextStage{nullptr}; + cv::Mat in; }; +class DummyStage : public Stage { +public: + explicit DummyStage() = default; + ~DummyStage() = default; + cv::Mat get() override { return cv::Mat(); } + void next() override {} + bool hasNext() override { return false; } +}; + +/* CLAHE */ class ClaheStage : public Stage { public: - explicit ClaheStage(cv::Mat &in, Range &clipLimit, - Range &tileSize); + explicit ClaheStage(Range &clipLimit, Range &tileSize); + ~ClaheStage() = default; cv::Mat get() override; @@ -42,7 +91,6 @@ class ClaheStage : public Stage { bool hasNext() override; private: - cv::Mat ∈ Range &clipLimit; Range &tileSize; @@ -52,7 +100,7 @@ class ClaheStage : public Stage { class BilateralStage : public Stage { public: - explicit BilateralStage(cv::Mat &in, Range &d, Range &sigmaColor, + explicit BilateralStage(Range &d, Range &sigmaColor, Range &sigmaSpace); ~BilateralStage() = default; @@ -61,7 +109,6 @@ class BilateralStage : public Stage { bool hasNext() override; private: - cv::Mat ∈ Range &d; Range &sigmaColor; Range &sigmaSpace; @@ -73,7 +120,7 @@ class BilateralStage : public Stage { class GammaStage : public Stage { public: - explicit GammaStage(cv::Mat &in, Range &gamma); + explicit GammaStage(Range &gamma); ~GammaStage() = default; cv::Mat get() override; @@ -81,7 +128,6 @@ class GammaStage : public Stage { bool hasNext() override; private: - cv::Mat ∈ Range γ Range::iterator gammaIt; @@ -89,7 +135,7 @@ class GammaStage : public Stage { class BlurStage : public Stage { public: - explicit BlurStage(cv::Mat &in, Range &sigmaX); + explicit BlurStage(Range &sigmaX); ~BlurStage() = default; cv::Mat get() override; @@ -97,7 +143,6 @@ class BlurStage : public Stage { bool hasNext() override; private: - cv::Mat ∈ Range &sigmaX; Range::iterator sigmaXIt; diff --git a/u5/common/opencv/include/gf/opencv_utils.hpp b/u5/common/opencv/include/gf/opencv_utils.hpp index 9df19a0..351c8c1 100644 --- a/u5/common/opencv/include/gf/opencv_utils.hpp +++ b/u5/common/opencv/include/gf/opencv_utils.hpp @@ -3,7 +3,7 @@ #include -#include "opencv2/opencv.hpp" +#include "gf/opencv_pipeline.hpp" namespace gf_cv { @@ -12,52 +12,6 @@ bool loadCascade(cv::CascadeClassifier &cascade, cv::FileStorage &fs); bool loadCascade(cv::CascadeClassifier &cascade, const uint8_t *buffer, size_t len); -template struct Range { - static_assert(std::is_arithmetic::value, - "struct Range: T must be arithmetic"); - - T start, stop, step; - - struct sentinel {}; - - struct iterator { - T value, stop, step; - - T operator*() const { return value; }; - iterator &operator++() { - value += step; - return *this; - } - bool operator!=(sentinel) { return value <= stop; } - bool operator==(sentinel) { return value > stop; } - }; - - iterator begin() const { - assert(step > T(0) && "Range: step must be > 0"); - return {start, stop, step}; - } - - sentinel end() const { return {}; } -}; - -struct ROIKernelParamsSweep { - struct Clahe { - Range clipLimit; - Range tileSize; - } clahe; - struct BiFilter { - Range d; - Range sigmaColor; - Range sigmaSpace; - } biFilter; - struct Gamma { - Range gamma; - } gamma; - struct Blur { - Range sigmaX; - } blur; -}; - /** * @brief * @@ -72,11 +26,11 @@ struct ROIKernelParamsSweep { * @param roIKernelParamsSweep * @return vector of cv::Rects for detected objects */ -std::vector -detectFaceAndSmile(cv::CascadeClassifier &faceCascade, - cv::CascadeClassifier &smileCascade, cv::Mat &thumbnailFrame, - cv::Mat &fullFrame, cv::Rect &faceROIMax, - ROIKernelParamsSweep &roIKernelParamsSweep); +std::vector detectFaceAndSmile(cv::CascadeClassifier &faceCascade, + cv::CascadeClassifier &smileCascade, + cv::Mat &thumbnailFrame, + cv::Mat &fullFrame, + cv::Rect &faceROIMax, Stage &pipeline); cv::Mat cv_preprocessForQR(const cv::Mat &bgr); diff --git a/u5/common/opencv/src/opencv_pipeline.cpp b/u5/common/opencv/src/opencv_pipeline.cpp index 0876df0..15196e4 100644 --- a/u5/common/opencv/src/opencv_pipeline.cpp +++ b/u5/common/opencv/src/opencv_pipeline.cpp @@ -3,9 +3,28 @@ namespace gf_cv { -ClaheStage::ClaheStage(cv::Mat &in, Range &clipLimit, - Range &tileSize) - : in(in), clipLimit(clipLimit), tileSize(tileSize) { +Stage &Stage::setNextStage(Stage *_nextStage) { + nextStage = _nextStage; + return *this; +} + +bool Stage::invoke(cv::Mat &_in, InvokeCallback callback) { + if (!nextStage) { + return callback(_in); + } + setInput(_in); + for (auto mat : *this) { + if (nextStage->invoke(mat, callback)) { + return true; + } + } + return false; +} + +void Stage::setInput(cv::Mat &_in) { in = _in; } + +ClaheStage::ClaheStage(Range &clipLimit, Range &tileSize) + : clipLimit(clipLimit), tileSize(tileSize) { clipLimitIt = clipLimit.begin(); tileSizeIt = tileSize.begin(); @@ -31,10 +50,9 @@ bool ClaheStage::hasNext() { return (clipLimitIt != clipLimit.end()) && (tileSizeIt != tileSize.end()); } -BilateralStage::BilateralStage(cv::Mat &in, Range &d, - Range &sigmaColor, +BilateralStage::BilateralStage(Range &d, Range &sigmaColor, Range &sigmaSpace) - : in(in), d(d), sigmaColor(sigmaColor), sigmaSpace(sigmaSpace) { + : d(d), sigmaColor(sigmaColor), sigmaSpace(sigmaSpace) { dIt = d.begin(); sigmaColorIt = sigmaColor.begin(); @@ -64,8 +82,7 @@ bool BilateralStage::hasNext() { (sigmaSpaceIt != sigmaSpace.end()); } -GammaStage::GammaStage(cv::Mat &in, Range &gamma) - : in(in), gamma(gamma) { +GammaStage::GammaStage(Range &gamma) : gamma(gamma) { gammaIt = gamma.begin(); } @@ -81,8 +98,7 @@ void GammaStage::next() { gammaIt = ++gammaIt; } bool GammaStage::hasNext() { return (gammaIt != gamma.end()); } -BlurStage::BlurStage(cv::Mat &in, Range &sigmaX) - : in(in), sigmaX(sigmaX) { +BlurStage::BlurStage(Range &sigmaX) : sigmaX(sigmaX) { sigmaXIt = sigmaX.begin(); } diff --git a/u5/common/opencv/src/opencv_utils.cpp b/u5/common/opencv/src/opencv_utils.cpp index b258622..234cd69 100644 --- a/u5/common/opencv/src/opencv_utils.cpp +++ b/u5/common/opencv/src/opencv_utils.cpp @@ -74,79 +74,10 @@ static bool detectSmile(cv::CascadeClassifier &cascade, cv::Mat &in, return !rects.empty(); } -static bool sweepBlur(cv::CascadeClassifier &cascade, cv::Mat &in, - std::vector &rects, - ROIKernelParamsSweep ¶ms) { - - BlurStage blurStage = BlurStage(in, params.blur.sigmaX); - - for (auto blur : blurStage) { - if (detectSmile(cascade, blur, rects)) { - return true; - } - } - - return false; -} - -static bool sweepGamma(cv::CascadeClassifier &cascade, cv::Mat &in, - std::vector &rects, - ROIKernelParamsSweep ¶ms) { - - GammaStage gammaStage = GammaStage(in, params.gamma.gamma); - - for (auto gamma : gammaStage) { - if (sweepBlur(cascade, gamma, rects, params)) { - return true; - } - } - return false; -} - -static bool sweepBilateral(cv::CascadeClassifier &cascade, cv::Mat &in, - std::vector &rects, - ROIKernelParamsSweep ¶ms) { - - BilateralStage bilateralStage = - BilateralStage(in, params.biFilter.d, params.biFilter.sigmaColor, - params.biFilter.sigmaSpace); - - for (auto bilateral : bilateralStage) { - if (sweepGamma(cascade, bilateral, rects, params)) { - return true; - } - } - return false; -} - -static bool sweepClahe(cv::CascadeClassifier &cascade, cv::Mat &in, - std::vector &rects, - ROIKernelParamsSweep ¶ms) { - - ClaheStage claheStage = - ClaheStage(in, params.clahe.clipLimit, params.clahe.tileSize); - - for (auto clahe : claheStage) { - if (sweepBilateral(cascade, clahe, rects, params)) { - return true; - } - } - - return false; -} - -static void detectSmile(cv::CascadeClassifier &cascade, cv::Mat &in, - std::vector &rects, - ROIKernelParamsSweep ¶ms) { - - sweepClahe(cascade, in, rects, params); -} - std::vector detectFaceAndSmile(cv::CascadeClassifier &faceCascade, cv::CascadeClassifier &smileCascade, cv::Mat &thumbnailFrame, - cv::Mat &fullFrame, cv::Rect &faceROIMax, - ROIKernelParamsSweep &roIKernelParamsSweep) { + cv::Mat &fullFrame, cv::Rect &faceROIMax, Stage &pipeline) { std::vector rects; @@ -198,9 +129,16 @@ detectFaceAndSmile(cv::CascadeClassifier &faceCascade, } std::vector smileRects; - detectSmile(smileCascade, faceROIFrameResized, smileRects, - roIKernelParamsSweep); - // detectSmile(smileCascade, faceROIFrameResized, smileRects); + + detectSmile(smileCascade, faceROIFrameResized, smileRects); + + // pipeline.invoke(faceROIFrameResized, [&](cv::Mat &in) -> bool { + // return detectSmile(smileCascade, in, smileRects); + // }); + + // detectSmile(smileCascade, faceROIFrameResized, smileRects, + // roIKernelParamsSweep); + // detectSmile(smileCascade, faceROIFrameResized, smileRects); float ssx_inv = (float)thumbnailFrame.cols / (float)fullFrame.cols; float ssy_inv = (float)thumbnailFrame.rows / (float)fullFrame.rows; diff --git a/u5/common/src/video.cpp b/u5/common/src/video.cpp index 2f708b8..51762cb 100644 --- a/u5/common/src/video.cpp +++ b/u5/common/src/video.cpp @@ -18,6 +18,8 @@ static struct video_buffer *buffers[2]; #if defined(CONFIG_GRINREFLEX_JPEG_VIDEO) static struct video_buffer second_buffer {}; +__attribute__((aligned(32))) +static uint8_t video_large_buffer[CONFIG_GRINREFLEX_VIDEO_WIDTH * CONFIG_GRINREFLEX_VIDEO_HEIGHT * 2]; #endif void setup() { @@ -143,10 +145,9 @@ void setup() { buffers[1] = &second_buffer; - bsize = CONFIG_GRINREFLEX_VIDEO_WIDTH * CONFIG_GRINREFLEX_VIDEO_HEIGHT * 2; buffers[1]->type = type; - buffers[1]->buffer = new uint8_t[bsize]; - buffers[1]->size = bsize; + buffers[1]->buffer = video_large_buffer; + buffers[1]->size = sizeof(video_large_buffer); if (buffers[1]->buffer == NULL) { LOG_ERR("Unable to alloc video buffer"); diff --git a/u5/drivers/jpeg/jpeg.c b/u5/drivers/jpeg/jpeg.c index eff7a50..d6a6b00 100644 --- a/u5/drivers/jpeg/jpeg.c +++ b/u5/drivers/jpeg/jpeg.c @@ -64,8 +64,8 @@ void HAL_JPEG_InfoReadyCallback(JPEG_HandleTypeDef *hjpeg, data->out_prop.height = pInfo->ImageWidth; data->out_prop.quality = pInfo->ImageQuality; - //LOG_INF("ColorSpace = %u, ChromaSubsampling = %u", pInfo->ColorSpace, - // pInfo->ChromaSubsampling); + // LOG_INF("ColorSpace = %u, ChromaSubsampling = %u", pInfo->ColorSpace, + // pInfo->ChromaSubsampling); // LOG_INF("HAL_JPEG_InfoReadyCallback"); } @@ -277,9 +277,9 @@ static int _jpeg_hw_decode(const struct device *dev, const uint8_t *src, &dev_data->hjpeg, (uint8_t *)dev_data->in_prop.src_addr, CHUNK_SIZE_IN, (uint8_t *)dev_data->in_prop.dst_addr, CHUNK_SIZE_OUT); - //LOG_INF("JPEG_Decode_DMA : src = %x dst = %x size = %d ret = %d", - // dev_data->in_prop.src_addr, dev_data->in_prop.dst_addr, - // dev_data->in_prop.src_size, ret); + // LOG_INF("JPEG_Decode_DMA : src = %x dst = %x size = %d ret = %d", + // dev_data->in_prop.src_addr, dev_data->in_prop.dst_addr, + // dev_data->in_prop.src_size, ret); return ret == HAL_OK ? 0 : -EINVAL; } @@ -357,8 +357,8 @@ static int _jpeg_color_convert_helper(const struct device *dev, pInfo.ImageWidth = prop->height; pInfo.ImageQuality = prop->quality; - //LOG_INF("ColorSpace = %u, ChromaSubsampling = %u", pInfo.ColorSpace, - // pInfo.ChromaSubsampling); + // LOG_INF("ColorSpace = %u, ChromaSubsampling = %u", pInfo.ColorSpace, + // pInfo.ChromaSubsampling); JPEG_YCbCrToRGB_Convert_Function convert_function; uint32_t ImageNbMCUs = 0; @@ -373,7 +373,7 @@ static int _jpeg_color_convert_helper(const struct device *dev, uint32_t ConvertedDataCount = 0; uint32_t inSize = _jpeg_get_in_size(&pInfo, ImageNbMCUs); - //LOG_INF("ImageNbMCUs = %d size = %d", ImageNbMCUs, inSize); + // LOG_INF("ImageNbMCUs = %d size = %d", ImageNbMCUs, inSize); if (0 == inSize) { return -EINVAL; @@ -382,30 +382,28 @@ static int _jpeg_color_convert_helper(const struct device *dev, uint32_t NrBlocks = convert_function((uint8_t *)src, dst, 0, inSize, &ConvertedDataCount); - //LOG_INF("Decoded image : NrBlocks = %d, size = %d", NrBlocks, - // ConvertedDataCount); + // LOG_INF("Decoded image : NrBlocks = %d, size = %d", NrBlocks, + // ConvertedDataCount); return NrBlocks ? NrBlocks : -EINVAL; } static int _jpeg_register_cplt_callback(const struct device *dev, - const struct device *parent_dev, - jpeg_cplt_callback_t callback) -{ - struct jpeg_hw_data *dev_data = dev->data; - /* Might be DCMI or any other source of jpeg frame */ - dev_data->parent_dev = parent_dev; - dev_data->cplt_callback = callback; + const struct device *parent_dev, + jpeg_cplt_callback_t callback) { + struct jpeg_hw_data *dev_data = dev->data; + /* Might be DCMI or any other source of jpeg frame */ + dev_data->parent_dev = parent_dev; + dev_data->cplt_callback = callback; - return 0; + return 0; } static DEVICE_API(jpeg_hw, jpeg_hw_driver_api) = { .decode = _jpeg_hw_decode, .poll = _jpeg_hw_poll, .cc_helper = _jpeg_color_convert_helper, - .register_cplt_callback = _jpeg_register_cplt_callback -}; + .register_cplt_callback = _jpeg_register_cplt_callback}; #define JPEG_DMA_CHANNEL_INIT(stream, index, dir, src_dev, dest_dev) \ .stream = { \ diff --git a/u5/gf_dkx/prj.conf b/u5/gf_dkx/prj.conf index c0937b5..9190576 100644 --- a/u5/gf_dkx/prj.conf +++ b/u5/gf_dkx/prj.conf @@ -31,9 +31,9 @@ CONFIG_NEWLIB_LIBC=y CONFIG_PICOLIBC=n CONFIG_COMMON_LIBC_MALLOC=y -CONFIG_HEAP_MEM_POOL_SIZE=49152 -CONFIG_COMMON_LIBC_MALLOC_ARENA_SIZE=2097152 -#CONFIG_COMMON_LIBC_MALLOC_ARENA_SIZE=1048576 +CONFIG_HEAP_MEM_POOL_SIZE=8192 +#CONFIG_COMMON_LIBC_MALLOC_ARENA_SIZE=2097152 +CONFIG_COMMON_LIBC_MALLOC_ARENA_SIZE=1503232 CONFIG_POSIX_API=y @@ -88,7 +88,7 @@ CONFIG_VIDEO_SHELL=y CONFIG_STM32_JPEG_RGB_FORMAT=3 CONFIG_GRINREFLEX_JPEG_VIDEO=y -CONFIG_GRINREFLEX_JPEG_VIDEO_QUALITY=50 +CONFIG_GRINREFLEX_JPEG_VIDEO_QUALITY=20 CONFIG_GRINREFLEX_VIDEO_WIDTH=640 CONFIG_GRINREFLEX_VIDEO_HEIGHT=480 diff --git a/u5/gf_dkx/src/lvgl_video_app.cpp b/u5/gf_dkx/src/lvgl_video_app.cpp index 0333c88..674cd42 100644 --- a/u5/gf_dkx/src/lvgl_video_app.cpp +++ b/u5/gf_dkx/src/lvgl_video_app.cpp @@ -103,8 +103,17 @@ int loop() { jpeg_color_convert_helper(jpeg_dev, &jpeg_prop, vbuf_ptr->buffer, fullFrameBuffer); - cropFullFrameToRoi(dummy_canvas, fullFrameBuffer, roiFrameBuffer); - fitRoiFrameToThumbnail(screen_canvas, roiFrameBuffer, thumbnailFrameBuffer); + cropFullFrameToRoi(dummy_canvas, fullFrameBuffer, roiFrameBuffer, + lvgl::Size(FULL_FRAME_WIDTH, FULL_FRAME_HEIGHT), + lvgl::Size(ROI_FRAME_WIDTH, ROI_FRAME_HEIGHT), + FULL_FRAME_COLOR_FORMAT, LV_COLOR_FORMAT_L8, + FULL_FRAME_FLIP_Y); + + fitRoiFrameToThumbnail( + screen_canvas, roiFrameBuffer, thumbnailFrameBuffer, + lvgl::Size(ROI_FRAME_WIDTH, ROI_FRAME_HEIGHT), + lvgl::Size(THUMBNAIL_FRAME_WIDTH, THUMBNAIL_FRAME_HEIGHT), + LV_COLOR_FORMAT_L8, LV_COLOR_FORMAT_L8); lv_task_handler(); err = video_enqueue(video_dev, vbuf_ptr); diff --git a/u5/gf_dkx/src/opencv_fd_app.cpp b/u5/gf_dkx/src/opencv_fd_app.cpp index 3cd1c3d..2d8d0bc 100644 --- a/u5/gf_dkx/src/opencv_fd_app.cpp +++ b/u5/gf_dkx/src/opencv_fd_app.cpp @@ -61,8 +61,6 @@ static lv_obj_t *ROIRectSmile; static cv::CascadeClassifier faceCascade; static cv::CascadeClassifier smileCascade; -ROIKernelParamsSweep smileROIKernelParams; - int init() { if (!device_is_ready(display_dev)) { LOG_ERR("Display device not ready, aborting test"); @@ -118,14 +116,6 @@ int init() { ROIRectSmile = allocLvROIRect(lv_scr_act(), 4, lv_palette_main(LV_PALETTE_RED)); - smileROIKernelParams.clahe.clipLimit = Range(4.0, 4.0, 1.0); - smileROIKernelParams.clahe.tileSize = Range(4, 4, 1); - smileROIKernelParams.biFilter.d = Range(3, 3, 1); - smileROIKernelParams.biFilter.sigmaColor = Range(20.0, 20.0, 1.0); - smileROIKernelParams.biFilter.sigmaSpace = Range(50.0, 50.0, 1.0); - smileROIKernelParams.gamma.gamma = Range(0.44, 0.50, 0.1); - smileROIKernelParams.blur.sigmaX = Range(0.75, 0.85, 0.05); - return 0; } @@ -183,10 +173,28 @@ int loop() { // But that may miss smile area cv::Rect faceROIMax(0, 0, 64, 40); + auto sigmaX = Range{0.75, 0.85, 0.05}; + BlurStage blurStage = BlurStage(sigmaX); + + auto _gamma = Range{0.35, 0.55, 0.05}; + GammaStage gammaStage = GammaStage(_gamma); + + auto d = Range{3, 3, 1}; + auto sigmaColor = Range{10.0, 30.0, 10.0}; + auto sigmaSpace = Range{50.0, 50.0, 1.0}; + BilateralStage bilateralStage = BilateralStage(d, sigmaColor, sigmaSpace); + + auto clipLimit = Range{4.0, 4.0, 1.0}; + auto tileSize = Range{4, 4, 1}; + ClaheStage claheStage = ClaheStage(clipLimit, tileSize); + claheStage.setNextStage(&bilateralStage); + bilateralStage.setNextStage(&gammaStage); + gammaStage.setNextStage(nullptr); + // blurStage.setNextStage(nullptr); + cv::equalizeHist(matGraySmall, matGraySmall); - auto rects = - detectFaceAndSmile(faceCascade, smileCascade, matGraySmall, matGrayFull, - faceROIMax, smileROIKernelParams); + auto rects = detectFaceAndSmile(faceCascade, smileCascade, matGraySmall, + matGrayFull, faceROIMax, claheStage); cv::Rect face, smile; if (rects.size()) { From 8bfa3276b5d083cc2ac89da878ee9c63a1aad9e7 Mon Sep 17 00:00:00 2001 From: Roman Pustobaiev Date: Fri, 17 Oct 2025 22:42:01 +0200 Subject: [PATCH 07/18] Some fine tuning Adding release overlay to rip off display --- u5/common/opencv/src/opencv_pipeline.cpp | 2 +- u5/common/opencv/src/opencv_utils.cpp | 8 +++---- u5/gf_dkx/CMakeLists.txt | 4 ++++ u5/gf_dkx/Kconfig | 5 ++++ u5/gf_dkx/overlays/release.overlay | 14 ++++++++++++ u5/gf_dkx/src/opencv_fd_app.cpp | 29 +++++++++++++----------- 6 files changed, 43 insertions(+), 19 deletions(-) create mode 100644 u5/gf_dkx/overlays/release.overlay diff --git a/u5/common/opencv/src/opencv_pipeline.cpp b/u5/common/opencv/src/opencv_pipeline.cpp index 15196e4..9bdfbb9 100644 --- a/u5/common/opencv/src/opencv_pipeline.cpp +++ b/u5/common/opencv/src/opencv_pipeline.cpp @@ -5,7 +5,7 @@ namespace gf_cv { Stage &Stage::setNextStage(Stage *_nextStage) { nextStage = _nextStage; - return *this; + return *_nextStage; } bool Stage::invoke(cv::Mat &_in, InvokeCallback callback) { diff --git a/u5/common/opencv/src/opencv_utils.cpp b/u5/common/opencv/src/opencv_utils.cpp index 234cd69..b06e3b8 100644 --- a/u5/common/opencv/src/opencv_utils.cpp +++ b/u5/common/opencv/src/opencv_utils.cpp @@ -130,11 +130,9 @@ detectFaceAndSmile(cv::CascadeClassifier &faceCascade, std::vector smileRects; - detectSmile(smileCascade, faceROIFrameResized, smileRects); - - // pipeline.invoke(faceROIFrameResized, [&](cv::Mat &in) -> bool { - // return detectSmile(smileCascade, in, smileRects); - // }); + pipeline.invoke(faceROIFrameResized, [&](cv::Mat &in) -> bool { + return detectSmile(smileCascade, in, smileRects); + }); // detectSmile(smileCascade, faceROIFrameResized, smileRects, // roIKernelParamsSweep); diff --git a/u5/gf_dkx/CMakeLists.txt b/u5/gf_dkx/CMakeLists.txt index 820e360..dfaf1ce 100644 --- a/u5/gf_dkx/CMakeLists.txt +++ b/u5/gf_dkx/CMakeLists.txt @@ -10,6 +10,10 @@ set(EXTRA_CONF_FILE ${CMAKE_CURRENT_SOURCE_DIR}/lvgl.conf; ) +if (CONFIG_VARIANT_RELEASE) +set(DTC_OVERLAY_FILE ${DTC_OVERLAY_FILE};${CMAKE_CURRENT_SOURCE_DIR}/overlays/release.overlay CACHE STRING "" FORCE) +endif() + find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) project(gf_dkx) diff --git a/u5/gf_dkx/Kconfig b/u5/gf_dkx/Kconfig index 46f6b7b..be97d2d 100644 --- a/u5/gf_dkx/Kconfig +++ b/u5/gf_dkx/Kconfig @@ -55,4 +55,9 @@ config FLASH_LOG_LEVEL endif # DISK_DRIVER_FLASH +config VARIANT_RELEASE + bool "Use release variant, strip everything that is unnecessary" + default n + depends on BOARD_GRINREFLEX_DK2 + endmenu \ No newline at end of file diff --git a/u5/gf_dkx/overlays/release.overlay b/u5/gf_dkx/overlays/release.overlay new file mode 100644 index 0000000..502a2fe --- /dev/null +++ b/u5/gf_dkx/overlays/release.overlay @@ -0,0 +1,14 @@ + +/ { + + chosen { + zephyr,display = &zephyr_dc; + }; + + zephyr_dc: zephyr_dc { + compatible = "zephyr,dummy-dc"; + width = <80>; + height = <80>; + }; + +}; \ No newline at end of file diff --git a/u5/gf_dkx/src/opencv_fd_app.cpp b/u5/gf_dkx/src/opencv_fd_app.cpp index 2d8d0bc..d2e74bf 100644 --- a/u5/gf_dkx/src/opencv_fd_app.cpp +++ b/u5/gf_dkx/src/opencv_fd_app.cpp @@ -36,9 +36,12 @@ LOG_MODULE_REGISTER(grinreflex_app); using namespace gf_cv; static const struct device *video_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_camera)); + static const struct device *display_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_display)); + const struct device *jpeg_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_jpeg)); + #if DT_NODE_EXISTS(DT_ALIAS(led0)) static const struct gpio_dt_spec led = GPIO_DT_SPEC_GET(DT_ALIAS(led0), gpios); #endif @@ -54,7 +57,6 @@ static uint8_t static lv_obj_t *screen_canvas; static lv_obj_t *dummy_canvas; -static lv_obj_t *screen; static lv_obj_t *ROIRectFace; static lv_obj_t *ROIRectSmile; @@ -109,8 +111,6 @@ int init() { display_blanking_off(display_dev); - screen = lv_img_create(lv_scr_act()); - ROIRectFace = allocLvROIRect(lv_scr_act(), 4, lv_palette_main(LV_PALETTE_GREEN)); ROIRectSmile = @@ -171,26 +171,29 @@ int loop() { // This is the best configuration I could find, 64 x 30 works even better // But that may miss smile area - cv::Rect faceROIMax(0, 0, 64, 40); + auto smileCascadeSize = smileCascade.getOriginalWindowSize(); + cv::Rect faceROIMax(0, 0, smileCascadeSize.width * 2, + smileCascadeSize.height * 2); - auto sigmaX = Range{0.75, 0.85, 0.05}; + auto sigmaX = Range{0.85, 0.85, 0.01}; BlurStage blurStage = BlurStage(sigmaX); - auto _gamma = Range{0.35, 0.55, 0.05}; + auto _gamma = Range{0.30, 0.50, 0.05}; GammaStage gammaStage = GammaStage(_gamma); auto d = Range{3, 3, 1}; - auto sigmaColor = Range{10.0, 30.0, 10.0}; + auto sigmaColor = Range{20.0, 20.0, 10.0}; auto sigmaSpace = Range{50.0, 50.0, 1.0}; BilateralStage bilateralStage = BilateralStage(d, sigmaColor, sigmaSpace); - auto clipLimit = Range{4.0, 4.0, 1.0}; + auto clipLimit = Range{5.0, 5.0, 1.0}; auto tileSize = Range{4, 4, 1}; ClaheStage claheStage = ClaheStage(clipLimit, tileSize); - claheStage.setNextStage(&bilateralStage); - bilateralStage.setNextStage(&gammaStage); - gammaStage.setNextStage(nullptr); - // blurStage.setNextStage(nullptr); + + claheStage.setNextStage(&bilateralStage) + .setNextStage(&gammaStage) + .setNextStage(&blurStage) + .setNextStage(nullptr); cv::equalizeHist(matGraySmall, matGraySmall); auto rects = detectFaceAndSmile(faceCascade, smileCascade, matGraySmall, @@ -218,7 +221,7 @@ int loop() { lv_obj_remove_flag(ROIRectSmile, LV_OBJ_FLAG_HIDDEN); #if DT_NODE_EXISTS(DT_ALIAS(led0)) - gpio_pin_set_dt(&led, 0); + gpio_pin_toggle_dt(&led); #endif } } else { From 05e64c488770a88fc5bc0a8bde985d72a6c05016 Mon Sep 17 00:00:00 2001 From: Roman Pustobaiev Date: Sat, 18 Oct 2025 12:52:04 +0200 Subject: [PATCH 08/18] Opencv : gate 3d party libraries Adding pure opencv frame preprocessing: crop/resize Some memory tweaks --- u5/common/include/gf/utils.hpp | 2 ++ u5/common/src/utils.cpp | 14 +++++++++ u5/gf_dkx/lvgl.conf | 4 +-- u5/gf_dkx/src/config.hpp | 1 + u5/gf_dkx/src/opencv_fd_app.cpp | 56 +++++++++++++++++++++++++++------ u5/lib/cv/CMakeLists.txt | 22 +++++++++++-- u5/lib/cv/Kconfig | 7 ++++- u5/lib/cv/opencv | 2 +- 8 files changed, 92 insertions(+), 16 deletions(-) diff --git a/u5/common/include/gf/utils.hpp b/u5/common/include/gf/utils.hpp index 0c677c6..32f11e8 100644 --- a/u5/common/include/gf/utils.hpp +++ b/u5/common/include/gf/utils.hpp @@ -12,3 +12,5 @@ void cropFullFrameToRoi(lv_obj_t *parent, uint8_t *fullFb, uint8_t *roiFb, void fitRoiFrameToThumbnail(lv_obj_t *parent, uint8_t *roiFb, uint8_t *tFb, lvgl::Size roiSz, lvgl::Size tSz, uint32_t roiCf, uint32_t tCf); + +void displayFrame(lv_obj_t *parent, uint8_t *fb, lvgl::Size sz, uint32_t cf); \ No newline at end of file diff --git a/u5/common/src/utils.cpp b/u5/common/src/utils.cpp index 415b688..8c04146 100644 --- a/u5/common/src/utils.cpp +++ b/u5/common/src/utils.cpp @@ -82,3 +82,17 @@ void fitRoiFrameToThumbnail(lv_obj_t *parent, uint8_t *roiFb, uint8_t *tFb, lvgl::fit(parent, src, dst); // lv_task_handler(); } + +void displayFrame(lv_obj_t *parent, uint8_t *fb, lvgl::Size sz, uint32_t cf) { + static lv_image_dsc_t img_dsc{}; + + img_dsc.header.magic = LV_IMAGE_HEADER_MAGIC; + img_dsc.header.w = sz.width; + img_dsc.header.h = sz.height; + img_dsc.header.stride = sz.width * LV_COLOR_FORMAT_GET_SIZE(cf); + img_dsc.header.cf = cf; /* Set the color format */ + img_dsc.data_size = sz.width * sz.height * LV_COLOR_FORMAT_GET_SIZE(cf); + img_dsc.data = fb; + + lv_img_set_src(parent, &img_dsc); // point the widget to our descriptor +} \ No newline at end of file diff --git a/u5/gf_dkx/lvgl.conf b/u5/gf_dkx/lvgl.conf index c6a38ad..545a8ef 100644 --- a/u5/gf_dkx/lvgl.conf +++ b/u5/gf_dkx/lvgl.conf @@ -4,8 +4,8 @@ CONFIG_LV_Z_MEM_POOL_SIZE=16384 CONFIG_LV_Z_SHELL=n CONFIG_LVGL=y -CONFIG_LV_USE_LOG=y -CONFIG_LV_USE_LABEL=y +CONFIG_LV_USE_LOG=n +CONFIG_LV_USE_LABEL=n CONFIG_LV_USE_ARC=y CONFIG_LV_USE_IMAGE=y #CONFIG_LV_USE_MONKEY=y diff --git a/u5/gf_dkx/src/config.hpp b/u5/gf_dkx/src/config.hpp index fcf2c0c..717d2f3 100644 --- a/u5/gf_dkx/src/config.hpp +++ b/u5/gf_dkx/src/config.hpp @@ -32,4 +32,5 @@ #define THUMBNAIL_FRAME_WIDTH 80 #define THUMBNAIL_FRAME_HEIGHT 80 +#define INPUT_PREPROCESS_OPENCV (0) #define FULL_FRAME_FLIP_Y (0) \ No newline at end of file diff --git a/u5/gf_dkx/src/opencv_fd_app.cpp b/u5/gf_dkx/src/opencv_fd_app.cpp index d2e74bf..202d2cf 100644 --- a/u5/gf_dkx/src/opencv_fd_app.cpp +++ b/u5/gf_dkx/src/opencv_fd_app.cpp @@ -58,6 +58,10 @@ static uint8_t static lv_obj_t *screen_canvas; static lv_obj_t *dummy_canvas; +#if INPUT_PREPROCESS_OPENCV +static lv_obj_t *lvgl_image; +#endif + static lv_obj_t *ROIRectFace; static lv_obj_t *ROIRectSmile; static cv::CascadeClassifier faceCascade; @@ -109,6 +113,11 @@ int init() { lv_canvas_fill_bg(screen_canvas, lv_color_hex3(0xccc), LV_OPA_COVER); lv_obj_center(screen_canvas); +#if INPUT_PREPROCESS_OPENCV + lvgl_image = lv_img_create(lv_scr_act()); + lv_obj_center(lvgl_image); +#endif + display_blanking_off(display_dev); ROIRectFace = @@ -144,6 +153,31 @@ int loop() { jpeg_color_convert_helper(jpeg_dev, &jpeg_prop, vbuf_ptr->buffer, fullFrameBuffer); + cv::Mat matGrayFull(FULL_FRAME_HEIGHT, FULL_FRAME_WIDTH, CV_8UC1, + fullFrameBuffer); + + cv::Mat matGrayRoi(ROI_FRAME_HEIGHT, ROI_FRAME_WIDTH, CV_8UC1, + roiFrameBuffer); + + cv::Mat matGraySmall(THUMBNAIL_FRAME_HEIGHT, THUMBNAIL_FRAME_WIDTH, CV_8UC1, + thumbnailFrameBuffer); + +#if INPUT_PREPROCESS_OPENCV + cv::Rect crop; + crop.width = ROI_FRAME_WIDTH; + crop.height = ROI_FRAME_HEIGHT; + crop.x = (matGrayFull.cols - crop.width) / 2; + crop.y = crop.height; + cv::Mat matCrop = matGrayFull(crop); + matCrop.copyTo(matGrayRoi); + cv::resize(matGrayRoi, matGraySmall, + cv::Size(THUMBNAIL_FRAME_WIDTH, THUMBNAIL_FRAME_HEIGHT), 0.0f, + 0.0f, cv::INTER_NEAREST); + + displayFrame(lvgl_image, thumbnailFrameBuffer, + lvgl::Size(THUMBNAIL_FRAME_WIDTH, THUMBNAIL_FRAME_HEIGHT), + LV_COLOR_FORMAT_L8); +#else cropFullFrameToRoi(dummy_canvas, fullFrameBuffer, roiFrameBuffer, lvgl::Size(FULL_FRAME_WIDTH, FULL_FRAME_HEIGHT), lvgl::Size(ROI_FRAME_WIDTH, ROI_FRAME_HEIGHT), @@ -155,6 +189,8 @@ int loop() { lvgl::Size(ROI_FRAME_WIDTH, ROI_FRAME_HEIGHT), lvgl::Size(THUMBNAIL_FRAME_WIDTH, THUMBNAIL_FRAME_HEIGHT), LV_COLOR_FORMAT_L8, LV_COLOR_FORMAT_L8); + +#endif /* #if INPUT_PREPROCESS_OPENCV */ lv_task_handler(); // Return buffer back, we don't need it anymore, thank you @@ -164,13 +200,8 @@ int loop() { return 0; } - cv::Mat matGraySmall(THUMBNAIL_FRAME_HEIGHT, THUMBNAIL_FRAME_WIDTH, CV_8UC1, - thumbnailFrameBuffer); - cv::Mat matGrayFull(ROI_FRAME_HEIGHT, ROI_FRAME_WIDTH, CV_8UC1, - roiFrameBuffer); - // This is the best configuration I could find, 64 x 30 works even better - // But that may miss smile area + // But that may miss mouth area auto smileCascadeSize = smileCascade.getOriginalWindowSize(); cv::Rect faceROIMax(0, 0, smileCascadeSize.width * 2, smileCascadeSize.height * 2); @@ -195,16 +226,21 @@ int loop() { .setNextStage(&blurStage) .setNextStage(nullptr); - cv::equalizeHist(matGraySmall, matGraySmall); + //cv::equalizeHist(matGraySmall, matGraySmall); auto rects = detectFaceAndSmile(faceCascade, smileCascade, matGraySmall, - matGrayFull, faceROIMax, claheStage); + matGrayRoi, faceROIMax, claheStage); + + lv_obj_t *draw_widget = screen_canvas; +#if INPUT_PREPROCESS_OPENCV + draw_widget = lvgl_image; +#endif cv::Rect face, smile; if (rects.size()) { lv_obj_remove_flag(ROIRectFace, LV_OBJ_FLAG_HIDDEN); face = rects[0]; - lv_obj_align_to(ROIRectFace, screen_canvas, LV_ALIGN_TOP_LEFT, face.x, + lv_obj_align_to(ROIRectFace, draw_widget, LV_ALIGN_TOP_LEFT, face.x, face.y); lv_obj_set_size(ROIRectFace, face.width, face.height); @@ -215,7 +251,7 @@ int loop() { if (rects.size() > 1) { smile = rects[1]; if ((smile & face).area() != 0) { - lv_obj_align_to(ROIRectSmile, screen_canvas, LV_ALIGN_TOP_LEFT, smile.x, + lv_obj_align_to(ROIRectSmile, draw_widget, LV_ALIGN_TOP_LEFT, smile.x, smile.y); lv_obj_set_size(ROIRectSmile, smile.width, smile.height); diff --git a/u5/lib/cv/CMakeLists.txt b/u5/lib/cv/CMakeLists.txt index 6cc67cb..4d5ecc5 100644 --- a/u5/lib/cv/CMakeLists.txt +++ b/u5/lib/cv/CMakeLists.txt @@ -5,6 +5,13 @@ zephyr_get_compile_options_for_lang_as_string( C options) zephyr_get_compile_definitions_for_lang_as_string( CXX definitions) zephyr_get_compile_options_for_lang_as_string( CXX options) +set(OPENCV_FORCE_3RDPARTY_BUILD OFF) +set(USE_ZLIB OFF) +if (CONFIG_OPENCV_FORCE_3RDPARTY_BUILD) + set(OPENCV_FORCE_3RDPARTY_BUILD ON) + set(USE_ZLIB ON) +endif() + set(external_project_cflags "${definitions} ${options}" ) @@ -54,7 +61,7 @@ add_custom_command( -DWITH_JASPER=OFF -DBUILD_JPEG=OFF -DWITH_WEBP=OFF - -DBUILD_ZLIB=ON + -DBUILD_ZLIB=${USE_ZLIB} -DBUILD_PNG=OFF -DBUILD_OPENEXR=OFF -DWITH_TIFF=OFF @@ -87,7 +94,7 @@ add_custom_command( -DCV_TRACE=OFF -DOPENCV_ENABLE_MEMALIGN=OFF -DOPENCV_DISABLE_THREAD_SUPPORT=ON - -DOPENCV_FORCE_3RDPARTY_BUILD=ON + -DOPENCV_FORCE_3RDPARTY_BUILD=${OPENCV_FORCE_3RDPARTY_BUILD} -DENABLE_VFPV3=OFF -DCPU_BASELINE_DISABLE=ALL -DCPU_DISPATCH="" @@ -113,12 +120,17 @@ set(BUILD_BYPRODUCTS ${OPENCV_INSTALL_DIR}/lib/libopencv_flann.a ${OPENCV_INSTALL_DIR}/lib/libopencv_objdetect.a ${OPENCV_INSTALL_DIR}/lib/libopencv_core.a +) +if (OPENCV_FORCE_3RDPARTY_BUILD) +set(BUILD_BYPRODUCTS + ${BUILD_BYPRODUCTS} ${OPENCV_INSTALL_DIR}/lib/opencv4/3rdparty/liblibpng.a ${OPENCV_INSTALL_DIR}/lib/opencv4/3rdparty/liblibopenjp2.a ${OPENCV_INSTALL_DIR}/lib/opencv4/3rdparty/libzlib.a ${OPENCV_INSTALL_DIR}/lib/opencv4/3rdparty/liblibjpeg-turbo.a ) +endif() include(ExternalProject) @@ -156,10 +168,12 @@ opencv_add_lib(libopencv_flann) opencv_add_lib(libopencv_objdetect) opencv_add_lib(libopencv_core) +if (OPENCV_FORCE_3RDPARTY_BUILD) opencv_add_lib_3p(liblibpng) opencv_add_lib_3p(liblibopenjp2) opencv_add_lib_3p(libzlib) opencv_add_lib_3p(liblibjpeg-turbo) +endif() add_library(opencv_lib INTERFACE) target_link_libraries(opencv_lib INTERFACE @@ -170,11 +184,15 @@ libopencv_flann libopencv_features2d libopencv_imgproc libopencv_core +) +if (OPENCV_FORCE_3RDPARTY_BUILD) +target_link_libraries(opencv_lib INTERFACE liblibpng liblibopenjp2 libzlib liblibjpeg-turbo ) +endif() set_target_properties(opencv_lib PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${OPENCV_INSTALL_DIR}/include/opencv4) \ No newline at end of file diff --git a/u5/lib/cv/Kconfig b/u5/lib/cv/Kconfig index 95ff80b..4987fad 100644 --- a/u5/lib/cv/Kconfig +++ b/u5/lib/cv/Kconfig @@ -1,2 +1,7 @@ config OPENCV_LIB - bool "Support for opencv custom library" \ No newline at end of file + bool "Support for opencv custom library" + +config OPENCV_FORCE_3RDPARTY_BUILD + bool "Build opencv 3d party libraries" + default n + depends on OPENCV_LIB \ No newline at end of file diff --git a/u5/lib/cv/opencv b/u5/lib/cv/opencv index 7baf8f3..f86661d 160000 --- a/u5/lib/cv/opencv +++ b/u5/lib/cv/opencv @@ -1 +1 @@ -Subproject commit 7baf8f38e889a9e87e4b5f568a596fbdbcdc2763 +Subproject commit f86661d31ae09987ecbf814932f88a3e8547fe44 From e7510a87c6c4e8129a14ad8fb10f60fe65c753d3 Mon Sep 17 00:00:00 2001 From: Roman Pustobaiev Date: Mon, 20 Oct 2025 10:37:25 +0200 Subject: [PATCH 09/18] Bugfixing in opencv pipeline Refactoring, cleanup --- host/Src/main.cpp | 64 +++++++++++++++---- .../opencv/include/gf/opencv_pipeline.hpp | 8 ++- u5/common/opencv/src/opencv_pipeline.cpp | 48 ++++++++------ u5/common/opencv/src/opencv_utils.cpp | 10 +-- u5/common/src/video.cpp | 1 - u5/gf_dkx/src/config.hpp | 4 +- u5/gf_dkx/src/opencv_fd_app.cpp | 15 ++++- 7 files changed, 104 insertions(+), 46 deletions(-) diff --git a/host/Src/main.cpp b/host/Src/main.cpp index 86b65aa..84abf6c 100644 --- a/host/Src/main.cpp +++ b/host/Src/main.cpp @@ -82,23 +82,24 @@ int main(int, char **) { frameCroppedGray = frameOriginalGray; } - cv::Rect faceROIMax{}; - faceROIMax.width = 64; - faceROIMax.height = 40; + auto smileCascadeSize = smile_cascade.getOriginalWindowSize(); + cv::Rect faceROIMax(0, 0, smileCascadeSize.width * 2, + smileCascadeSize.height * 2); - cv::resize(frameCroppedGray, frameScreenGray, cv::Size(80, 80), 0.0f, 0.0f, + cv::resize(frameCroppedGray, frameScreenGray, + face_cascade.getOriginalWindowSize() * 2, 0.0f, 0.0f, cv::INTER_NEAREST); std::vector faces, smiles; - auto sigmaX = Range{0.82, 0.87, 0.01}; + auto sigmaX = Range{0.85, 0.85, 0.01}; BlurStage blurStage = BlurStage(sigmaX); - auto gamma = Range{0.44, 0.50, 0.02}; - GammaStage gammaStage = GammaStage(gamma); + auto _gamma = Range{0.30, 0.50, 0.05}; + GammaStage gammaStage = GammaStage(_gamma); auto d = Range{3, 3, 1}; - auto sigmaColor = Range{10.0, 10.0, 1.0}; + auto sigmaColor = Range{20.0, 20.0, 10.0}; auto sigmaSpace = Range{50.0, 50.0, 1.0}; BilateralStage bilateralStage = BilateralStage(d, sigmaColor, sigmaSpace); @@ -106,11 +107,47 @@ int main(int, char **) { auto tileSize = Range{4, 4, 1}; ClaheStage claheStage = ClaheStage(clipLimit, tileSize); - claheStage.setNextStage(&bilateralStage); - bilateralStage.setNextStage(&gammaStage); - gammaStage.setNextStage(nullptr); - // blurStage.setNextStage(nullptr); + claheStage.setNextStage(&bilateralStage) + .setNextStage(&gammaStage) + .setNextStage(&blurStage) + .setNextStage(nullptr); + bool break_key = false; + +#if 0 + std::vector rects; + + auto clipLimitFace = Range{4.0, 4.0, 1.0}; + auto tileSizeFace = Range{2, 2, 1}; + ClaheStage claheFaceStage = ClaheStage(clipLimitFace, tileSizeFace); + + auto _gammaFace = Range{0.30, 2.0, 0.1}; + GammaStage gammaStageFace = GammaStage(_gammaFace); + + claheFaceStage.setNextStage(nullptr); + + claheFaceStage.invoke( + frameScreenGray, [&](Stage &stage, cv::Mat in) -> bool { + rects = detectFaceAndSmile(face_cascade, smile_cascade, in, + frameCroppedGray, faceROIMax, claheStage); + + if (!rects.empty()) { + stage.dump(); + } + for (const auto &rect : rects) { + rectangle(in, rect, cv::Scalar(0, 0, 255), 1, 1, 0); + } + + cv::imshow(window_name, in); + int c = cv::waitKey(1); + if ((char)c == 27) { + break_key = true; + return true; + } + return !rects.empty(); + }); +#else + // cv::equalizeHist(frameScreenGray, frameScreenGray); auto rects = detectFaceAndSmile(face_cascade, smile_cascade, frameScreenGray, frameCroppedGray, faceROIMax, claheStage); @@ -121,8 +158,9 @@ int main(int, char **) { cv::imshow(window_name, frameScreenGray); +#endif int c = cv::waitKey(1); - if ((char)c == 27) { + if ((char)c == 27 || break_key) { break; } diff --git a/u5/common/opencv/include/gf/opencv_pipeline.hpp b/u5/common/opencv/include/gf/opencv_pipeline.hpp index 976b477..085c9ee 100644 --- a/u5/common/opencv/include/gf/opencv_pipeline.hpp +++ b/u5/common/opencv/include/gf/opencv_pipeline.hpp @@ -37,7 +37,7 @@ template struct Range { class Stage { public: - using InvokeCallback = std::function; + using InvokeCallback = std::function; explicit Stage() = default; virtual ~Stage() = default; @@ -60,6 +60,8 @@ class Stage { virtual cv::Mat get() = 0; virtual void next() = 0; virtual bool hasNext() = 0; + virtual void reset() = 0; + virtual void dump() {}; Stage &setNextStage(Stage *nextStage); bool invoke(cv::Mat &in, InvokeCallback callback); @@ -89,6 +91,7 @@ class ClaheStage : public Stage { cv::Mat get() override; void next() override; bool hasNext() override; + void reset() override; private: Range &clipLimit; @@ -107,6 +110,7 @@ class BilateralStage : public Stage { cv::Mat get() override; void next() override; bool hasNext() override; + void reset() override; private: Range &d; @@ -126,6 +130,7 @@ class GammaStage : public Stage { cv::Mat get() override; void next() override; bool hasNext() override; + void reset() override; private: Range γ @@ -141,6 +146,7 @@ class BlurStage : public Stage { cv::Mat get() override; void next() override; bool hasNext() override; + void reset() override; private: Range &sigmaX; diff --git a/u5/common/opencv/src/opencv_pipeline.cpp b/u5/common/opencv/src/opencv_pipeline.cpp index 9bdfbb9..c002c4b 100644 --- a/u5/common/opencv/src/opencv_pipeline.cpp +++ b/u5/common/opencv/src/opencv_pipeline.cpp @@ -9,13 +9,17 @@ Stage &Stage::setNextStage(Stage *_nextStage) { } bool Stage::invoke(cv::Mat &_in, InvokeCallback callback) { - if (!nextStage) { - return callback(_in); - } + reset(); setInput(_in); for (auto mat : *this) { - if (nextStage->invoke(mat, callback)) { - return true; + if (nextStage) { + if (nextStage->invoke(mat, callback)) { + return true; + } + } else { + if (callback(*this, mat)) { + return true; + } } } return false; @@ -24,11 +28,7 @@ bool Stage::invoke(cv::Mat &_in, InvokeCallback callback) { void Stage::setInput(cv::Mat &_in) { in = _in; } ClaheStage::ClaheStage(Range &clipLimit, Range &tileSize) - : clipLimit(clipLimit), tileSize(tileSize) { - - clipLimitIt = clipLimit.begin(); - tileSizeIt = tileSize.begin(); -} + : clipLimit(clipLimit), tileSize(tileSize) {} cv::Mat ClaheStage::get() { cv::Ptr clahe = @@ -50,14 +50,14 @@ bool ClaheStage::hasNext() { return (clipLimitIt != clipLimit.end()) && (tileSizeIt != tileSize.end()); } +void ClaheStage::reset() { + clipLimitIt = clipLimit.begin(); + tileSizeIt = tileSize.begin(); +} + BilateralStage::BilateralStage(Range &d, Range &sigmaColor, Range &sigmaSpace) - : d(d), sigmaColor(sigmaColor), sigmaSpace(sigmaSpace) { - - dIt = d.begin(); - sigmaColorIt = sigmaColor.begin(); - sigmaSpaceIt = sigmaSpace.begin(); -} + : d(d), sigmaColor(sigmaColor), sigmaSpace(sigmaSpace) {} cv::Mat BilateralStage::get() { cv::Mat out; @@ -82,10 +82,14 @@ bool BilateralStage::hasNext() { (sigmaSpaceIt != sigmaSpace.end()); } -GammaStage::GammaStage(Range &gamma) : gamma(gamma) { - gammaIt = gamma.begin(); +void BilateralStage::reset() { + dIt = d.begin(); + sigmaColorIt = sigmaColor.begin(); + sigmaSpaceIt = sigmaSpace.begin(); } +GammaStage::GammaStage(Range &gamma) : gamma(gamma) {} + cv::Mat GammaStage::get() { cv::Mat out = in.clone(); out.convertTo(out, CV_32FC1, 1.0 / 255.0); @@ -98,9 +102,9 @@ void GammaStage::next() { gammaIt = ++gammaIt; } bool GammaStage::hasNext() { return (gammaIt != gamma.end()); } -BlurStage::BlurStage(Range &sigmaX) : sigmaX(sigmaX) { - sigmaXIt = sigmaX.begin(); -} +void GammaStage::reset() { gammaIt = gamma.begin(); } + +BlurStage::BlurStage(Range &sigmaX) : sigmaX(sigmaX) {} cv::Mat BlurStage::get() { cv::Mat blurFrame; @@ -115,4 +119,6 @@ void BlurStage::next() { sigmaXIt = ++sigmaXIt; } bool BlurStage::hasNext() { return (sigmaXIt != sigmaX.end()); } +void BlurStage::reset() { sigmaXIt = sigmaX.begin(); } + } // namespace gf_cv \ No newline at end of file diff --git a/u5/common/opencv/src/opencv_utils.cpp b/u5/common/opencv/src/opencv_utils.cpp index b06e3b8..8db7b0d 100644 --- a/u5/common/opencv/src/opencv_utils.cpp +++ b/u5/common/opencv/src/opencv_utils.cpp @@ -83,8 +83,7 @@ detectFaceAndSmile(cv::CascadeClassifier &faceCascade, const double scaleFactor = 1.04; const int minNeighbors = 3; - const int flags = - cv::CASCADE_FIND_BIGGEST_OBJECT | cv::CASCADE_DO_ROUGH_SEARCH; + const int flags = cv::CASCADE_FIND_BIGGEST_OBJECT; // cv::equalizeHist(thumbnailFrame, thumbnailFrame); std::vector faceRects; @@ -130,9 +129,10 @@ detectFaceAndSmile(cv::CascadeClassifier &faceCascade, std::vector smileRects; - pipeline.invoke(faceROIFrameResized, [&](cv::Mat &in) -> bool { - return detectSmile(smileCascade, in, smileRects); - }); + pipeline.invoke(faceROIFrameResized, + [&](Stage &stage, cv::Mat &in) -> bool { + return detectSmile(smileCascade, in, smileRects); + }); // detectSmile(smileCascade, faceROIFrameResized, smileRects, // roIKernelParamsSweep); diff --git a/u5/common/src/video.cpp b/u5/common/src/video.cpp index 51762cb..f979b93 100644 --- a/u5/common/src/video.cpp +++ b/u5/common/src/video.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #include "gf/video.hpp" diff --git a/u5/gf_dkx/src/config.hpp b/u5/gf_dkx/src/config.hpp index 717d2f3..a86b997 100644 --- a/u5/gf_dkx/src/config.hpp +++ b/u5/gf_dkx/src/config.hpp @@ -29,8 +29,8 @@ #define ROI_FRAME_WIDTH 320 #define ROI_FRAME_HEIGHT 240 -#define THUMBNAIL_FRAME_WIDTH 80 -#define THUMBNAIL_FRAME_HEIGHT 80 +#define THUMBNAIL_FRAME_WIDTH 90 +#define THUMBNAIL_FRAME_HEIGHT 90 #define INPUT_PREPROCESS_OPENCV (0) #define FULL_FRAME_FLIP_Y (0) \ No newline at end of file diff --git a/u5/gf_dkx/src/opencv_fd_app.cpp b/u5/gf_dkx/src/opencv_fd_app.cpp index 202d2cf..409f21a 100644 --- a/u5/gf_dkx/src/opencv_fd_app.cpp +++ b/u5/gf_dkx/src/opencv_fd_app.cpp @@ -159,9 +159,15 @@ int loop() { cv::Mat matGrayRoi(ROI_FRAME_HEIGHT, ROI_FRAME_WIDTH, CV_8UC1, roiFrameBuffer); + // TODO: matGraySmall can be dynamically allocated, and freed after face + // detection stage passed. That will free some resources for smile detection + // phase cv::Mat matGraySmall(THUMBNAIL_FRAME_HEIGHT, THUMBNAIL_FRAME_WIDTH, CV_8UC1, thumbnailFrameBuffer); +// TODO hide logic below into utils.cpp +// when INPUT_PREPROCESS_OPENCV==1 that doesn't allow flipping at the moment, +// so not all cameras would work #if INPUT_PREPROCESS_OPENCV cv::Rect crop; crop.width = ROI_FRAME_WIDTH; @@ -206,6 +212,7 @@ int loop() { cv::Rect faceROIMax(0, 0, smileCascadeSize.width * 2, smileCascadeSize.height * 2); + // That pipeline can be static, no need to create it each iteration auto sigmaX = Range{0.85, 0.85, 0.01}; BlurStage blurStage = BlurStage(sigmaX); @@ -226,7 +233,7 @@ int loop() { .setNextStage(&blurStage) .setNextStage(nullptr); - //cv::equalizeHist(matGraySmall, matGraySmall); + cv::equalizeHist(matGraySmall, matGraySmall); auto rects = detectFaceAndSmile(faceCascade, smileCascade, matGraySmall, matGrayRoi, faceROIMax, claheStage); @@ -256,13 +263,15 @@ int loop() { lv_obj_set_size(ROIRectSmile, smile.width, smile.height); lv_obj_remove_flag(ROIRectSmile, LV_OBJ_FLAG_HIDDEN); + } #if DT_NODE_EXISTS(DT_ALIAS(led0)) - gpio_pin_toggle_dt(&led); + gpio_pin_set_dt(&led, 0); #endif - } } else { lv_obj_add_flag(ROIRectSmile, LV_OBJ_FLAG_HIDDEN); } + k_msleep(1); + return 0; } \ No newline at end of file From 433ea8c1fcfa440174d6b78f4f589556c264b18c Mon Sep 17 00:00:00 2001 From: Roman Pustobaiev Date: Mon, 20 Oct 2025 11:57:20 +0200 Subject: [PATCH 10/18] Adding git revision to print --- u5/gf_dkx/CMakeLists.txt | 15 +++++++++++++++ u5/gf_dkx/src/main.c | 5 ++++- u5/gf_dkx/src/version.h.in | 3 +++ 3 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 u5/gf_dkx/src/version.h.in diff --git a/u5/gf_dkx/CMakeLists.txt b/u5/gf_dkx/CMakeLists.txt index dfaf1ce..5c6bcac 100644 --- a/u5/gf_dkx/CMakeLists.txt +++ b/u5/gf_dkx/CMakeLists.txt @@ -37,3 +37,18 @@ target_sources(app PRIVATE src/lvgl_video_app.cpp ) endif() + +execute_process( + COMMAND git describe --always --dirty + OUTPUT_VARIABLE GIT_REVISION +) + +string(REGEX REPLACE "\n$" "" GIT_REVISION "${GIT_REVISION}") + +configure_file( + src/version.h.in + src/version.h + @ONLY +) + +target_include_directories(app PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/src") diff --git a/u5/gf_dkx/src/main.c b/u5/gf_dkx/src/main.c index 6d65f01..5d80d8a 100644 --- a/u5/gf_dkx/src/main.c +++ b/u5/gf_dkx/src/main.c @@ -10,13 +10,15 @@ #include #include #include -#include #define CONFIG_VIDEO_WIDTH 160 #define CONFIG_VIDEO_HEIGHT 120 #include "grinreflex.h" +#include "version.h" +#define LOG_LEVEL CONFIG_LOG_DEFAULT_LEVEL +#include LOG_MODULE_REGISTER(main, CONFIG_LOG_DEFAULT_LEVEL); #if !DT_HAS_CHOSEN(zephyr_camera) @@ -28,6 +30,7 @@ LOG_MODULE_REGISTER(main, CONFIG_LOG_DEFAULT_LEVEL); #endif int main(void) { + LOG_INF("Git revision : %s", GIT_REVISION); init(); while (1) { diff --git a/u5/gf_dkx/src/version.h.in b/u5/gf_dkx/src/version.h.in new file mode 100644 index 0000000..e6480df --- /dev/null +++ b/u5/gf_dkx/src/version.h.in @@ -0,0 +1,3 @@ + + +#define GIT_REVISION "@GIT_REVISION@" \ No newline at end of file From 83111aa8102fcd3916a8a02d345aa29b3dfda38b Mon Sep 17 00:00:00 2001 From: Roman Pustobaiev Date: Mon, 20 Oct 2025 16:28:09 +0200 Subject: [PATCH 11/18] Adding face detect only application --- u5/common/opencv/include/gf/opencv_utils.hpp | 16 ++ u5/common/opencv/src/opencv_utils.cpp | 15 ++ u5/gf_dkx/CMakeLists.txt | 18 +- u5/gf_dkx/Kconfig | 4 + u5/gf_dkx/src/opencv_fd_only_app.cpp | 220 +++++++++++++++++++ 5 files changed, 267 insertions(+), 6 deletions(-) create mode 100644 u5/gf_dkx/src/opencv_fd_only_app.cpp diff --git a/u5/common/opencv/include/gf/opencv_utils.hpp b/u5/common/opencv/include/gf/opencv_utils.hpp index 351c8c1..19ffd26 100644 --- a/u5/common/opencv/include/gf/opencv_utils.hpp +++ b/u5/common/opencv/include/gf/opencv_utils.hpp @@ -32,6 +32,22 @@ std::vector detectFaceAndSmile(cv::CascadeClassifier &faceCascade, cv::Mat &fullFrame, cv::Rect &faceROIMax, Stage &pipeline); +/** + * @brief + * + * @param faceCascade + * @param thumbnailFrame + * @return std::vector + */ +std::vector detectFace(cv::CascadeClassifier &faceCascade, + cv::Mat &thumbnailFrame); + +/** + * @brief + * + * @param bgr + * @return cv::Mat + */ cv::Mat cv_preprocessForQR(const cv::Mat &bgr); /** diff --git a/u5/common/opencv/src/opencv_utils.cpp b/u5/common/opencv/src/opencv_utils.cpp index 8db7b0d..15789cc 100644 --- a/u5/common/opencv/src/opencv_utils.cpp +++ b/u5/common/opencv/src/opencv_utils.cpp @@ -159,6 +159,21 @@ detectFaceAndSmile(cv::CascadeClassifier &faceCascade, return rects; } +std::vector detectFace(cv::CascadeClassifier &faceCascade, + cv::Mat &thumbnailFrame) { + + const double scaleFactor = 1.04; + const int minNeighbors = 3; + const int flags = cv::CASCADE_FIND_BIGGEST_OBJECT; + + // cv::equalizeHist(thumbnailFrame, thumbnailFrame); + std::vector faceRects; + faceCascade.detectMultiScale(thumbnailFrame, faceRects, scaleFactor, + minNeighbors, flags); + + return faceRects; +} + cv::Rect remapROI(cv::Rect &ROI, cv::Mat &inFrame, cv::Mat &outFrame) { float sx = (float)outFrame.cols / (float)inFrame.cols; float sy = (float)outFrame.rows / (float)inFrame.rows; diff --git a/u5/gf_dkx/CMakeLists.txt b/u5/gf_dkx/CMakeLists.txt index 5c6bcac..c2deb04 100644 --- a/u5/gf_dkx/CMakeLists.txt +++ b/u5/gf_dkx/CMakeLists.txt @@ -29,13 +29,19 @@ target_sources(app PRIVATE include(${ZEPHYR_BASE}/samples/subsys/usb/common/common.cmake) if (CONFIG_OPENCV_LIB) -target_sources(app PRIVATE - src/opencv_fd_app.cpp -) + if (CONFIG_VARIAND_FACE_DETECT_ONLY) + target_sources(app PRIVATE + src/opencv_fd_only_app.cpp + ) + else() + target_sources(app PRIVATE + src/opencv_fd_app.cpp + ) + endif() else() -target_sources(app PRIVATE - src/lvgl_video_app.cpp -) + target_sources(app PRIVATE + src/lvgl_video_app.cpp + ) endif() execute_process( diff --git a/u5/gf_dkx/Kconfig b/u5/gf_dkx/Kconfig index be97d2d..1d5971f 100644 --- a/u5/gf_dkx/Kconfig +++ b/u5/gf_dkx/Kconfig @@ -60,4 +60,8 @@ config VARIANT_RELEASE default n depends on BOARD_GRINREFLEX_DK2 +config VARIAND_FACE_DETECT_ONLY + bool "Build only face detector" + default n + endmenu \ No newline at end of file diff --git a/u5/gf_dkx/src/opencv_fd_only_app.cpp b/u5/gf_dkx/src/opencv_fd_only_app.cpp new file mode 100644 index 0000000..27262e0 --- /dev/null +++ b/u5/gf_dkx/src/opencv_fd_only_app.cpp @@ -0,0 +1,220 @@ + +#include +#include + +#if defined(CONFIG_OPENCV_LIB) +#include "cascades.h" +#include "gf/opencv_utils.hpp" +#include "opencv2/opencv.hpp" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#define LOG_LEVEL CONFIG_LOG_DEFAULT_LEVEL +#include +LOG_MODULE_REGISTER(grinreflex_app); + +#include "gf/lvgl_utils.hpp" +#include "gf/video.hpp" + +#include + +#include "config.hpp" +#include "gf/utils.hpp" +#include "grinreflex.h" + +using namespace gf_cv; + +static const struct device *video_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_camera)); + +static const struct device *display_dev = + DEVICE_DT_GET(DT_CHOSEN(zephyr_display)); + +const struct device *jpeg_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_jpeg)); + +#if DT_NODE_EXISTS(DT_ALIAS(led0)) +static const struct gpio_dt_spec led = GPIO_DT_SPEC_GET(DT_ALIAS(led0), gpios); +#endif + +static uint8_t + fullFrameBuffer[FULL_FRAME_WIDTH * FULL_FRAME_HEIGHT * + LV_COLOR_FORMAT_GET_SIZE(FULL_FRAME_COLOR_FORMAT)]; +static uint8_t roiFrameBuffer[ROI_FRAME_WIDTH * ROI_FRAME_HEIGHT * + LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8)]; +static uint8_t + thumbnailFrameBuffer[THUMBNAIL_FRAME_WIDTH * THUMBNAIL_FRAME_HEIGHT * + LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8)]; + +static lv_obj_t *screen_canvas; +static lv_obj_t *dummy_canvas; + +#if INPUT_PREPROCESS_OPENCV +static lv_obj_t *lvgl_image; +#endif + +static lv_obj_t *ROIRectFace; +static cv::CascadeClassifier faceCascade; + +int init() { + if (!device_is_ready(display_dev)) { + LOG_ERR("Display device not ready, aborting test"); + return -ENODEV; + } + + if (!device_is_ready(video_dev)) { + LOG_ERR("Video device not ready, aborting test"); + return -ENODEV; + } + + if (!device_is_ready(jpeg_dev)) { + printf("%s JPEG device not ready", jpeg_dev->name); + return -ENODEV; + } + +#if DT_NODE_EXISTS(DT_ALIAS(led0)) + if (!gpio_is_ready_dt(&led)) { + return -ENODEV; + } + + if (gpio_pin_configure_dt(&led, GPIO_OUTPUT_ACTIVE) < 0) { + return -ENODEV; + } +#endif + + cv::setNumThreads(1); + cv::setUseOptimized(false); + + if (false == + loadCascade(faceCascade, cascade_face_data(), cascade_face_len())) { + return -3; + } + + Video::setup(); + + dummy_canvas = lv_canvas_create(NULL); + screen_canvas = lv_canvas_create(lv_screen_active()); + lv_canvas_fill_bg(screen_canvas, lv_color_hex3(0xccc), LV_OPA_COVER); + lv_obj_center(screen_canvas); + +#if INPUT_PREPROCESS_OPENCV + lvgl_image = lv_img_create(lv_scr_act()); + lv_obj_center(lvgl_image); +#endif + + display_blanking_off(display_dev); + + ROIRectFace = + allocLvROIRect(lv_scr_act(), 4, lv_palette_main(LV_PALETTE_GREEN)); + + return 0; +} + +int loop() { + struct video_buffer vbuf {}; + struct video_buffer *vbuf_ptr = &vbuf; + struct jpeg_out_prop jpeg_prop; + + vbuf_ptr->type = VIDEO_BUF_TYPE_OUTPUT; + + int err = video_dequeue(video_dev, &vbuf_ptr, K_FOREVER); + if (err) { + LOG_ERR("Unable to dequeue video buf"); + return 0; + } + + // jpeg_hw_decode(jpeg_dev, (uint8_t *)vbuf_ptr->buffer, vbuf_ptr->bytesused, + // jpeg_frame_buffer); + + jpeg_hw_poll(jpeg_dev, 0, &jpeg_prop); + + // printf("JPEG done w = %d, h = %d, color = %d chroma = %d\n", + // jpeg_prop.width, + // jpeg_prop.height, jpeg_prop.color_space, jpeg_prop.chroma); + + jpeg_color_convert_helper(jpeg_dev, &jpeg_prop, vbuf_ptr->buffer, + fullFrameBuffer); + + // TODO: matGraySmall can be dynamically allocated, and freed after face + // detection stage passed. That will free some resources for smile detection + // phase + cv::Mat matGraySmall(THUMBNAIL_FRAME_HEIGHT, THUMBNAIL_FRAME_WIDTH, CV_8UC1, + thumbnailFrameBuffer); + +// TODO hide logic below into utils.cpp +// when INPUT_PREPROCESS_OPENCV==1 that doesn't allow flipping at the moment, +// so not all cameras would work +#if INPUT_PREPROCESS_OPENCV + cv::Rect crop; + crop.width = ROI_FRAME_WIDTH; + crop.height = ROI_FRAME_HEIGHT; + crop.x = (matGrayFull.cols - crop.width) / 2; + crop.y = crop.height; + cv::Mat matCrop = matGrayFull(crop); + matCrop.copyTo(matGrayRoi); + cv::resize(matGrayRoi, matGraySmall, + cv::Size(THUMBNAIL_FRAME_WIDTH, THUMBNAIL_FRAME_HEIGHT), 0.0f, + 0.0f, cv::INTER_NEAREST); + + displayFrame(lvgl_image, thumbnailFrameBuffer, + lvgl::Size(THUMBNAIL_FRAME_WIDTH, THUMBNAIL_FRAME_HEIGHT), + LV_COLOR_FORMAT_L8); +#else + cropFullFrameToRoi(dummy_canvas, fullFrameBuffer, roiFrameBuffer, + lvgl::Size(FULL_FRAME_WIDTH, FULL_FRAME_HEIGHT), + lvgl::Size(ROI_FRAME_WIDTH, ROI_FRAME_HEIGHT), + FULL_FRAME_COLOR_FORMAT, LV_COLOR_FORMAT_L8, + FULL_FRAME_FLIP_Y); + + fitRoiFrameToThumbnail( + screen_canvas, roiFrameBuffer, thumbnailFrameBuffer, + lvgl::Size(ROI_FRAME_WIDTH, ROI_FRAME_HEIGHT), + lvgl::Size(THUMBNAIL_FRAME_WIDTH, THUMBNAIL_FRAME_HEIGHT), + LV_COLOR_FORMAT_L8, LV_COLOR_FORMAT_L8); + +#endif /* #if INPUT_PREPROCESS_OPENCV */ + lv_task_handler(); + + // Return buffer back, we don't need it anymore, thank you + err = video_enqueue(video_dev, vbuf_ptr); + if (err) { + LOG_ERR("Unable to requeue video buf"); + return 0; + } + + cv::equalizeHist(matGraySmall, matGraySmall); + auto rects = detectFace(faceCascade, matGraySmall); + + lv_obj_t *draw_widget = screen_canvas; +#if INPUT_PREPROCESS_OPENCV + draw_widget = lvgl_image; +#endif + + cv::Rect face; + if (rects.size()) { + lv_obj_remove_flag(ROIRectFace, LV_OBJ_FLAG_HIDDEN); + + face = rects[0]; + lv_obj_align_to(ROIRectFace, draw_widget, LV_ALIGN_TOP_LEFT, face.x, + face.y); + lv_obj_set_size(ROIRectFace, face.width, face.height); +#if DT_NODE_EXISTS(DT_ALIAS(led0)) + gpio_pin_set_dt(&led, 0); +#endif + } else { + lv_obj_add_flag(ROIRectFace, LV_OBJ_FLAG_HIDDEN); + } + + k_msleep(1); + + return 0; +} \ No newline at end of file From 053c2d0ecb559ffb70d00f97ce03ad8e117a196e Mon Sep 17 00:00:00 2001 From: Roman Pustobaiev Date: Mon, 10 Nov 2025 18:03:59 +0100 Subject: [PATCH 12/18] Minimal support for stm32n6 soc --- u5/app/CMakeLists.txt | 60 --- u5/app/Kconfig | 19 - u5/app/boards/grinreflex_dk1.conf | 10 - u5/app/boards/grinreflex_dk1.overlay | 50 -- u5/app/boards/grinreflex_dk2.conf | 2 - u5/app/boards/grinreflex_dk2.overlay | 22 - u5/app/boards/stm32u5g9j_dk2.conf | 10 - u5/app/boards/stm32u5g9j_dk2.overlay | 94 ---- u5/app/lvgl.conf | 24 - u5/app/nema.cmake | 30 -- u5/app/opencv.cmake | 26 -- u5/app/prj.conf | 94 ---- u5/app/src/grinreflex.cpp | 426 ------------------ u5/app/src/lvgl_fs.h | 8 - u5/app/src/lvgl_fs_sample.c | 210 --------- u5/app/src/main.c | 38 -- u5/app/src/sample_usbd.h | 47 -- u5/app/src/sample_usbd_init.c | 212 --------- .../grinreflex_n6_dk3/Kconfig.defconfig | 15 + .../Kconfig.grinreflex_n6_dk3 | 7 + .../vendor/grinreflex_n6_dk3/board.cmake | 19 + u5/boards/vendor/grinreflex_n6_dk3/board.yml | 8 + .../doc/img/nucleo_n657x0_q.webp | Bin 0 -> 18466 bytes .../vendor/grinreflex_n6_dk3/doc/index.rst | 298 ++++++++++++ .../grinreflex_n6_dk3/grinreflex_n6_dk3.dts | 13 + .../grinreflex_n6_dk3_common.dtsi | 169 +++++++ .../grinreflex_n6_dk3_defconfig | 21 + .../grinreflex_n6_dk3_stm32n657xx_sb.dts | 13 + ...grinreflex_n6_dk3_stm32n657xx_sb_defconfig | 24 + .../vendor/grinreflex_n6_dk3/twister.yaml | 18 + u5/common/CMakeLists.txt | 9 +- u5/gf_n6_dk3/CMakeLists.txt | 8 + u5/gf_n6_dk3/README.rst | 33 ++ u5/gf_n6_dk3/prj.conf | 1 + u5/gf_n6_dk3/sample.yaml | 18 + u5/gf_n6_dk3/src/main.c | 14 + 36 files changed, 687 insertions(+), 1383 deletions(-) delete mode 100644 u5/app/CMakeLists.txt delete mode 100644 u5/app/Kconfig delete mode 100644 u5/app/boards/grinreflex_dk1.conf delete mode 100644 u5/app/boards/grinreflex_dk1.overlay delete mode 100644 u5/app/boards/grinreflex_dk2.conf delete mode 100644 u5/app/boards/grinreflex_dk2.overlay delete mode 100644 u5/app/boards/stm32u5g9j_dk2.conf delete mode 100644 u5/app/boards/stm32u5g9j_dk2.overlay delete mode 100644 u5/app/lvgl.conf delete mode 100644 u5/app/nema.cmake delete mode 100644 u5/app/opencv.cmake delete mode 100644 u5/app/prj.conf delete mode 100644 u5/app/src/grinreflex.cpp delete mode 100644 u5/app/src/lvgl_fs.h delete mode 100644 u5/app/src/lvgl_fs_sample.c delete mode 100644 u5/app/src/main.c delete mode 100644 u5/app/src/sample_usbd.h delete mode 100644 u5/app/src/sample_usbd_init.c create mode 100644 u5/boards/vendor/grinreflex_n6_dk3/Kconfig.defconfig create mode 100644 u5/boards/vendor/grinreflex_n6_dk3/Kconfig.grinreflex_n6_dk3 create mode 100644 u5/boards/vendor/grinreflex_n6_dk3/board.cmake create mode 100644 u5/boards/vendor/grinreflex_n6_dk3/board.yml create mode 100644 u5/boards/vendor/grinreflex_n6_dk3/doc/img/nucleo_n657x0_q.webp create mode 100644 u5/boards/vendor/grinreflex_n6_dk3/doc/index.rst create mode 100644 u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3.dts create mode 100644 u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_common.dtsi create mode 100644 u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_defconfig create mode 100644 u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_sb.dts create mode 100644 u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_sb_defconfig create mode 100644 u5/boards/vendor/grinreflex_n6_dk3/twister.yaml create mode 100644 u5/gf_n6_dk3/CMakeLists.txt create mode 100644 u5/gf_n6_dk3/README.rst create mode 100644 u5/gf_n6_dk3/prj.conf create mode 100644 u5/gf_n6_dk3/sample.yaml create mode 100644 u5/gf_n6_dk3/src/main.c diff --git a/u5/app/CMakeLists.txt b/u5/app/CMakeLists.txt deleted file mode 100644 index d1495a4..0000000 --- a/u5/app/CMakeLists.txt +++ /dev/null @@ -1,60 +0,0 @@ -#------------------------------------------------------------------------------- -# Zephyr Example Application -# -# Copyright (c) 2021 Nordic Semiconductor ASA -# SPDX-License-Identifier: Apache-2.0 - -cmake_minimum_required(VERSION 3.13.1) - -message(FATAL_ERROR This application is deprecated, please use uvc_gf_dk1 / gf_dkx / ...) - -set(EXTRA_CONF_FILE - ${CMAKE_CURRENT_SOURCE_DIR}/lvgl.conf; -) - -find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) - -project(app) - -# TODO: below two lines are unclear -set(NO_THREADSAFE_STATICS $) -zephyr_compile_options($<$:${NO_THREADSAFE_STATICS}>) - -if (CONFIG_OPENCV_LIB) - include(opencv.cmake) -endif() - -if (CONFIG_NEMA) - include(nema.cmake) -endif() - -target_include_directories(app PRIVATE - src -) - -target_sources(app PRIVATE - src/main.c - src/gfx_utils.cpp - src/video.cpp -) - -if (CONFIG_BOARD_GRINREFLEX_DK1 OR CONFIG_BOARD_GRINREFLEX_DK2) -target_sources(app PRIVATE - src/grinreflex_utils.cpp -) -endif() - -if (CONFIG_BOARD_GRINREFLEX_DK1) -target_sources(app PRIVATE - src/sample_usbd_init.c - src/lvgl_fs_sample.c - src/uart.cpp - src/grinreflex.cpp -) -endif() - -if (CONFIG_BOARD_GRINREFLEX_DK2) -target_sources(app PRIVATE - src/grinreflex_v2.cpp -) -endif() diff --git a/u5/app/Kconfig b/u5/app/Kconfig deleted file mode 100644 index c004ca1..0000000 --- a/u5/app/Kconfig +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (c) 2021 Nordic Semiconductor ASA -# SPDX-License-Identifier: Apache-2.0 -# -# This file is the application Kconfig entry point. All application Kconfig -# options can be defined here or included via other application Kconfig files. -# You can browse these options using the west targets menuconfig (terminal) or -# guiconfig (GUI). - -#menu "Zephyr" -#source "Kconfig.zephyr" -#endmenu - -module = grinreflex -module-str = grinreflex - -source "subsys/logging/Kconfig.template.log_config" -source "Kconfig.zephyr" -rsource "Kconfig.sample_usbd" -rsource "Kconfig.sample_msc" diff --git a/u5/app/boards/grinreflex_dk1.conf b/u5/app/boards/grinreflex_dk1.conf deleted file mode 100644 index 3bab102..0000000 --- a/u5/app/boards/grinreflex_dk1.conf +++ /dev/null @@ -1,10 +0,0 @@ -CONFIG_USB_DEVICE_STACK_NEXT=y -CONFIG_USBD_MSC_CLASS=y -CONFIG_USBD_MSC_LUNS_PER_INSTANCE=3 -CONFIG_USBD_LOG_LEVEL_WRN=y -CONFIG_UDC_DRIVER_LOG_LEVEL_WRN=y - -CONFIG_SAMPLE_USBD_PID=0x0008 -CONFIG_SAMPLE_USBD_PRODUCT="USBD MSC sample" - -CONFIG_FILE_SYSTEM_SHELL=y \ No newline at end of file diff --git a/u5/app/boards/grinreflex_dk1.overlay b/u5/app/boards/grinreflex_dk1.overlay deleted file mode 100644 index 20a6118..0000000 --- a/u5/app/boards/grinreflex_dk1.overlay +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2024, Eve Redero - * - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include - -/ { - chosen: chosen { - zephyr,spi1 = &spi1; - zephyr,usart2 = &usart2; - }; - - fstab { - compatible = "zephyr,fstab"; - - ffs2: ffs2 { - compatible = "zephyr,fstab,fatfs"; - automount; - disk-access; - mount-point = "/NAND:"; - }; - }; - - flashdisk0: flash-disk { - compatible = "zephyr,flash-disk"; - flash-controller = <&mx66lm1g45>; - partition = <&extflash_partition>; - disk-name = "NAND"; - cache-size = <4096>; - }; -}; - -&usart2 { - dmas = <&gpdma1 0 27 (STM32_DMA_PERIPH_TX) - &gpdma1 1 26 (STM32_DMA_MODE_CYCLIC | STM32_DMA_PERIPH_RX | STM32_DMA_MEM_8BITS)>; - dma-names = "tx", "rx"; - current-speed = <921600>; - fifo-enable; -}; - -&gpdma1 { - status = "okay"; -}; - -&dcmi { - jpeg-dev = <&jpeg0>; -}; \ No newline at end of file diff --git a/u5/app/boards/grinreflex_dk2.conf b/u5/app/boards/grinreflex_dk2.conf deleted file mode 100644 index 523ec3d..0000000 --- a/u5/app/boards/grinreflex_dk2.conf +++ /dev/null @@ -1,2 +0,0 @@ -CONFIG_FAT_FILESYSTEM_ELM=n -CONFIG_POSIX_FILE_SYSTEM=n \ No newline at end of file diff --git a/u5/app/boards/grinreflex_dk2.overlay b/u5/app/boards/grinreflex_dk2.overlay deleted file mode 100644 index 62a9e81..0000000 --- a/u5/app/boards/grinreflex_dk2.overlay +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright (c) 2024, Eve Redero - * - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include - -/ { - chosen: chosen { - zephyr,spi1 = &spi1; - }; -}; - -&gpdma1 { - status = "okay"; -}; - -&dcmi { - jpeg-dev = <&jpeg0>; -}; \ No newline at end of file diff --git a/u5/app/boards/stm32u5g9j_dk2.conf b/u5/app/boards/stm32u5g9j_dk2.conf deleted file mode 100644 index 3bab102..0000000 --- a/u5/app/boards/stm32u5g9j_dk2.conf +++ /dev/null @@ -1,10 +0,0 @@ -CONFIG_USB_DEVICE_STACK_NEXT=y -CONFIG_USBD_MSC_CLASS=y -CONFIG_USBD_MSC_LUNS_PER_INSTANCE=3 -CONFIG_USBD_LOG_LEVEL_WRN=y -CONFIG_UDC_DRIVER_LOG_LEVEL_WRN=y - -CONFIG_SAMPLE_USBD_PID=0x0008 -CONFIG_SAMPLE_USBD_PRODUCT="USBD MSC sample" - -CONFIG_FILE_SYSTEM_SHELL=y \ No newline at end of file diff --git a/u5/app/boards/stm32u5g9j_dk2.overlay b/u5/app/boards/stm32u5g9j_dk2.overlay deleted file mode 100644 index 4069f06..0000000 --- a/u5/app/boards/stm32u5g9j_dk2.overlay +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (c) 2024, Eve Redero - * - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include - -/ { - chosen: chosen { - zephyr,nema = &nema0; - zephyr,spi1 = &spi1; - zephyr,usart2 = &usart2; - zephyr,jpeg = &jpeg0; - }; - - pointer { - compatible = "zephyr,lvgl-pointer-input"; - input = <>911>; - }; - - fstab { - compatible = "zephyr,fstab"; - - ffs2: ffs2 { - compatible = "zephyr,fstab,fatfs"; - automount; - disk-access; - mount-point = "/NAND:"; - }; - }; - - flashdisk0: flash-disk { - compatible = "zephyr,flash-disk"; - flash-controller = <&mx66lm1g45>; - partition = <&extflash_partition>; - disk-name = "NAND"; - cache-size = <4096>; - }; - - nema0: nema-gfx { - compatible = "st,nema-gfx"; - status = "okay"; - }; - - zephyr_camera_i2c: i2c1 { - #address-cells = <1>; - #size-cells = <0>; - - status = "okay"; - clock-frequency = ; - - ov2640: ov2640@30 { - compatible = "ovti,ov2640"; - reg = <0x30>; - status = "okay"; - - port { - }; - }; - }; - - jpeg0: jpeg_hw { - compatible = "st,jpeg-hw"; - dmas = <&gpdma1 2 125 (STM32_DMA_PERIPH_RX | STM32_DMA_PERIPH_32BITS | STM32_DMA_MEM_32BITS) - &gpdma1 3 124 (STM32_DMA_PERIPH_TX | STM32_DMA_PERIPH_32BITS | STM32_DMA_MEM_32BITS)>; - dma-names = "tx", "rx"; - status = "okay"; - }; -}; - -/* -&spi1 { - clock-frequency = ; - - pinctrl-0 = <&spi1_sck_pa5 &spi1_miso_pa6 &spi1_mosi_pb5>; - pinctrl-names = "default"; - cs-gpios = <&gpiob 11 (GPIO_ACTIVE_LOW | GPIO_PULL_UP)>; - status = "okay"; -}; -*/ - -&usart2 { - dmas = <&gpdma1 0 27 (STM32_DMA_PERIPH_TX) - &gpdma1 1 26 (STM32_DMA_MODE_CYCLIC | STM32_DMA_PERIPH_RX | STM32_DMA_MEM_8BITS)>; - dma-names = "tx", "rx"; - current-speed = <921600>; - fifo-enable; -}; - -&gpdma1 { - status = "okay"; -}; \ No newline at end of file diff --git a/u5/app/lvgl.conf b/u5/app/lvgl.conf deleted file mode 100644 index c6a38ad..0000000 --- a/u5/app/lvgl.conf +++ /dev/null @@ -1,24 +0,0 @@ - - -CONFIG_LV_Z_MEM_POOL_SIZE=16384 -CONFIG_LV_Z_SHELL=n - -CONFIG_LVGL=y -CONFIG_LV_USE_LOG=y -CONFIG_LV_USE_LABEL=y -CONFIG_LV_USE_ARC=y -CONFIG_LV_USE_IMAGE=y -#CONFIG_LV_USE_MONKEY=y -CONFIG_LV_FONT_MONTSERRAT_14=y -#CONFIG_LV_Z_POINTER_INPUT=y - -CONFIG_LV_CACHE_DEF_SIZE=4096 -CONFIG_LV_IMAGE_HEADER_CACHE_DEF_CNT=1 -#CONFIG_LV_USE_TJPGD=y -#CONFIG_LV_USE_FS_MEMFS=y -#CONFIG_LV_USE_FS=y -CONFIG_LV_USE_BUILTIN_MALLOC=n - -#CONFIG_LV_USE_PERF_MONITOR=y -#CONFIG_LV_USE_DROPDOWN=y -#CONFIG_LV_USE_SYSMON=y \ No newline at end of file diff --git a/u5/app/nema.cmake b/u5/app/nema.cmake deleted file mode 100644 index 9f698b6..0000000 --- a/u5/app/nema.cmake +++ /dev/null @@ -1,30 +0,0 @@ - - -if (NOT CONFIG_FPU) - message(FATAL_ERROR "FPU must be enabled: CONFIG_FPU=y") -endif() - -set(LVGL_LIB_PATH ${CMAKE_CURRENT_LIST_DIR}/../../modules/lib/gui/lvgl) -set(LVGL_NEMA_PATH ${LVGL_LIB_PATH}/libs/nema_gfx) -set(LVGL_NEMA_LIBA_PATH ${LVGL_NEMA_PATH}/lib/core/cortex_m33_revC/gcc) - - -zephyr_include_directories( - ${LVGL_NEMA_PATH}/include - ${LVGL_LIB_PATH}/src/draw/nema_gfx -) - -if (CONFIG_FP_SOFTABI) -target_link_libraries(app PRIVATE - ${LVGL_NEMA_LIBA_PATH}/libnemagfx.a - ${LVGL_NEMA_LIBA_PATH}/libnemagfx-wc16.a -) -else() -target_link_libraries(app PRIVATE - ${LVGL_NEMA_LIBA_PATH}/libnemagfx-float-abi-hard.a - ${LVGL_NEMA_LIBA_PATH}/libnemagfx-float-abi-hard-wc16.a -) -endif() # CONFIG_FP_SOFTABI - -file(GLOB_RECURSE SOURCES ${LVGL_LIB_PATH}/src/draw/nema_gfx/*.c) -zephyr_library_sources(${SOURCES}) diff --git a/u5/app/opencv.cmake b/u5/app/opencv.cmake deleted file mode 100644 index e5c4ddb..0000000 --- a/u5/app/opencv.cmake +++ /dev/null @@ -1,26 +0,0 @@ - - -zephyr_compile_options(-DOPENCV_DISABLE_THREAD_SUPPORT) -zephyr_compile_options(-DOPENCV_FLANN_USE_STD_RAND) - -target_sources(app PRIVATE - src/opencv_utils.cpp -) - -if (CONFIG_BOARD_GRINREFLEX_DK2) -set(OPENCV_LBP_CASCADES ${OPENCV_INSTALL_DIR}/share/opencv4/lbpcascades) -set(OPENCV_HAAR_CASCADES ${OPENCV_INSTALL_DIR}/share/opencv4/haarcascades) - -set(OPENCV_CASCADE_FACE_PATH ${OPENCV_LBP_CASCADES}/lbpcascade_frontalface_improved.xml) -set(OPENCV_CASCADE_SMILE_PATH ${OPENCV_HAAR_CASCADES}/haarcascade_smile.xml) - -configure_file( - src/cascades.h.in - src/cascades.h - @ONLY -) - -target_include_directories(app PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/src") -endif() - -target_link_libraries(app PUBLIC opencv_lib) \ No newline at end of file diff --git a/u5/app/prj.conf b/u5/app/prj.conf deleted file mode 100644 index 4850df5..0000000 --- a/u5/app/prj.conf +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright (c) 2021 Nordic Semiconductor ASA -# SPDX-License-Identifier: Apache-2.0 -# -# This is a Kconfig fragment which can be used to enable debug-related options -# in the application. See the README for more details. - -CONFIG_CPP=y -CONFIG_STD_CPP20=y - -CONFIG_MAIN_STACK_SIZE=65536 - -CONFIG_LOG=y -CONFIG_LOG_DEFAULT_LEVEL=3 -CONFIG_SHELL=y - -CONFIG_INPUT=y - -CONFIG_LOG=y -CONFIG_LOG_MODE_MINIMAL=y -CONFIG_USB_DEVICE_INITIALIZE_AT_BOOT=n - -CONFIG_REQUIRES_FULL_LIBC=y -CONFIG_GLIBCXX_LIBCPP=y - -CONFIG_NEWLIB_LIBC=y -CONFIG_PICOLIBC=n - -CONFIG_POSIX_API=y - -CONFIG_ASSERT=y -CONFIG_STACK_SENTINEL=y -CONFIG_THREAD_STACK_INFO=y -CONFIG_INIT_STACKS=y # fill stacks, helps detect overflow -CONFIG_LOG=y -CONFIG_LOG_MODE_DEFERRED=n # immediate logs -CONFIG_DEBUG_OPTIMIZATIONS=y - -CONFIG_COMMON_LIBC_MALLOC=y -CONFIG_HEAP_MEM_POOL_SIZE=49152 -CONFIG_COMMON_LIBC_MALLOC_ARENA_SIZE=2097152 -#CONFIG_COMMON_LIBC_MALLOC_ARENA_SIZE=1048576 - -#CONFIG_TFLITE_LIB=y -CONFIG_OPENCV_LIB=y -#CONFIG_QUIRC_LIB=y -CONFIG_NEMA=y - -CONFIG_FPU=y -CONFIG_FP_SOFTABI=y - -CONFIG_GPIO=y -#CONFIG_SPI=y -#CONFIG_SPI_LOG_LEVEL_DBG=y - -CONFIG_DMA=y -CONFIG_SERIAL=y -CONFIG_UART_ASYNC_API=y -CONFIG_RING_BUFFER=y -#CONFIG_UART_LOG_LEVEL_DBG=y - - -CONFIG_UART_INTERRUPT_DRIVEN=y - -CONFIG_STM32_JPEG=y - -CONFIG_CPP_EXCEPTIONS=y -CONFIG_CPP_RTTI=y - -#CONFIG_DISPLAY_LOG_LEVEL_DBG=y - -CONFIG_DMA_LOG_LEVEL_INF=y -#CONFIG_HW_STACK_PROTECTION=y - -# Make asserts print file:line and message -CONFIG_ASSERT=y -CONFIG_ASSERT_VERBOSE=y - -# Ensure logs/printk come out before the crash -CONFIG_PRINTK=y -CONFIG_LOG=y -CONFIG_LOG_PRINTK=y -#CONFIG_LOG_MODE_IMMEDIATE=y - -# Better symbols for gdb -#CONFIG_DEBUG_INFO=y -# (Optional) minimal optimizations for nicer backtraces -# CONFIG_NO_OPTIMIZATIONS=y - -# Richer fault dump on ARM (registers, etc.) -CONFIG_FAULT_DUMP=2 - -CONFIG_SPEED_OPTIMIZATIONS=y - -CONFIG_VIDEO_SHELL=y diff --git a/u5/app/src/grinreflex.cpp b/u5/app/src/grinreflex.cpp deleted file mode 100644 index 6ecb26b..0000000 --- a/u5/app/src/grinreflex.cpp +++ /dev/null @@ -1,426 +0,0 @@ - - -#include -#include - -#if defined(CONFIG_OPENCV_LIB) -#include "opencv2/opencv.hpp" -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include -#include -#include -#include - -#include -#include - -#define LOG_LEVEL CONFIG_LOG_DEFAULT_LEVEL -#include -LOG_MODULE_REGISTER(grinreflex_app); - -#include "grinreflex.h" -#include "video.hpp" -#include "gfx_utils.hpp" - -#include - -#if defined(CONFIG_OPENCV_LIB) -#include "opencv_utils.hpp" -#endif - -#if defined(CONFIG_QUIRC_LIB) -#include "quirc.h" -#endif - -#if defined(CONFIG_TFLITE_LIB) -#include "app/lib/tflm.h" -#endif - -#if defined(CONFIG_TFLITE_LIB) && defined(CONFIG_OPENCV_LIB) -#error "Can't use both CONFIG_TFLITE_LIB and CONFIG_OPENCV_LIB at the same time" -#endif - -#define NAND_PATH(_path) ("/NAND:" _path) - -#define AUTOMOUNT_NODE DT_NODELABEL(ffs2) -FS_FSTAB_DECLARE_ENTRY(AUTOMOUNT_NODE); - -static struct usbd_context *sample_usbd; -static lv_obj_t *vid_canvas; - -#if defined(CONFIG_GRINREFLEX_JPEG_VIDEO) - -//static uint8_t jpeg_frame_buffer[CONFIG_GRINREFLEX_VIDEO_WIDTH * CONFIG_GRINREFLEX_VIDEO_HEIGHT * LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_RGB888)]; -static uint8_t rgb_frame_buffer[CONFIG_GRINREFLEX_VIDEO_WIDTH * CONFIG_GRINREFLEX_VIDEO_HEIGHT * LV_COLOR_FORMAT_GET_SIZE(JPEG_COLOR_FORMAT)]; - -#endif - -USBD_DEFINE_MSC_LUN(flash, "NAND", "Zephyr", "FlashDisk", "0.00"); - -static const struct device *video_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_camera)); -static const struct device *display_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_display)); -const struct device *jpeg_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_jpeg)); - -#if defined(CONFIG_OPENCV_LIB) -static lv_obj_t *RedGreenDot; -static lv_obj_t *FaceRect; -static cv::CascadeClassifier faceCascade; -static cv::QRCodeDetector qrCodeDetector; - -#define OPECV_SHOWCASE OCV_SC_FACE_DETECT - -#endif /* defined(CONFIG_OPENCV_LIB) */ - -static uint8_t gray_frame_buffer[GRAY_FRAME_WIDTH * GRAY_FRAME_HEIGHT * LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8)]; - -#if defined(CONFIG_TFLITE_LIB) -#define TFLM_INFERENCE_WIDTH 96 -#define TFLM_INFERENCE_HEIGHT 96 - -static uint8_t tflm_rgb_frame_buffer[TFLM_INFERENCE_WIDTH * TFLM_INFERENCE_HEIGHT * LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_RGB888)]; -#endif - -static int enable_usb_device_next(void) { - int err; - - sample_usbd = sample_usbd_init_device(NULL); - if (sample_usbd == NULL) { - LOG_ERR("Failed to initialize USB device"); - return -ENODEV; - } - - err = usbd_enable(sample_usbd); - if (err) { - LOG_ERR("Failed to enable device support"); - return err; - } - - LOG_DBG("USB device support enabled"); - - return 0; -} - -static lv_img_dsc_t video_img; -static lv_obj_t *screen; - -int init() -{ - if (!device_is_ready(display_dev)) { - LOG_ERR("Device not ready, aborting test"); - return -ENODEV; - } - - if (!device_is_ready(jpeg_dev)) { - printf("%s JPEG device not ready", jpeg_dev->name); - return -ENODEV; - } - - if (0 && enable_usb_device_next() != 0) { - LOG_ERR("Failed to enable USB"); - return -ENODEV; - } - -#if defined(CONFIG_TFLITE_LIB) - tflm_init(); -#endif - -#if defined(CONFIG_OPENCV_LIB) - - cv::setNumThreads(1); - cv::setUseOptimized(false); - -#if (OPECV_SHOWCASE == OCV_SC_FACE_DETECT) - const char *faceCascadePath = NAND_PATH("/cascface.xml"); - if (false == loadCascade(faceCascade, faceCascadePath)) { - return -3; - } -#endif -#endif - - Video::setup(); - - vid_canvas = lv_canvas_create(lv_screen_active()); - lv_canvas_fill_bg(vid_canvas, lv_color_hex3(0xccc), LV_OPA_COVER); - lv_obj_center(vid_canvas); - - display_blanking_off(display_dev); - - video_img.header.w = CONFIG_GRINREFLEX_VIDEO_WIDTH; - video_img.header.h = CONFIG_GRINREFLEX_VIDEO_HEIGHT; -#if defined(CONFIG_GRINREFLEX_JPEG_VIDEO) - video_img.data_size = CONFIG_GRINREFLEX_VIDEO_WIDTH * CONFIG_GRINREFLEX_VIDEO_HEIGHT * LV_COLOR_FORMAT_GET_SIZE(JPEG_COLOR_FORMAT); - video_img.header.cf = JPEG_COLOR_FORMAT; -#else - video_img.data_size = CONFIG_GRINREFLEX_VIDEO_WIDTH * CONFIG_GRINREFLEX_VIDEO_HEIGHT * LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_NATIVE); - video_img.header.cf = LV_COLOR_FORMAT_NATIVE; -#endif - screen = lv_img_create(lv_scr_act()); - -#if defined(CONFIG_OPENCV_LIB) - RedGreenDot = lv_obj_create(lv_scr_act()); - lv_obj_set_size(RedGreenDot, 30, 30); // dot size - lv_obj_set_style_radius(RedGreenDot, LV_RADIUS_CIRCLE, 0); // make it round - lv_obj_set_style_border_width(RedGreenDot, 0, 0); - lv_obj_set_style_bg_opa(RedGreenDot, LV_OPA_COVER, 0); - lv_obj_clear_flag(RedGreenDot, LV_OBJ_FLAG_SCROLLABLE); - lv_obj_align(RedGreenDot, LV_ALIGN_LEFT_MID, 4, 0); // 4px from left - lv_obj_move_foreground(RedGreenDot); // keep it on top - - FaceRect = lv_obj_create(lv_scr_act()); - lv_obj_set_pos(FaceRect, 0, 0); - lv_obj_set_size(FaceRect, 1, 1); - - // Unfilled, just a thick border - lv_obj_set_style_bg_opa(FaceRect, LV_OPA_TRANSP, LV_PART_MAIN); - lv_obj_set_style_border_width(FaceRect, 4, LV_PART_MAIN); - lv_obj_set_style_border_opa(FaceRect, LV_OPA_COVER, LV_PART_MAIN); - lv_obj_set_style_border_color(FaceRect, lv_palette_main(LV_PALETTE_GREEN), LV_PART_MAIN); - lv_obj_set_style_radius(FaceRect, 0, LV_PART_MAIN); // 0 = sharp corners - lv_obj_set_style_pad_all(FaceRect, 0, LV_PART_MAIN); - lv_obj_set_style_outline_opa(FaceRect, LV_OPA_TRANSP, LV_PART_MAIN); // no focus outline - lv_obj_set_style_shadow_opa(FaceRect, LV_OPA_TRANSP, LV_PART_MAIN); // no shadow - lv_obj_move_foreground(FaceRect); - lv_obj_add_flag(FaceRect, LV_OBJ_FLAG_HIDDEN); - -#endif - return 0; -} - -int loop() -{ - struct video_buffer vbuf{}; - struct video_buffer *vbuf_ptr = &vbuf; - struct jpeg_out_prop jpeg_prop; - - vbuf_ptr->type = VIDEO_BUF_TYPE_OUTPUT; - - int err = video_dequeue(video_dev, &vbuf_ptr, K_FOREVER); - if (err) { - LOG_ERR("Unable to dequeue video buf"); - return 0; - } - -#if defined(CONFIG_GRINREFLEX_JPEG_VIDEO) - //jpeg_hw_decode(jpeg_dev, (uint8_t *)vbuf_ptr->buffer, vbuf_ptr->bytesused, jpeg_frame_buffer); - - jpeg_hw_poll(jpeg_dev, 0, &jpeg_prop); - - //printf("JPEG done w = %d, h = %d, color = %d chroma = %d\n", jpeg_prop.width, - // jpeg_prop.height, jpeg_prop.color_space, jpeg_prop.chroma); - - jpeg_color_convert_helper(jpeg_dev, &jpeg_prop, vbuf_ptr->buffer, rgb_frame_buffer); - -#else - video_img.data = (uint8_t *)vbuf_ptr->buffer; - lv_img_set_src(screen, &video_img); - lv_obj_align(screen, LV_ALIGN_BOTTOM_LEFT, 0, 0); - lv_task_handler(); -#endif - -#if !defined(CONFIG_OPENCV_LIB) && !defined(CONFIG_TFLITE_LIB) - Gfx::GfxBuffer src{}; - Gfx::GfxBuffer dst{}; - - src.buf = rgb_frame_buffer; - src.width = CONFIG_GRINREFLEX_VIDEO_WIDTH; - src.height = CONFIG_GRINREFLEX_VIDEO_HEIGHT; - src.stride = CONFIG_GRINREFLEX_VIDEO_WIDTH * LV_COLOR_FORMAT_GET_SIZE(JPEG_COLOR_FORMAT); - src.cf = JPEG_COLOR_FORMAT; - src.size = CONFIG_GRINREFLEX_VIDEO_HEIGHT * CONFIG_GRINREFLEX_VIDEO_WIDTH * LV_COLOR_FORMAT_GET_SIZE(JPEG_COLOR_FORMAT); - - dst.buf = gray_frame_buffer; - dst.width = GRAY_FRAME_WIDTH; - dst.height = GRAY_FRAME_HEIGHT; - dst.stride = GRAY_FRAME_WIDTH * LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8); - dst.cf = LV_COLOR_FORMAT_L8; - dst.size = sizeof(gray_frame_buffer); - - Gfx::fit(vid_canvas, src, dst); - lv_task_handler(); - - err = video_enqueue(video_dev, vbuf_ptr); - if (err) { - LOG_ERR("Unable to requeue video buf"); - return 0; - } -#endif - -#if defined(CONFIG_TFLITE_LIB) - - Gfx::GfxBuffer src{}; - Gfx::GfxBuffer dst{}; - - src.buf = rgb_frame_buffer; - src.width = CONFIG_GRINREFLEX_VIDEO_WIDTH; - src.height = CONFIG_GRINREFLEX_VIDEO_HEIGHT; - src.stride = CONFIG_GRINREFLEX_VIDEO_WIDTH * LV_COLOR_FORMAT_GET_SIZE(JPEG_COLOR_FORMAT); - src.cf = JPEG_COLOR_FORMAT; - src.size = sizeof(rgb_frame_buffer); - - dst.buf = tflm_rgb_frame_buffer; - dst.width = TFLM_INFERENCE_WIDTH; - dst.height = TFLM_INFERENCE_HEIGHT; - dst.stride = TFLM_INFERENCE_WIDTH * LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_RGB888); - dst.cf = LV_COLOR_FORMAT_RGB888; - dst.size = sizeof(tflm_rgb_frame_buffer); - - Gfx::fit(vid_canvas, src, dst); - lv_task_handler(); - - err = video_enqueue(video_dev, vbuf_ptr); - if (err) { - LOG_ERR("Unable to requeue video buf"); - return 0; - } - - inference_task(tflm_rgb_frame_buffer); -#endif /* defined(CONFIG_TFLITE_LIB) */ - -#if defined(CONFIG_OPENCV_LIB) - Gfx::GfxBuffer src{}; - Gfx::GfxBuffer dst{}; - - src.buf = rgb_frame_buffer; - src.width = CONFIG_GRINREFLEX_VIDEO_WIDTH; - src.height = CONFIG_GRINREFLEX_VIDEO_HEIGHT; - src.stride = CONFIG_GRINREFLEX_VIDEO_WIDTH * LV_COLOR_FORMAT_GET_SIZE(JPEG_COLOR_FORMAT); - src.cf = JPEG_COLOR_FORMAT; - src.size = sizeof(rgb_frame_buffer); - - dst.buf = gray_frame_buffer; - dst.width = GRAY_FRAME_WIDTH; - dst.height = GRAY_FRAME_HEIGHT; - dst.stride = GRAY_FRAME_WIDTH * LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8); - dst.cf = LV_COLOR_FORMAT_L8; - dst.size = sizeof(gray_frame_buffer); - - Gfx::fit(vid_canvas, src, dst); - lv_task_handler(); - - err = video_enqueue(video_dev, vbuf_ptr); - if (err) { - LOG_ERR("Unable to requeue video buf"); - return 0; - } - -#if (OPECV_SHOWCASE == OCV_SC_FACE_DETECT) - cv::Mat mat_gray(GRAY_FRAME_WIDTH, GRAY_FRAME_HEIGHT, CV_8UC1, gray_frame_buffer); - - std::vector faces; - - bool facesFound = detectFaces(faceCascade, mat_gray, faces); - - lv_color_t c = facesFound ? lv_palette_main(LV_PALETTE_GREEN) - : lv_palette_main(LV_PALETTE_RED); - lv_obj_set_style_bg_color(RedGreenDot, c, 0); - - if (facesFound) lv_obj_remove_flag(FaceRect, LV_OBJ_FLAG_HIDDEN); - else lv_obj_add_flag(FaceRect, LV_OBJ_FLAG_HIDDEN); - - for (const auto &r : faces) { - //cv::rectangle(mat_gray, r, cv::Scalar(255, 255, 255), 2, cv::LINE_8); - printf("faces = %d, %d %d\n", faces.size(), r.x, r.y); - - lv_obj_align_to(FaceRect, vid_canvas, LV_ALIGN_TOP_LEFT, r.x, r.y); - lv_obj_set_size(FaceRect, r.width, r.height); - } -#elif (OPECV_SHOWCASE == OCV_SC_QRCODE) - cv::Mat gray(GRAY_FRAME_WIDTH, GRAY_FRAME_HEIGHT, CV_8UC1, gray_frame_buffer); - cv::Mat bgr(jpeg_prop.width, jpeg_prop.height, CV_8UC3, rgb_frame_buffer); - - try { - cv::Mat bin = cv_preprocessForQR(gray); - - cv::Mat ptsSingle; - uint32_t start_ms = k_uptime_get_32(); - - std::string text = qrCodeDetector.detectAndDecode(bin, ptsSingle); - uint32_t end_ms = k_uptime_get_32(); - uint32_t elapsed = end_ms - start_ms; - - if (!text.empty()) { - std::cout << "Decoded text : " << text << std::endl; - std::cout << "Time taken " << elapsed << " ms" << std::endl; - } - } catch (cv::Exception e) { - std::cout << "Caught exception " << e.what() << std::endl; - } -#endif - -#if defined(CONFIG_QUIRC_LIB) -#if (OPECV_SHOWCASE == OCV_SC_QRCODE_QUIRC) - cv::Mat gray(GRAY_FRAME_WIDTH, GRAY_FRAME_HEIGHT, CV_8UC1, gray_frame_buffer); - struct quirc *qr; - - qr = quirc_new(); - if (!qr) { - std::cout << "quirc: Failed to allocate memory" << std::endl; - return -ENOMEM; - } - - if (quirc_resize(qr, GRAY_FRAME_WIDTH, GRAY_FRAME_HEIGHT) < 0) { - std::cout << "quirc: Failed to allocate video memory" << std::endl; - return -ENOMEM; - } - - uint8_t *image; - int w, h; - bool qrDecodedOk = false; - - uint32_t start_ms = k_uptime_get_32(); - - image = quirc_begin(qr, &w, &h); - - std::memcpy(image, gray.data, (size_t)(w * h)); - quirc_end(qr); - - int num_codes; - int i; - - num_codes = quirc_count(qr); - for (i = 0; i < num_codes; i++) { - struct quirc_code code; - struct quirc_data data; - quirc_decode_error_t err; - - quirc_extract(qr, i, &code); - - /* Decoding stage */ - err = quirc_decode(&code, &data); - if (err) { - printf("DECODE FAILED: %s\n", quirc_strerror(err)); - return -EPERM; - } else { - qrDecodedOk = true; - printf("Data: %s\n", data.payload); - } - } - - quirc_destroy(qr); - - uint32_t end_ms = k_uptime_get_32(); - uint32_t elapsed = end_ms - start_ms; - - if (qrDecodedOk) { - std::cout << "Time Taken : " << elapsed << " ms" << std::endl; - } - -#endif /* (OPECV_SHOWCASE == OCV_SC_QRCODE_QUIRC) */ -#endif /* defined(CONFIG_QUIRC_LIB) */ -#endif /* defined(CONFIG_OPENCV_LIB) */ - return 0; -} \ No newline at end of file diff --git a/u5/app/src/lvgl_fs.h b/u5/app/src/lvgl_fs.h deleted file mode 100644 index 4b08722..0000000 --- a/u5/app/src/lvgl_fs.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef _LVGL_FS_H_ -#define _LVGL_FS_H_ - -void lvgl_fs_sample (); - -uint8_t *lvgl_fs_load_raw (const char *path, uint8_t *buffer, size_t size); - -#endif /*_LVGL_FS_H_*/ \ No newline at end of file diff --git a/u5/app/src/lvgl_fs_sample.c b/u5/app/src/lvgl_fs_sample.c deleted file mode 100644 index 550ca59..0000000 --- a/u5/app/src/lvgl_fs_sample.c +++ /dev/null @@ -1,210 +0,0 @@ -//#include - -#include -#include -#include -#include - -#include -#include -LOG_MODULE_REGISTER(lvgl_fs_sample_config); - -#include - -#define MOUNT_POINT "/NAND:" -#define MAX_PATH 128 - -// File structure wrapper -typedef struct { - struct fs_file_t file; -} lvgl_fs_file_t; - -// Directory structure wrapper -typedef struct { - struct fs_dir_t dir; - struct fs_dirent entry; -} lvgl_fs_dir_t; - -static bool _ready_cb(struct _lv_fs_drv_t *drv); -static void *_open_cb(lv_fs_drv_t *drv, const char *path, lv_fs_mode_t mode); -static lv_fs_res_t _close_cb(lv_fs_drv_t *drv, void *file_p); -static lv_fs_res_t _read_cb(lv_fs_drv_t *drv, void *file_p, void *buf, uint32_t btr, uint32_t *br); -static lv_fs_res_t _write_cb(lv_fs_drv_t *drv, void *file_p, const void *buf, uint32_t btw, uint32_t *bw); -static lv_fs_res_t _seek_cb(lv_fs_drv_t *drv, void *file_p, uint32_t pos, lv_fs_whence_t whence); -static lv_fs_res_t _tell_cb(lv_fs_drv_t *drv, void *file_p, uint32_t *pos_p); -static void *_dir_open_cb(lv_fs_drv_t *drv, const char *path); -static lv_fs_res_t _dir_read_cb(lv_fs_drv_t *drv, void *rdir_p, char *fn, uint32_t fn_len); -static lv_fs_res_t _dir_close_cb(lv_fs_drv_t *drv, void *rdir_p); - -void lvgl_fs_sample () -{ - static lv_fs_drv_t drv; /*Needs to be static or global*/ - lv_fs_drv_init(&drv); /*Basic initialization*/ - - drv.letter = 'S'; /*An uppercase letter to identify the drive */ - drv.ready_cb = _ready_cb; /*Callback to tell if the drive is ready to use */ - drv.open_cb = _open_cb; /*Callback to open a file */ - drv.close_cb = _close_cb; /*Callback to close a file */ - drv.read_cb = _read_cb; /*Callback to read a file */ - drv.write_cb = _write_cb; /*Callback to write a file */ - drv.seek_cb = _seek_cb; /*Callback to seek in a file (Move cursor) */ - drv.tell_cb = _tell_cb; /*Callback to tell the cursor position */ - - drv.dir_open_cb = _dir_open_cb; /*Callback to open directory to read its content */ - drv.dir_read_cb = _dir_read_cb; /*Callback to read a directory's content */ - drv.dir_close_cb = _dir_close_cb; /*Callback to close a directory */ - - drv.user_data = NULL; /*Any custom data if required*/ - - lv_fs_drv_register(&drv); /*Finally register the drive*/ -} - - -static bool _ready_cb(struct _lv_fs_drv_t *drv) -{ - return true; // For most cases, if mounted properly -} - -static void *_open_cb(lv_fs_drv_t *drv, const char *path, lv_fs_mode_t mode) -{ - lvgl_fs_file_t *f = (lvgl_fs_file_t *)malloc(sizeof(lvgl_fs_file_t)); - - if (!f) return NULL; - - fs_file_t_init(&f->file); - - char full_path[MAX_PATH]; - snprintf(full_path, sizeof(full_path), MOUNT_POINT"%s", path); - - int flags = 0; - if (mode == LV_FS_MODE_WR) - flags = FS_O_WRITE | FS_O_CREATE; - else if (mode == LV_FS_MODE_RD) - flags = FS_O_READ; - else if (mode == (LV_FS_MODE_WR | LV_FS_MODE_RD)) - flags = FS_O_READ | FS_O_WRITE | FS_O_CREATE; - - int ret = fs_open(&f->file, full_path, flags); - if (ret < 0) { - free(f); - return NULL; - } - - return f; -} - -static lv_fs_res_t _close_cb(lv_fs_drv_t *drv, void *file_p) -{ - lvgl_fs_file_t *f = file_p; - - fs_close(&f->file); - free(f); - return LV_FS_RES_OK; -} - -static lv_fs_res_t _read_cb(lv_fs_drv_t *drv, void *file_p, void *buf, uint32_t btr, uint32_t *br) -{ - lvgl_fs_file_t *f = file_p; - - ssize_t r = fs_read(&f->file, buf, btr); - if (r < 0) return LV_FS_RES_UNKNOWN; - *br = r; - return LV_FS_RES_OK; -} - -static lv_fs_res_t _write_cb(lv_fs_drv_t *drv, void *file_p, const void *buf, uint32_t btw, uint32_t *bw) -{ - lvgl_fs_file_t *f = file_p; - ssize_t w = fs_write(&f->file, buf, btw); - if (w < 0) return LV_FS_RES_UNKNOWN; - *bw = w; - return LV_FS_RES_OK; -} - -static lv_fs_res_t _seek_cb(lv_fs_drv_t *drv, void *file_p, uint32_t pos, lv_fs_whence_t whence) -{ - lvgl_fs_file_t *f = file_p; - - int z_whence; - switch (whence) { - case LV_FS_SEEK_SET: z_whence = FS_SEEK_SET; break; - case LV_FS_SEEK_CUR: z_whence = FS_SEEK_CUR; break; - case LV_FS_SEEK_END: z_whence = FS_SEEK_END; break; - default: return LV_FS_RES_INV_PARAM; - } - - int ret = fs_seek(&f->file, pos, z_whence); - return (ret < 0) ? LV_FS_RES_UNKNOWN : LV_FS_RES_OK; -} - -static lv_fs_res_t _tell_cb(lv_fs_drv_t *drv, void *file_p, uint32_t *pos_p) -{ - lvgl_fs_file_t *f = file_p; - - off_t pos = fs_tell(&f->file); - if (pos < 0) return LV_FS_RES_UNKNOWN; - *pos_p = pos; - return LV_FS_RES_OK; -} - -static void *_dir_open_cb(lv_fs_drv_t *drv, const char *path) -{ - lvgl_fs_dir_t *d = (lvgl_fs_dir_t *)malloc(sizeof(lvgl_fs_dir_t)); - - if (!d) return NULL; - - fs_dir_t_init(&d->dir); - - char full_path[128]; - snprintf(full_path, sizeof(full_path), MOUNT_POINT"/%s", path); - - if (fs_opendir(&d->dir, full_path) < 0) { - free(d); - return NULL; - } - - return d; -} - -static lv_fs_res_t _dir_read_cb(lv_fs_drv_t *drv, void *rdir_p, char *fn, uint32_t fn_len) -{ - lvgl_fs_dir_t *d = rdir_p; - int ret = fs_readdir(&d->dir, &d->entry); - if (ret < 0 || d->entry.name[0] == 0) { - fn[0] = '\0'; // No more files - return LV_FS_RES_OK; - } - - strncpy(fn, d->entry.name, LV_FS_MAX_FN_LENGTH); - return LV_FS_RES_OK; -} - -static lv_fs_res_t _dir_close_cb(lv_fs_drv_t *drv, void *rdir_p) -{ - lvgl_fs_dir_t *d = rdir_p; - fs_closedir(&d->dir); - free(d); - return LV_FS_RES_OK; -} - -uint8_t *lvgl_fs_load_raw (const char *path, uint8_t *buffer, size_t size) -{ - struct fs_file_t file; - fs_file_t_init(&file); - - int ret = fs_open(&file, path, FS_O_READ); - if (ret < 0) { - LOG_ERR("Failed to open %s: %d\n", path, ret); - return NULL; - } - - ssize_t read_bytes = fs_read(&file, buffer, size); - fs_close(&file); - - if (read_bytes != size) { - LOG_ERR("Read %d bytes, expected %d\n", (int)read_bytes, (int)size); - return NULL; - } - - return buffer; -} \ No newline at end of file diff --git a/u5/app/src/main.c b/u5/app/src/main.c deleted file mode 100644 index 672166a..0000000 --- a/u5/app/src/main.c +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2024 Charles Dias - * - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include -#include -#include -#include -#include -#include - -#define CONFIG_VIDEO_WIDTH 160 -#define CONFIG_VIDEO_HEIGHT 120 - -#include "grinreflex.h" - -LOG_MODULE_REGISTER(main, CONFIG_LOG_DEFAULT_LEVEL); - -#if !DT_HAS_CHOSEN(zephyr_camera) -#error No camera chosen in devicetree. Missing "--shield" or "--snippet video-sw-generator" flag? -#endif - -#if !DT_HAS_CHOSEN(zephyr_display) -#error No display chosen in devicetree. Missing "--shield" flag? -#endif - -int main(void) -{ - init(); - - while (1) - { - loop(); - } -} diff --git a/u5/app/src/sample_usbd.h b/u5/app/src/sample_usbd.h deleted file mode 100644 index 55fe0ee..0000000 --- a/u5/app/src/sample_usbd.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2023 Nordic Semiconductor ASA. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef ZEPHYR_SAMPLES_SUBSYS_USB_COMMON_SAMPLE_USBD_H -#define ZEPHYR_SAMPLES_SUBSYS_USB_COMMON_SAMPLE_USBD_H - -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/* - * The scope of this header is limited to use in USB samples together with the - * new experimental USB device stack, you should not use it in your own - * application. However, you can use the code as a template. - */ - -/* - * This function uses Kconfig.sample_usbd options to configure and initialize a - * USB device. It configures sample's device context, default string descriptors, - * USB device configuration, registers any available class instances, and - * finally initializes USB device. It is limited to a single device with a - * single configuration instantiated in sample_usbd_init.c, which should be - * enough for a simple USB device sample. - * - * It returns the configured and initialized USB device context on success, - * otherwise it returns NULL. - */ -struct usbd_context *sample_usbd_init_device(usbd_msg_cb_t msg_cb); - -/* - * This function is similar to sample_usbd_init_device(), but does not - * initialize the device. It allows the application to set additional features, - * such as additional descriptors. - */ -struct usbd_context *sample_usbd_setup_device(usbd_msg_cb_t msg_cb); - -#ifdef __cplusplus -} -#endif - -#endif /* ZEPHYR_SAMPLES_SUBSYS_USB_COMMON_SAMPLE_USBD_H */ diff --git a/u5/app/src/sample_usbd_init.c b/u5/app/src/sample_usbd_init.c deleted file mode 100644 index 97f4fa9..0000000 --- a/u5/app/src/sample_usbd_init.c +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Copyright (c) 2023 Nordic Semiconductor ASA - * - * SPDX-License-Identifier: Apache-2.0 - */ - -#include - -#include -#include -#include - -#include -LOG_MODULE_REGISTER(usbd_sample_config); - -#define ZEPHYR_PROJECT_USB_VID 0x2fe3 - -/* By default, do not register the USB DFU class DFU mode instance. */ -static const char *const blocklist[] = { - "dfu_dfu", - NULL, -}; - -/* doc device instantiation start */ -/* - * Instantiate a context named sample_usbd using the default USB device - * controller, the Zephyr project vendor ID, and the sample product ID. - * Zephyr project vendor ID must not be used outside of Zephyr samples. - */ -USBD_DEVICE_DEFINE(sample_usbd, - DEVICE_DT_GET(DT_NODELABEL(zephyr_udc0)), - ZEPHYR_PROJECT_USB_VID, CONFIG_SAMPLE_USBD_PID); -/* doc device instantiation end */ - -/* doc string instantiation start */ -USBD_DESC_LANG_DEFINE(sample_lang); -USBD_DESC_MANUFACTURER_DEFINE(sample_mfr, CONFIG_SAMPLE_USBD_MANUFACTURER); -USBD_DESC_PRODUCT_DEFINE(sample_product, CONFIG_SAMPLE_USBD_PRODUCT); -IF_ENABLED(CONFIG_HWINFO, (USBD_DESC_SERIAL_NUMBER_DEFINE(sample_sn))); - -/* doc string instantiation end */ - -USBD_DESC_CONFIG_DEFINE(fs_cfg_desc, "FS Configuration"); -USBD_DESC_CONFIG_DEFINE(hs_cfg_desc, "HS Configuration"); - -/* doc configuration instantiation start */ -static const uint8_t attributes = (IS_ENABLED(CONFIG_SAMPLE_USBD_SELF_POWERED) ? - USB_SCD_SELF_POWERED : 0) | - (IS_ENABLED(CONFIG_SAMPLE_USBD_REMOTE_WAKEUP) ? - USB_SCD_REMOTE_WAKEUP : 0); - -/* Full speed configuration */ -USBD_CONFIGURATION_DEFINE(sample_fs_config, - attributes, - CONFIG_SAMPLE_USBD_MAX_POWER, &fs_cfg_desc); - -/* High speed configuration */ -USBD_CONFIGURATION_DEFINE(sample_hs_config, - attributes, - CONFIG_SAMPLE_USBD_MAX_POWER, &hs_cfg_desc); -/* doc configuration instantiation end */ - -#if CONFIG_SAMPLE_USBD_20_EXTENSION_DESC -/* - * This does not yet provide valuable information, but rather serves as an - * example, and will be improved in the future. - */ -static const struct usb_bos_capability_lpm bos_cap_lpm = { - .bLength = sizeof(struct usb_bos_capability_lpm), - .bDescriptorType = USB_DESC_DEVICE_CAPABILITY, - .bDevCapabilityType = USB_BOS_CAPABILITY_EXTENSION, - .bmAttributes = 0UL, -}; - -USBD_DESC_BOS_DEFINE(sample_usbext, sizeof(bos_cap_lpm), &bos_cap_lpm); -#endif - -static void sample_fix_code_triple(struct usbd_context *uds_ctx, - const enum usbd_speed speed) -{ - /* Always use class code information from Interface Descriptors */ - if (IS_ENABLED(CONFIG_USBD_CDC_ACM_CLASS) || - IS_ENABLED(CONFIG_USBD_CDC_ECM_CLASS) || - IS_ENABLED(CONFIG_USBD_CDC_NCM_CLASS) || - IS_ENABLED(CONFIG_USBD_MIDI2_CLASS) || - IS_ENABLED(CONFIG_USBD_AUDIO2_CLASS) || - IS_ENABLED(CONFIG_USBD_VIDEO_CLASS)) { - /* - * Class with multiple interfaces have an Interface - * Association Descriptor available, use an appropriate triple - * to indicate it. - */ - usbd_device_set_code_triple(uds_ctx, speed, - USB_BCC_MISCELLANEOUS, 0x02, 0x01); - } else { - usbd_device_set_code_triple(uds_ctx, speed, 0, 0, 0); - } -} - -struct usbd_context *sample_usbd_setup_device(usbd_msg_cb_t msg_cb) -{ - int err; - - /* doc add string descriptor start */ - err = usbd_add_descriptor(&sample_usbd, &sample_lang); - if (err) { - LOG_ERR("Failed to initialize language descriptor (%d)", err); - return NULL; - } - - err = usbd_add_descriptor(&sample_usbd, &sample_mfr); - if (err) { - LOG_ERR("Failed to initialize manufacturer descriptor (%d)", err); - return NULL; - } - - err = usbd_add_descriptor(&sample_usbd, &sample_product); - if (err) { - LOG_ERR("Failed to initialize product descriptor (%d)", err); - return NULL; - } - - IF_ENABLED(CONFIG_HWINFO, ( - err = usbd_add_descriptor(&sample_usbd, &sample_sn); - )) - if (err) { - LOG_ERR("Failed to initialize SN descriptor (%d)", err); - return NULL; - } - /* doc add string descriptor end */ - - if (USBD_SUPPORTS_HIGH_SPEED && - usbd_caps_speed(&sample_usbd) == USBD_SPEED_HS) { - err = usbd_add_configuration(&sample_usbd, USBD_SPEED_HS, - &sample_hs_config); - if (err) { - LOG_ERR("Failed to add High-Speed configuration"); - return NULL; - } - - err = usbd_register_all_classes(&sample_usbd, USBD_SPEED_HS, 1, - blocklist); - if (err) { - LOG_ERR("Failed to add register classes"); - return NULL; - } - - sample_fix_code_triple(&sample_usbd, USBD_SPEED_HS); - } - - /* doc configuration register start */ - err = usbd_add_configuration(&sample_usbd, USBD_SPEED_FS, - &sample_fs_config); - if (err) { - LOG_ERR("Failed to add Full-Speed configuration"); - return NULL; - } - /* doc configuration register end */ - - /* doc functions register start */ - err = usbd_register_all_classes(&sample_usbd, USBD_SPEED_FS, 1, blocklist); - if (err) { - LOG_ERR("Failed to add register classes"); - return NULL; - } - /* doc functions register end */ - - sample_fix_code_triple(&sample_usbd, USBD_SPEED_FS); - usbd_self_powered(&sample_usbd, attributes & USB_SCD_SELF_POWERED); - - if (msg_cb != NULL) { - /* doc device init-and-msg start */ - err = usbd_msg_register_cb(&sample_usbd, msg_cb); - if (err) { - LOG_ERR("Failed to register message callback"); - return NULL; - } - /* doc device init-and-msg end */ - } - -#if CONFIG_SAMPLE_USBD_20_EXTENSION_DESC - (void)usbd_device_set_bcd_usb(&sample_usbd, USBD_SPEED_FS, 0x0201); - (void)usbd_device_set_bcd_usb(&sample_usbd, USBD_SPEED_HS, 0x0201); - - err = usbd_add_descriptor(&sample_usbd, &sample_usbext); - if (err) { - LOG_ERR("Failed to add USB 2.0 Extension Descriptor"); - return NULL; - } -#endif - - return &sample_usbd; -} - -struct usbd_context *sample_usbd_init_device(usbd_msg_cb_t msg_cb) -{ - int err; - - if (sample_usbd_setup_device(msg_cb) == NULL) { - return NULL; - } - - /* doc device init start */ - err = usbd_init(&sample_usbd); - if (err) { - LOG_ERR("Failed to initialize device support"); - return NULL; - } - /* doc device init end */ - - return &sample_usbd; -} diff --git a/u5/boards/vendor/grinreflex_n6_dk3/Kconfig.defconfig b/u5/boards/vendor/grinreflex_n6_dk3/Kconfig.defconfig new file mode 100644 index 0000000..170ddd3 --- /dev/null +++ b/u5/boards/vendor/grinreflex_n6_dk3/Kconfig.defconfig @@ -0,0 +1,15 @@ +# STM32N657X0_Q Nucleo board configuration + +# Copyright (c) 2025 STMicroelectronics +# SPDX-License-Identifier: Apache-2.0 + +if BOARD_NUCLEO_N657X0_Q + +if NETWORKING + +config NET_L2_ETHERNET + default y + +endif # NETWORKING + +endif # BOARD_NUCLEO_N657X0_Q diff --git a/u5/boards/vendor/grinreflex_n6_dk3/Kconfig.grinreflex_n6_dk3 b/u5/boards/vendor/grinreflex_n6_dk3/Kconfig.grinreflex_n6_dk3 new file mode 100644 index 0000000..22ec64e --- /dev/null +++ b/u5/boards/vendor/grinreflex_n6_dk3/Kconfig.grinreflex_n6_dk3 @@ -0,0 +1,7 @@ +# STM32N657X0_Q Nucleo board configuration + +# Copyright (c) 2024 STMicroelectronics +# SPDX-License-Identifier: Apache-2.0 + +config BOARD_GRINREFLEX_N6_DK3 + select SOC_STM32N657XX diff --git a/u5/boards/vendor/grinreflex_n6_dk3/board.cmake b/u5/boards/vendor/grinreflex_n6_dk3/board.cmake new file mode 100644 index 0000000..1b42800 --- /dev/null +++ b/u5/boards/vendor/grinreflex_n6_dk3/board.cmake @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 STMicroelectronics + +if(CONFIG_STM32N6_BOOT_SERIAL) + board_runner_args(stm32cubeprogrammer "--port=usb1") + board_runner_args(stm32cubeprogrammer "--download-modifiers=0x1") + board_runner_args(stm32cubeprogrammer "--start-modifiers=noack") +else() + board_runner_args(stm32cubeprogrammer "--port=swd") + board_runner_args(stm32cubeprogrammer "--tool-opt= mode=HOTPLUG ap=1") + board_runner_args(stm32cubeprogrammer "--extload=MX25UM51245G_STM32N6570-NUCLEO.stldr") + board_runner_args(stm32cubeprogrammer "--download-address=0x70000000") +endif() + +board_runner_args(stlink_gdbserver "--apid=1") +board_runner_args(stlink_gdbserver "--extload=MX25UM51245G_STM32N6570-NUCLEO.stldr") + +include(${ZEPHYR_BASE}/boards/common/stm32cubeprogrammer.board.cmake) +include(${ZEPHYR_BASE}/boards/common/stlink_gdbserver.board.cmake) diff --git a/u5/boards/vendor/grinreflex_n6_dk3/board.yml b/u5/boards/vendor/grinreflex_n6_dk3/board.yml new file mode 100644 index 0000000..26fee19 --- /dev/null +++ b/u5/boards/vendor/grinreflex_n6_dk3/board.yml @@ -0,0 +1,8 @@ +board: + name: grinreflex_n6_dk3 + full_name: Grinreflex N6 DK3 from Nucleo N657X0-Q + vendor: grinreflex + socs: + - name: stm32n657xx + variants: + - name: sb diff --git a/u5/boards/vendor/grinreflex_n6_dk3/doc/img/nucleo_n657x0_q.webp b/u5/boards/vendor/grinreflex_n6_dk3/doc/img/nucleo_n657x0_q.webp new file mode 100644 index 0000000000000000000000000000000000000000..c6d918de66d9c3db64e1f9924e6a22e6d2c29d94 GIT binary patch literal 18466 zcmZ^}Q;;xB%q2XwZQHhO+qP$(v2EM7ZQHhOoB#Xm*4AHpv0as`u0HAH;&jqEN|It? zSP}pL>Y_plY6=`0Pyhe`2>;n0Ab>m|fS`27haWVUZnddej<46d$hAH# zZsjFW63cT;gF0J-Y{GQK0Y>Szg7O?nBGy-bKSY%oVq#fzJUej%p`kxMY1oV%Ld<%S zD`lgZ2ucI`7rN*!6yz&WzQvf~ZPufqPSxf^##E?)=uPgyjd%uqURkQ}Zly3i8L%XG z*$00NiyW0z9y6t$uwk(a3&RTLw}ojT`Yo=|2=2T0NiL3_G}nFm$GW25chHX~g<_`B zU5|K*(C&i=0egs7Fpe)ewYfe4>u4FD>XT>iz$Bu-idcBaSAqd~$I+Pk4V)n`hi1#k zy-~+qxrcLb&*G@00ep>Swl_)dlCzs{GEB+e&epxG1V14#$iqxdR3S>dK3Kd~Md!+5 z_n}GMDY4{9wTKZsfJ8%qE%)Rsr(@8{aMw>OY)rtku&pF**f@<2GSBr8Y-EJ^6zxiOH-#15 zVh@ouaU`yRut;E#LFn0Fpp|OyaVIQEN@GGBq7*cGL^uwfnLgwcSK(C{3wpXLp+iFq z1kg!rU7kxII@6uYzvWtB6k&iry92KH`C6pS5xx5H-oo>DzI$PSciW~WfJZDRm#sQY z4p6CUY*o64jMjdq~%A!AgL6@|}BrX{LbL$$(AwRqW0lhV%aC&!e z%!_MbY05zNkK7=gD*Qb{m+%|{Jx|q=Bq1Bi15iJ0PgF}OXSuw~dR~y}CdS9j^^J3%t+CgIFY=O%V{j9|e zV{@NYaJR#rs*XN>V!gYj&abT;cYDbNXiR#kd~M2m2}kosDn!+=?qC3i1uOd2dI~y| z;rdNVQE@*bN+1Wv=v6Xn z!g|c2A#@{p(p1!VimoWlYg3+>fEfrfCkMrdS~eQW0H^TU@DoHjvGdoE^(qb}A+@1c zw1;dCJR{Ts=0goZwSt7)!N_P9JQ7v--_e8^*ngy=ij2ka3g&)&bwsfvq2b?lm6Q$M z!m1*A;?}tqDlXzYb0d-Cj0v!kCjXQlS-=!rRqsuJ$1g41xl26_yk;sWNBqQ|* zah@3m`_f-0LX!CxZl^vGvm6#XSaY6BCy(2G)p94rp3XVk!v1t9-^7P>{;7 z1{RzSs=zhVvSva1IL+{!XI-|qvLt%)+FAxHRYjCrm%)`&15a z!7xp&$XhY&Y%IZ>-nA<8kAq-Ld%0a%-YxTS$b6MHMz}%IOy*Lit?S zM|7LR_aSf_Fx{-Ix}h%~3cXrgbj113HHxn+I`WT5?B1^2dy&^n+q2-y?F#_O1?4- z&jeWdM5n^&7zmU2y0i*c=G&iT zBW(92tyhW+TJR(G62;B z{CPBW0Ne)xE}F3cTw@EcVGX65B5&b%1Tju$u+_s7#krx8(Z&@8D@Gng8!UfgNTrkV zjv|<)h{qUiK0<79c!NqHD`s$F=a^$Vrf!A&2Nx>|_-~I*g*XWaWV}6q#&t>_N^S*q;-|D~n z{qTKp#9h7p!i)4}_|pBVzt>s&Ex5%!Y+djh=uh+&_%%GIf8kB!xc$WC)3@|l|26%} zFXNy4U47!dR;>enn!nY_`W?8V{EGkF{p!E=0r0Q%jr{HWk$+S?&-{YloaM|_^?mqt zy@UUT{QNBi{f%|z?b`qSUhy~g{Yah4-;&riLB;SWyyc+j-k&$#g^8wlbJ27iEST=W zMbW*xXu1s*PW9sc-tGna3 ze+nTtfNX*KxBy3Oz03axPn*;I3(_KZFt&cG-2M<_>d=eFa^1l3q&{z4+#}!nDnA82 zlrjp%idwu^0YgkfUlyuKC3)yYV#^E_%e{cFlT{645(=qkXaVOM;@sY1yzxs=>uPd5 zyKZOyr-{UT?$Iiyhzh+ma0Tiz7(uei7QM4AP#4n))3ZAbe_NykhZlxLGWBo z?0+&Ub0$!D9z0UFwi7)=8xbpv{0GtL(_e<~4Zl!%b<7pAptyn9rE|%HyE_2q?m_J-S7XE?@u<89^O;6>~JoRG4`VmuU&q0Lnja*LR9+xXf-%!wm*&RlW1X?ntX6|f?!X6`A^ zy?%kEDe<%g|9FF>ZqO89g9MvHt5{|fBjxpA9qczgz6E1kO9_mKUqBcDcGA#T&K)iT zt%|53P;847uk3YRdL)}czP1MJs}KYfh43IZz9-kgCItKTjwp38Uo&HzD~$9jhMBec zog_dL{kxf`4Iq}cb9+VCCD})UMgXV_wlLIuCCN~&S)DzVLizKoMPoTE_8w?-q{aws zYXAzh8opVgTLm0!;=@a*6+H%Tte=8Y%@zJ>y$6_Kt!}$TPS(nqnn?Eg+ZQOC;rw)q z2K1gN?<~Al7b$`}?{h9)@Eu+!WndM}vV=t%|J%Fz612|d*OMt!9kMy|Aw2h4jq4Ui z;IN+~8FBM7`Y9E0o(i(SoDH0fq*;r?;6_=`*tpex61IS;uIcpA(NutPEbi|t!}4?m z1hgzXpm0wexQGAmtCYDW^SXS)R^7@Q&PU+6g1mM9tH-~LZ4t#UqC)@I%x9xFoQLrc zry^C`bDHHBAZWyPje7tqwpSRB8)J!EHgntWw}zs)PE|ixQO$EHi~;@;@syD` zMXE|W=&s7Yi6 zvttRE`_GjRQC4B<1h2lvk=Qm}1vdf9uXR#iC_dWX?T_powo`sZ4*>)Cvfh_bJ3rqN zndOV^0-W(Lm*h`~+aizVlvZ7Jm=X0tn!vJ5fp!{o-|lA-dtUr6q2b3ugnQHE6hq?v z$)r(`_HT|)7s~H{xgeDQyHcqP0Ka-zU4>&$!%#d>=M{nE!10~58&8#Q&F=PU@fo1` zc$DZOQW3><_FMet+fcKg@XM6dPBPh}=eA+3BL%E~e7O%+B#**-CYt6<97rpaR_{Ln zqeiOaP4zrOZ=FQcQBAjcDjdB9lg+j;Asc4-?ON0uP2oNYM!RHW;a0Dl0v zNmL*xg{GX*)lMQ%fF*iK+jhv5hToSp47K|<=AH}S;GP@e&U`G`s~A`OEq7IzNXx&I zUABhn&)uN&)x#YQJV}MwsYGe2`iJD_Nx7I;rvM9R8I^t6{N#IvI13pQ%P@U^N3E&f z(>R}5SPwUL<1MIZu6~JUB^*(>uxNdNdA$TTJD*Z_C0O|@M0|+ud!k9*AiD<3Xgr7V z#RhWHaiZ}g>$o;6L4q2&iAtC=y~0Y8;*zl^pw)DkYrtHeiKoRBu7OI+8N7QOT27ps zYif!35S!}>-4qt6+f`=Z@w}GcML5~9`9k^6#|99~UJ=MF{ATG{NkZ`^moq2u)BepT zXna%ld}~u^Q<>g75)Nb(^yBCmd2oeJw^2@DaP}>W7;+9n^ly%{`$*K*00OJg@Md{U z91e3Z>(Bdo5dZ)fIe(6wSFVFYU5=k9hZ%62?J)3?9Edp?j1r|j0<9~Ba{IzFJoDSk zI~3kwM_DVdi`Ez^n*4>^J4_Y*oiR#?a8<7mx|Ncnsf7kqC$J?;CDAMdwXypVn-oU= z+I_x4L0>d5tc`epibm%BwDpei(VD;^YaY1iqbSrY=mGZu<(4PYIGt|%7Hb>rb! z;ovAJPQjj7+;re}bd-*3aA+(#Nboe6;jP%eZ8Vqsj(}A8FCoC02jMR4nn`bEjF2q_ z4w$Arz$fi8)Vqta#OCPNW)e!FgFIz{?h zc?3TZ69yj?Xl{6u>PI^_?9VL9LlAH@LN@LU4kPly^J}#M6PT7P(;xFGpp%VZ9njyCFj$x6(n z$&%z#3Cj%riI|T8d(RC zPHWsOJubdg3Nw%PQHEvCJ&{5X7=K+D?>Oq$eEb+D@}Ms4M1wT#$H75=vbkz)e}AXF z#s}Q`M+LdbbWLOjkF(f`cn!o*nF3pTgjIn`|NaxwRb+_#n;3^u+=?yn`NO_a=qHJ( z3jpO_>|L~*2rCw*>W(EWMS(HrfPWRQwqj@36=(gC3OA#>rP)0{4zU|Kn4?C(lbW3x{?GeGk z%>9)OCbIsDYcQ&F841g0pT;C4AP@c%l%U$*h~*=YoSO@Li$Kum$35x2gfYNUpMV8O zKpBy4$QwtKUpn=TTzN8ud)KKC3Kr`j^?TY-~hbqE`-2`mH%$tvZJ* zh#Ugl3EnaEX@G3V@~Y(b8|djZnS@(gJezjLfS?lsI0;xmK;GPQwHwThTfo0mmX$a$2v-ni#Rd?vXu84u z7NSkRG$Q9G-f?zV${pui*L>vgJw{@Q%M5VD>D=*BH6NGyiH3Xi7GhpNjU>7nATBDq ziwqV`obeZm^YL!S3@KWKv0HuE(V0aQo2DUZ4c&kHs3o&g;N1c8AX zeH4aLIS-bpP~sn>zLW}(`vwmDB%!QO#;g($_etOjU58p9>2B-&|58#Cj)51*XH;H< zqmWc}IDQ#Ka(XysDh&dpNNs(yzSAMevC(Wu3PK5ci_pP!BC`{}A$UyvVS6yshmRK9 zY9#P&YG^8t4Awl#EFnh|wsxjG5Mw_>8RGW-E;N@n@2OZ~ik3*0A1&-Ze6oTZka#sL zxc}iI1>zz(Ht7aASKs)wsO=(QR`l==)nmC2Yd72D1tg_WIJ{6HR(64S)A2i`*B+_t zK;u2V18~)phME`zeL%dQEYkRB+ZaTgG~PNKd56Zp);5#zXFLb@N)Z$}o{F*@@Z3Sc z$cUJzG#s_SgcARD#{6^1PoZ#T?i&(xoV6gGmqBeOn4=m7;0aCvqu#oL@dU$UdZOq4 z<&5h#Kd(`9j1>U%IDR2pHGW+UMSRo@`$|_9TX8O_r?8W6_WFBN1Y>HXtKB4ko<KcWm3-q9AVCI}G+PlpGTwKr z<|eDfNofJqWH#LYlM90T^64+@QDy!tlMpHJyZDR{!q?i2|5Aq6&^AGfo}WYlN$CPpv(Gv z;7Y&V`rzV4i3VA+sQjfdFxGmcaluMd(tYuwfvc$2V|G(wNx1kbOhV4gc$5>fhJvV~ zW!Tc56at@au0=?9%6S|V!5#Zi z&!Jo(o-G5<@*Zm06NW9D0OIdIm95hCDv-I&%`8V$ntP0Tj|`@b6Ixs5n~L08k=MTr zkhDO9qQL;AYpVFCU{qds6jwJJS8WHY_Zm;t|0WJ^Y}vtihp$xJS$yiB4^%4oDY_^u zcyRD(@kOC5s=1~mN=+@C6S9pN!_+Fv;9lC@9Xa8ES8*32 z0%LyZgx<%Nwu2Hj4Rqco{i867-DJD^sZ8&!7r|zq0_zM};Q%#Rxh%1K{0o%eD6t96 zx!|vcS=;|6oXGveE!SFMx8Xw4m2%}u#31_3m%piT9ni7Wfe`$qiv(THu5*=>6EtCX4+ zS0m%#I+aAnKkoY1_x329Z)S#+YLtG7y#Oh=fm=Bkb75lI;shGua1!1#(@!R;*>0xc zQB;#fammR<{G`j(!zVG?=WRE)^Tdt!(;So-{%t5G*Vby?!icf(RP_=mfEnzp4TxuR z%7~zCUFz-b#~!^`)uD$2BZ|duVngImwY#|?kC%$)>w&IuB0Kq@KEQUX3%OYUsB29s zr=R}}6sZ3+vqve2$qGd85wkMnI>9TQmjG5P@(uf@751+cuTE~nCY`hy?7qy36tnjb z+O$@p`-oGI$(gw&iP`-5Hm@ap<=-a9DFR#maM-9x!6re(L?IW^|BDUpwib3$H1=BYybrF|*^`BfAfuI!-u5sHu_mG)W8wN8s^i_dQgN zI`}xUox6y;6XQ>Eef~H>`f(<7RTL#`%RbZc=ngScEEY$AXN+zm?R98WblrrGerbR$ zyHohS@gvXU{Vppp5LJ8{ zlv!s6J90YYKz1SyYgx=v(ZNUrh@CBD!(Ks<5~D;7QBgkL{V+Vw@WFkhQ~3~l*E&<17*MG*j!kCkj6jNAJ3Qz z?ERt9NbzKj@e=SrX~ZROx!@`emT^9%e>``kf}S z4PH}*o1_yV5g(Q?ch`zE+{8{kjXS$MI}mtBvFP;5F!`xyXq1Na2aHlo35Xo%Wst()24;gO~ zK_Z$g|70oMeujnkDjq%13nMzYwWF(W&I&09KU|DeNk$-G5g**FQ<8AiH#Rs2Rrf$Bo zg8>Zgpgc(>AwVZ%G`u>x@8xgRxqH0@%MQ*=hKcqQ0$lA^nv2I@+t=UqwOSb&LkT>9 z?j@Qd1Cs?7y#Smud;YM`YnjH(e$aMFRKK6D2u!FILiWQ$wiSki|3MwBlVm?x_BU60 zbfA^+n*{i~1<@WDPzt7DIu+=#!?-F#jYrU8w;_0g;!HE@Y#vS2P3;p0S>y^>fEjuN zUYzQYa{YmGmQL*9Tl*@vguJ=y! z(sb<@Ezy^6C2y0qXQ`9JoLK8t{aJadSDf;U>;8ddE z-aDD9Yh~aKN8|2^q~)e2{E_VVoL@U($10%4mDG~s$lo9fuY=JG6!sQ`7tjd`Nxx<; z!RenWvNzLt$t%{vS)6IwL$CVy&>MPi2n>Z^!0|kibUdQWpQlgX80*p@IaZS#aEdl# zs2L+S39b?FfndTyT&1S~4pxBJ(QiqX__JxSqrs(F`a{?*+4KPa%G8!HAfHe*pko{7 z^#n+ePiMWJ@}@(6C~k1#=85Jf-yblOAsOrDk zOQ#gQRDJGeDuV_8XBgr*y%K`jR$G6F|!ZgGfI3BREf|5;&$7a@N61V*wmZd*Bx zaCZn?302~U4o~Yos!NRlofDSRlu^DyNXTZ_nK-m)bCAx=lT?3;qP%TMlK}@LPRSlvbn@Zq#rV#yjt-MLwLsVV2!(=}s5F@Wb z*hy_RCq&)MhDQ!XL2Poi6${_!ma+};LIp;j6ifrbjvZuo&m|fn{LDy!*O~-@j1#TU zJHn>=6p&cN=W&(?rkGYI33OB>J^3p7l3;&+k6VHNRF%?4!yLQ8ns;tXwZCL~|k zEJ0wf%a&|R*>B~aZkSMr`TZpxo@CnSHkc+nE9$KPIHA|G^)Xy7$ZDABzr%8J8kspg zZQ}3M?rd(>6F3RT<5JzT*#w7@15pn^Jx9}jJBa*ur|)=V1s{1}FOZ{r8w6s50vknas0rm%Ihu*>ScSv-f-D0=!sMYxjs zgN2YKcmYyU#A@4TwD6bvifQEj2((DplT)aP`SEm|;lfC(Sb)K<;=goXk>(h0<>TI( zxtf-j@?|lT`b4v=v4Pi*b?h{<*0K~xy$pYQ^7M;5kNBh7hZWzQpWMt!B&Q6$YkgQgCOI{+TnCxOMl|9ZXuY(qF}Av~(kx9zlblTCU2 z^W|=MgpJW*goD~oDQ3=t_A9cU?f0F zVqJxQNb#7CG<%$_xx~)P4+<_E22)<*Gjw62n=wgWt2v(d`O>&l0A6oLD;L%=ALemD5X=ZQbJ{a0jR|sL#=;6O#hW& zsU~Kk+3=7rm^_d+!rBL6Amq$<0M3T|k%D-X71 zwMshHjb*bz7C(~a@F78=u2mUNc@V)a85g69dHi5Smn5p-e~Tqk9e{xPY}m|pHa_Ye z?>1<#2PuZro3u^=ad|wB3u#>uA5Tp&gQsPA5gP6N^*YF(p4%FUYYM}rKrJh()Wxe2 z%3}2)m}nz}n{xR!@u6McIE_G5BL1gy0w|5UbDIV?pK99FG(Egaf|2+Z+~67F|5)$v z_wQ3IGP7OTUnjf()AzDUX}L=(VA5}K6JGXTWfi=iL{AA<2${i$Xp7C;=WwM%Xy$UF zQOAjcm@8vYg7v=edrDxUUuAm7enbYFQZ|sWU!Fu>NEy<6{(C><8;^B~9^Tq2<%n;H zH6S22agAg=!UtW)3a|@FFA;KXS2KA(fD}O92Rt_*xGFI59)sB+T=_C!cYwp=eri|` zs<2g$e#{qjsW!SMqc&zVl|joeaEiBE$acKLy z78XsTg7}CLI7|v7$x7>osrlmIotgt-R~f_XL+fLG1gL2wT8D1db}dT-yIwPImoIy6 ziGI&$)R6EpT$3PpSs2DTj9(F)cKKal8`yS&Vq&w`18r5v%w_cvG&Vm?8`r+smvxV0 zyCB>OE5ZwQi10Lt%FPEFiLQY;K|wc~B~3`g28emAlCihu(qcLjCbHX zG;=OxSH)H?-s|V~7@~rAwWDWEyB>PL+W7-#BkhNUQ;r|TNGjkg&G!jVE56UVSTNP< zHYAF(^~9FXsb^tmirmj`Kd0QMO*vs!Hcv!prFtI$ZI$p$KZD>Gdm=`8&~m7X+y;!h=4je=#>bXb`>^zxBw;aG+hBnpkrvjIPO{o&Y4J)T$quV%#| z@Ft^SZ5slqST=)V@|XKTLd&u4)8Uj;3$3!o`nN7BlO1(i`g5rkpS9Y!;+#+O_?9T()(-;ZV4DzkG=VzFvZv{#K(+ucCh?ik{K5bWfTPirCSGJ4V~zop8Z zV+^u6G%azZb(=0_4+!bo(*k{8??>*V#l74`T%`f3TDepA<-}9V1c9}t0o(vQV@3ak zrnY1xi1GVomh(!^gjE5zv++jxud@ZE364UV4h!J%$VQ;$7#h9#t2ucdB0BGtP;UJtrE!UcRnSEjOU0 zaCl$p6$CscX5Sw}jU7>uW=FrtBxMWlA?dduT|_n_X_w$uH+-DgBZd2=i6BDux8-~>K=7@_pzrI+MORzZVS77mu#N-72!A|cn;##vaI{LQFRioK zRqbxUW^H3}l#~n7FWrB-K1>p>0GWHnNXW`9BE4%;@^{@Up&HQKGMS8dK&Utm_cm7< zX*^f)rPy0_rIq&Dk&o(G2>pxP1j=N=+`~*o^p!``dG9*d6Pz-EoAZbd1LCaBKb>_$ zO@LdJA$6A)ZO2m0#~qD~-_P-=N;-k#bg=3*<6k>^90_ z^|XnE;hP}WFAOdY^rr~{$-M*}oIMf){hLtyp9F(iRB`$LRDWi6Voa7(3nLJGTNggX zwX;h7kpCdh15uipIxHDZ_xrz3l40RD|5XvzJ3dxZYlLJe@`cx-vyP!_c;&Q{7da^@ zjJpz*b|t;_A1rXLGO|EyEeR%&$ChOdtm3Tq~y>p~1YB<|@* z=U)DOIs27KYRQF#$jdN5^070tqV2UKRrvFD6E6#ooeqtI2;pt2*kj+&_0 z685mf%eP|HPM-6F?GTKIr#(PZrcGlycDWQ4DV9FeNh}DL-Uu(Nph)t3A`#t`)x>&Z z=3?XT;lRE?`W7{xHMmp2z2$NgDbYWI59hJAfZCRio3CfSY-p^UVppfv%Zdg*Mx0>O zrWrOCcXcWEZ(k>lG)EJO0er?ucTB}`yb(-QZFHa=w|%c*Oz~*1u3}A-scfGZ4=5hL zRsLltmzp0Zur;)OGZ<0e zh?s{r5S5LfO$0Im!Z^xSQRIC;m@Vy6@UvZsIWlN|tsJzU-+HC&8n7wj9`7IAd;luJ z;Q~OZU>L(^Yv$CKTntB;M2;6sND+MO&YgH>YJ1k#PJ1!B{QI3qr9Qtj#L1Ek?ekhz z1ULY9qc%s0nJ{Q)q3lh~gL~qdvc;7VQO=-x+-`NaU*I5_pE{Dk;S%;APq>JQm%5W+ zqc(MX@!d0SNi7sWH(|Cr<~c^c&T%*USVDQU^UQM?%pE;k$@Pe*5FkBqZ8< z9O3IH7VC&mCL@@6@A6y0qrrnwv>-_dKXSlN2SA|n&0hAe>VcJ6rP{>TkhJs++hYL) z;gao9691 zaiwnOzUzt|6H_oL7!*9E2?l6G4C1EhDh&DRB9}3}#C-Fy8Rg#yfuo-Mrgxvm!!4?lRxRy+=*Aq{uu8;C=~I?fp`%aeZt=X_s5n(Mwb^`cWJUHPc@h9IpAEn_mg}YR2x)F3RLesR_W1ecp zh2sHbAh_NKH2S>4yU(6xJb-b*du5YF(#mpU99K`T=Dx%xzGLQ64F+lM#=B1SpuVob-5S*thysfa zm~{_{@02)Iq+HjiY9iuwbRUK5)R}o)2D>LuKrK5!erD zyZZ;Jo_$HNicu{Qbz|%RIah+ubmpJ+L?>droEuxVZ^vdWoVzDMlbC&? z1%%DDtHO-9rWfb%{MRIwquX%g&tqqGulgduehJ>~0N*O2eg^%fwq5o?=-&Cx9(VL? zmc!W24>_g4i3vVJ-Ce3jvySW^Sg}5hkiUq2^PGfPnq=8A-;T~iV%@G1 zdO=W0Ioik3j$Q;8Bp1*3M&VQ!E5!(G2c~8HS-4tjlbQ%+ZD$!wy8sQ|N1wLnJ!v*CPBHTTn)yIr#d1nkal7njV^M)dAvrm#j%>3XnA zOqE_@_nu(&K^ei(84EcuTr0pUsG%K}@+Hb5&(Adzs9E3%Sdb9nNb`yYm|e7;A7808k_>^)v{~Ed# z#8cd6A%ma3r|Aocbwzd|u4&Xt;U7CuHppexuDt!wnYdjtm8XF!7G*!0t*rFeTQBI5 z_E)}aYl;7va5}oo-vf-6^(HFRgZQL32!u=OZs=PBT!A9vF%kjDFGTTxSmla0)GtjC zpI>gQc?JTEWz@Y6)BHvxHXItiBnE3{$HPR*%MR?10XNX1*1takb7UPUe>I^9i#Pr} z`*L)*(S0EF|Be+qtFN^KINO14kKkz7%GhAegt6uE(3(T8SsCoM&P(9bdm1s8jpYl5 z)qlzMGoN2&pOdtrP0)n3Y0b4d(QYfvefS#N2W*XgN6n*n@b~b%Y@KnN}6`% zJ=fIomTs@ zFklBotn?5lOi4p;Ne2B+@7686VA1Frz*KAf)@5oM%`I;$%o2tLIJ7s1D}2Vypq-Eq z=xM_s(12rA*T*a82=>)r3}|8PcZ(2B8JXrlck#ewZ^u|hxLwAly_M0jRVTU5kd4gt z+P0h61`ZItOkmN#qJ(_g>7~JS;GFa`IAK3{j-RxOlS)sp(@QTLg`UeM-}HEgn-P~M za6oY+TbX*rhZ=|%UT#B^BX`!4QZ=u+1s@ zno?e<5$GwMR2JvR+Y6FccrLy%^J_iNd@fnT=}*TS^2xFf)!%gY`!SN1)eV)y8hlLlqORL0If(ggPe)D)`2pgC695=*tx`rqJ;!HSfSe zm8ppP$wI-Cr~>}VSra>#1Vp6ExBLbAT-2**{=P3hCMbaKLtgGp$1cblm809$@(rJ_ zhv4X;4-N8QMa`+3Z(Y=V@`w2v;-qpc862giN;;^L(9I&P6lPpD@1VPHd>YcR*0ECi}je|X}rjUxN&1vpX^wB0j3d0mFj5bMBQ(9 zy06XCQ;fA%wO2kw;XjcTJX8~==VtjUMZfb?7&Tr&63vY?{3|a)t4u46k$mrU3O=g( zYU}x#P(8;W002}F4J+;DS59GZTqUx<5uEmCJ>h{CzCMGu(csU4;&x0gDcz$15&kGDb&*7=Vz<w!^DgULv`GCnE?8 zHfw~vL(QWT-Tmc##c2kSo$@RPED0rDRTG|)P}DRJnC{Oh4eQoEnFBW?MtUDqoFKP< zw!BgipGtt~Eif_uwOs*myF)L$3|?HjCajsU7<0Hwtwin?vqiiAL8%Wagnq6Q|u#VH3c>n(1-O^Fyqq8zWSE>qJ zMX>fA6>x_AF*P|_n`m#NI6lba&?_h`AmO-1Ki{$8Fg1@t%!W#|;Xe+mSvgV3kQdIB z&~6OHF4tO>1NRMIF~7hA7NpHT=@eYs)=M~T21NWhhQ z?^ANzaEbK551km1+bH1gqrj?hk$*zI0*2@qn{6T*LjI9^U;%exz(7Mpx%KHAQKpI| zn^G~CVj2=H;V=0Y=)d6(i2Xx==iVKYv^cK;J&Bq5}YUQqQ9}FKL4G95lJsb+i{9LUf$L*;TrjRYo+HQT^73$iZ+Br7{vQ=0lfqh zUp&G~>l@ey9}seZg#{6JF<4&Tg9t_3CAd);1}3*UX--H4kOj{nC(G#TL|Bm%*c*7k znyYnW6=hIb$a7)M#&GSEeEGDr%e}G|V<4h8W~*$Q{%m^!YnX~;0yvZ(x+aZh&11A? zUGQ!-%hEB8fPK#GslG)An~aj#VzK~2W;NVvfF;v0uviB4WU|0dNj~}2nm-l~cqOfq z1ZU7~Oj=h=3j9N(*C34wzs`{US#&R-aq7-kjH0p}djVy$m~afh8`AnROd{K=f7OF# zQAqRM_pl8-9;L+K4}+yvCE`cVb&dRgr$vqGP^v(m~;I-JCK@s91}^=OB6%iJ{#l!ANO>hnO;`Siqd3-GKNWd< z@2=U~Tz21lZ7viHJ7{Q8q>{r#L!ZC|O+r9fqNPcX==ckNpyBYseB$i^vtrDqd}MoIJ+` zmN;7Z{{pWEQ1~a;&%&6vS`=}7cH+vfj-`3FWgxM|u_-no2|dn}krovtm-VVZ0GbIp zxpcXLA5+k;uD*Oh6JFo?d(CXijfY+O#1%KD{xt>NjGWass7%fc-Iay=(0r$SBV01l zd?Bb+z>uZNNVwg6jhar_nI6|0Og!aFMfEqCs;mJ9Kh@@i+<7v$bxk4`)T5Qy zT!cM};Vq}?5Xk5VmQT?Lk=FntbMB{5mE@8xT%9hnyIEaE*Qe95bvqK?(S3XK!(@h$ zk%kX0(d{2Xq7S+{A!sR(VLs#!F2_6xjFgAcNW0j8N_K}!rZi&{{{POhKl3)nRmguc zX~nT4nx_}xK^uW5v%vmyE*JOTI79g(Q~6GhJYj_PG4emj0P)WUo?Ap@zM)?$Hmw_a94!lmg+`%x{ z7o|17m+8;+@xZj|L7p^E9eEE#asZ$hr z+z^?lPfQYI!O7BI!~FR%<%d+d!MsbGOcGP*Qh5&z)Gs`zsfC|z5#;z!607qtfg&QW zAO11PDQC8Vfl#P-Lt6`FDt;z2PC?V>lIRU@09$733#-}M)bGEkfT%j3ED~z^5%F4X zs?vg5^8?*Hh|Uhj!mz2y-a+-UzeNRKm9R@r9PYN}Lpd;_1R%kK!wMFUi-{zvY0)0Uwd6S} z2>1cPk`5mtAyvlV9h$f$@0=jZY82FCt^IPzGNbczVbE((`mDsWacF--Cl7N*4f#`f zmK~c-KKz-K(qkbZm!LenurFO6-lE8GASb(qCA?xalozoxS7#qVD%&9}$+@icMsr1< z`0mpr*+jqViEA{2f)Dn5i8q$z8r!h11JaNg6(MF{Qj%}pl+Z)bd|R3RN{AI<#KpJ) z*p%6HU@IW?ky}5w+$bMSc7xy>=PMixGSJ#M2hMrnOwLN(eFjLf8a09@el<}g;$q(l zZo_9s3a%jT=QwHNt;x=gsV~iYu!Hd=zafwCbfiUDT0g&eY7zbe@Sp~Jqu~*ltv}3k zlXd2R07q@Q5O3eQF_KOClISZJIxS|+%@CGQt#cIUj@TmRK0f`%p-bcXIZ(TGhH@Y- z?b-$-0T0>@^$?5&HR=NuvHk0!#WhgDQ_$4vG@C-cg}k2`@GS3wNPf^48TdBcwhZeH z<4Mv15QAnk>M&`95p4idwecZ*S+wVb^sW3q6l+ruXh&A)L3ZEt(xaz}?y@zB*&9h9 z2=-o)i%-{CA(&!$<0r$b>%VW^3S~%aNO{L98OuDS87`TPy;nyORnLL@K>{jfxg3>d z$%(z)O+Y*{WHj48-#deXd^Baejt;;K_ssUF&oHuONq^qar2h=T`t&Znq4Xlr4IqQs zXRTG#8qgf|rtU%2eHz`9!(+2`P%>^-0_0BrK>aq^WA=iw)K^{pL!$1~8MkqcySjy; zb(CAl>lrM%4l97l^V~*MRJJQ>zN~JuvnS`cLHGuQ_+aOHW0wQ3Skq=&xXMS|a0_my z4YlViCI#$)?}Zizw5^^=g65Q=cGz+HPpiNnk4QIf0{*odvU|}Wo^e7SZqu*#JlJzx zfvu=QVEbIxzJ@ID?29c8lL?g|_`{7HIS=*FRcoRw0Y_>37jHmC^hKfEaCug4GtU!T z(skfkKerv5CjaYpoz0@{H(h%20@SKZT)DI}=cd&G$N`Ei-q_i#kowDxvJpfyEzxe>}1LFcE>a6fArCgG`JlQf$p$s_l z9Zn`2r;pJaPo6zaE6|N*Qe8}v zJi^yHp>kxxik5#``4!w%^TcR0x>bVJ0WPmVUjR~10q3!Hpf_W0B1L(4J)^*LHE7YT zFTKt2>cBd{T#rRs=&8QwJkuR7p!ZRmE7-^Z2ZR1CxbWBe7ytlVd0H2{Hsr+Z4qD`4 z%CZL8BTGP+WRLT4xaeu*48i~{I8<663!LwkNPTyo8wRl8m_RS5P98-+=%C0000000000 E0OJnjg#Z8m literal 0 HcmV?d00001 diff --git a/u5/boards/vendor/grinreflex_n6_dk3/doc/index.rst b/u5/boards/vendor/grinreflex_n6_dk3/doc/index.rst new file mode 100644 index 0000000..d7aa20f --- /dev/null +++ b/u5/boards/vendor/grinreflex_n6_dk3/doc/index.rst @@ -0,0 +1,298 @@ +.. zephyr:board:: nucleo_n657x0_q + +Overview +******** + +The NUCLEO-N657X0-Q board provides an affordable and flexible way for users to try out +new concepts and build prototypes by choosing from the various combinations of performance +and power consumption features, provided by the STM32 microcontroller. For the compatible boards, +the internal or external SMPS significantly reduces power consumption in Run mode. + +The ST Zio connector, which extends the ARDUINO® Uno V3 connectivity, and the ST morpho headers +provide an easy means of expanding the functionality of the Nucleo open development platform with +a wide choice of specialized shields. + +The NUCLEO-N657X0-Q board does not require any separate probe as it integrates the ST-LINK +debugger/programmer. + +The STM32 Nucleo-144 board comes with the STM32 comprehensive free software libraries and +examples available with the STM32Cube MCU Package. + +Hardware +******** + +- Common features: + + - STM32 microcontroller in an LQFP144, TFBGA225, or VFBGA264 package + - 3 user LEDs + - 1 user push-button and 1 reset push-button + - 32.768 kHz crystal oscillator + - Board connectors: + + - SWD + - ST morpho expansion connector + + - Flexible power-supply options: ST-LINK USB VBUS, USB connector, or external sources + +- Features specific to some of the boards (refer to the ordering information section + of the data brief for details); + + - External or internal SMPS to generate Vcore logic supply + - Ethernet compliant with IEEE-802.3-2002 + - USB Device only, USB OTG full speed, or SNK/UFP (full-speed or high-speed mode) + - Board connectors: + + - ARDUINO® Uno V3 connector or ST Zio expansion connector including ARDUINO® Uno V3 + - Camera module FPC + - MIPI20 compatible connector with trace signals + - USB with Micro-AB or USB Type-C® + - Ethernet RJ45 + + - On-board ST-LINK (STLINK/V2-1, STLINK-V3E, or STLINK-V3EC) debugger/programmer with + USB re-enumeration capability: mass storage, Virtual COM port, and debug port + +For more details, please refer to: + +* `NUCLEO-N657X0-Q website`_ +* `STM32N657X0 on www.st.com`_ +* `STM32N657 reference manual`_ + +Supported Features +================== + +.. zephyr:board-supported-hw:: + +USB +=== + +The USB pin assignments on the STM32N657XX microcontroller are immutable. This means that the specific +pins designated for USB functionality are fixed and cannot be changed or reassigned to other functions, +ensuring consistent and reliable USB communication. + +USB PIN (IOs) +============= + ++------------------+--------------------------------------+ +| Name | Description | ++==================+======================================+ +| OTG1_HSDM | USB OTG1 High-Speed Data- (negative) | ++------------------+--------------------------------------+ +| OTG1_HSDP | USB OTG1 High-Speed Data+ (positive) | ++------------------+--------------------------------------+ +| OTG1_ID | USB OTG1 ID Pin | ++------------------+--------------------------------------+ +| OTG1_TXRTUNE | USB OTG1 Transmit Retune | ++------------------+--------------------------------------+ +| OTG2_HSDM | USB OTG2 High-Speed Data- (negative) | ++------------------+--------------------------------------+ +| OTG2_HSDP | USB OTG2 High-Speed Data+ (positive) | ++------------------+--------------------------------------+ +| OTG2_ID | USB OTG2 ID Pin | ++------------------+--------------------------------------+ +| OTG2_TXRTUNE | USB OTG2 Transmit Retune | ++------------------+--------------------------------------+ + +Connections and IOs +=================== + +NUCLEO-N657X0-Q Board has 12 GPIO controllers. These controllers are responsible +for pin muxing, input/output, pull-up, etc. + +For more details please refer to `NUCLEO-N657X0-Q User Manual`_. + +Default Zephyr Peripheral Mapping: +---------------------------------- + +- ADC1_INP10 : PA9 +- ADC1_INP11 : PA10 +- FDCAN1_TX : PH2 +- FDCAN1_RX : PD0 +- I2C1_SCL : PH9 +- I2C1_SDA : PC1 +- I2C4_SCL : PE13 +- I2C4_SDA : PE14 +- LD1 : PO1 +- LD2 : PG10 +- SPI5_SCK : PE15 +- SPI5_MOSI : PG2 +- SPI5_MISO : PG1 +- SPI5_NSS : PA3 +- USART_1_TX : PE5 +- USART_1_RX : PE6 +- USART_3_TX : PD8 +- USART_3_RX : PD9 +- XSPI2_NCS1 : PN1 +- XSPI2_DQS0 : PN0 +- XSPI2_CLK : PN6 +- XSPI2_IO0 : PN2 +- XSPI2_IO1 : PN3 +- XSPI2_IO2 : PN4 +- XSPI2_IO3 : PN5 +- XSPI2_IO4 : PN8 +- XSPI2_IO5 : PN9 +- XSPI2_IO6 : PN10 +- XSPI2_IO7 : PN11 + +System Clock +------------ + +NUCLEO-N657X0-Q System Clock could be driven by internal or external oscillator, +as well as main PLL clock. By default System clock is driven by PLL clock at +400MHz, driven by 64MHz high speed internal oscillator. + +Serial Port +----------- + +NUCLEO-N657X0-Q board has 10 U(S)ARTs. The Zephyr console output is assigned to +USART1. Default settings are 115200 8N1. + +Programming and Debugging +************************* + +.. zephyr:board-supported-runners:: + +NUCLEO-N657X0-Q board includes an ST-LINK/V3 embedded debug tool interface. +This probe allows to flash and debug the board using various tools. + + + +Flashing or loading +=================== + +The board is configured to be programmed using west `STM32CubeProgrammer`_ runner, +so its :ref:`installation ` is needed. +Version 2.18.0 or later of `STM32CubeProgrammer`_ is required. + +.. note:: + Firmware is run in secure mode of execution, which requires a signature. + After build, the build system will automatically generate a signed version of the + binary using `STM32CubeProgrammer`_ utility ``STM32_SigningTool_CLI``. + This utility is installed along with `STM32CubeProgrammer`_, but make sure it is + available in your ``PATH`` variable. + +To program the board, there are two options: + +- Program the firmware in external flash. At boot, it will then be loaded on RAM + and executed from there. +- Optionally, it can also be taken advantage from the serial boot interface provided + by the boot ROM. In that case, firmware is directly loaded in RAM and executed from + there. It is not retained. + +Programming an application to NUCLEO-N657X0-Q +--------------------------------------------- + +Here is an example to build and run :zephyr:code-sample:`hello_world` application. + +First, connect the NUCLEO-N657X0-Q to your host computer using the ST-Link USB port. + + .. tabs:: + + .. group-tab:: ST-Link + + Build and flash an application using ``nucleo_n657x0_q`` target. + + .. zephyr-app-commands:: + :zephyr-app: samples/hello_world + :board: nucleo_n657x0_q + :goals: build flash + + .. note:: + For flashing, before powering the board, set the boot pins in the following configuration: + + * BOOT0: 0 + * BOOT1: 1 + + After flashing, to run the application, set the boot pins in the following configuration: + + * BOOT1: 0 + + Power off and on the board again. + + .. group-tab:: Serial Boot Loader (USB) + + Additionally, connect the NUCLEO-N657X0-Q to your host computer using the USB port. + In this configuration, ST-Link is used to power the board and for serial communication + over the Virtual COM Port. + + .. note:: + Before powering the board, set the boot pins in the following configuration: + + * BOOT0: 1 + * BOOT1: 0 + + Build and load an application using ``nucleo_n657x0_q/stm32n657xx/sb`` target (you + can also use the shortened form: ``nucleo_n657x0_q//sb``) + + .. zephyr-app-commands:: + :zephyr-app: samples/hello_world + :board: nucleo_n657x0_q + :goals: build flash + + +Run a serial host program to connect to your board: + +.. code-block:: console + + $ minicom -D /dev/ttyACM0 + +You should see the following message on the console: + +.. code-block:: console + + Hello World! nucleo_n657x0_q/stm32n657xx + + +Debugging +========= + +You can debug an application in the usual way using the :ref:`ST-LINK GDB Server `. +Here is an example for the :zephyr:code-sample:`hello_world` application. + +.. zephyr-app-commands:: + :zephyr-app: samples/hello_world + :board: nucleo_n657x0_q + :maybe-skip-config: + :goals: debug + +.. note:: + To enable debugging, before powering on the board, set the boot pins in the following configuration: + + * BOOT0: 0 + * BOOT1: 1 + +Another solution for debugging is to use STM32CubeIDE: + +* Go to :menuselection:`File --> Import` and select :menuselection:`C/C++ --> STM32 Cortex-M Executable`. +* In the :guilabel:`Executable` field, browse to your ``/build/zephyr/zephyr.elf``. +* In :guilabel:`MCU` field, select ``STM32N657X0HxQ``. +* Click on :guilabel:`Finish`. +* Finally, click on :guilabel:`Debug` to start the debugging session. + +Running tests with twister +========================== + +Due to the BOOT switches manipulation required when flashing the board using ``nucleo_n657x0_q`` +board target, it is only possible to run twister tests campaign on ``nucleo_n657x0_q/stm32n657xx/sb`` +board target which doesn't require BOOT pins changes to load and execute binaries. +To do so, it is advised to use Twister's hardware map feature with the following settings: + +.. code-block:: yaml + + - platform: nucleo_n657x0_q/stm32n657xx/sb + product: BOOT-SERIAL + pre_script: /boards/st/common/scripts/board_power_reset.sh + runner: stm32cubeprogrammer + +.. _NUCLEO-N657X0-Q website: + https://www.st.com/en/evaluation-tools/nucleo-n657x0-q.html + +.. _NUCLEO-N657X0-Q User Manual: + https://www.st.com/resource/en/user_manual/um3417-stm32n6-nucleo144-board-mb1940-stmicroelectronics.pdf +.. _STM32N657X0 on www.st.com: + https://www.st.com/en/microcontrollers-microprocessors/stm32n657x0.html + +.. _STM32N657 reference manual: + https://www.st.com/resource/en/reference_manual/rm0486-stm32n647657xx-armbased-32bit-mcus-stmicroelectronics.pdf + +.. _STM32CubeProgrammer: + https://www.st.com/en/development-tools/stm32cubeprog.html diff --git a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3.dts b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3.dts new file mode 100644 index 0000000..47c7089 --- /dev/null +++ b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3.dts @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2025 STMicroelectronics + * + * SPDX-License-Identifier: Apache-2.0 + */ + +/dts-v1/; +#include "grinreflex_n6_dk3_common.dtsi" + +/ { + model = "STMicroelectronics STM32N657X0-Q-NUCLEO board"; + compatible = "st,stm32n657x0-q-nucleo"; +}; diff --git a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_common.dtsi b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_common.dtsi new file mode 100644 index 0000000..882eb51 --- /dev/null +++ b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_common.dtsi @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2025 STMicroelectronics + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include + +/ { + chosen { + zephyr,console = &usart1; + zephyr,shell-uart = &usart1; + zephyr,sram = &axisram2; + spi-flash0 = &mx25um51245g; + }; + + aliases { + }; +}; + +&clk_hse { + hse-div2; + clock-frequency = ; + status = "okay"; +}; + +&clk_hsi { + hsi-div = <1>; + status = "okay"; +}; + +&pll1 { + clocks = <&clk_hse>; + div-m = <3>; + mul-n = <150>; + div-p1 = <1>; + div-p2 = <1>; + status = "okay"; +}; + +&pll3 { + clocks = <&clk_hse>; + div-m = <3>; + mul-n = <125>; + div-p1 = <1>; + div-p2 = <1>; + status = "okay"; +}; + +&ic1 { + pll-src = <1>; + ic-div = <3>; + status = "okay"; +}; + +&ic2 { + pll-src = <1>; + ic-div = <6>; + status = "okay"; +}; + +&ic3 { + pll-src = <1>; + ic-div = <6>; + status = "okay"; +}; + +&ic6 { + pll-src = <3>; + ic-div = <2>; + status = "okay"; +}; + +&ic11 { + pll-src = <1>; + ic-div = <3>; + status = "okay"; +}; + +&perck { + clocks = <&rcc STM32_SRC_HSI PER_SEL(0)>; + status = "okay"; +}; + +&cpusw { + clocks = <&rcc STM32_SRC_IC1 CPU_SEL(3)>; + clock-frequency = ; + status = "okay"; +}; + +&rcc { + /* ic2, ic6 & ic11 must all be enabled to set ic2 as SYSCLK */ + clocks = <&ic2>; + clock-frequency = ; + ahb-prescaler = <2>; + timg-prescaler = <2>; +}; + +&i2c1 { + clocks = <&rcc STM32_CLOCK(APB1, 21)>, + <&rcc STM32_SRC_CKPER I2C1_SEL(1)>; + pinctrl-0 = <&i2c1_scl_ph9 &i2c1_sda_pc1>; + pinctrl-names = "default"; + clock-frequency = ; + status = "okay"; +}; + +&spi4 { + clocks = <&rcc STM32_CLOCK(APB2, 20)>, + <&rcc STM32_SRC_CKPER SPI5_SEL(1)>; + pinctrl-0 = <&spi4_sck_pe2 &spi4_mosi_pb7>; + pinctrl-names = "default"; + status = "okay"; +}; + +&usart1 { + clocks = <&rcc STM32_CLOCK(APB2, 4)>, + <&rcc STM32_SRC_CKPER USART1_SEL(1)>; + pinctrl-0 = <&usart1_tx_pf13 &usart1_rx_pg8>; + pinctrl-names = "default"; + current-speed = <115200>; + status = "okay"; +}; + +&usart3 { + clocks = <&rcc STM32_CLOCK(APB1, 18)>, + <&rcc STM32_SRC_CKPER USART3_SEL(1)>; + pinctrl-0 = <&usart3_tx_pd8 &usart3_rx_pd9>; + pinctrl-names = "default"; + current-speed = <115200>; + status = "okay"; +}; + +&xspi2 { + pinctrl-0 = <&xspim_p2_ncs1_pn1 &xspim_p2_dqs0_pn0 &xspim_p2_clk_pn6 + &xspim_p2_io0_pn2 &xspim_p2_io1_pn3 &xspim_p2_io2_pn4 + &xspim_p2_io3_pn5 &xspim_p2_io4_pn8 &xspim_p2_io5_pn9 + &xspim_p2_io6_pn10 &xspim_p2_io7_pn11>; + pinctrl-names = "default"; + clocks = <&rcc STM32_CLOCK(AHB5, 12)>, + <&rcc STM32_SRC_IC3 XSPI1_SEL(2)>, + <&rcc STM32_CLOCK(AHB5, 13)>; + status = "okay"; + + mx25um51245g: ospi-nor-flash@0 { + compatible = "st,stm32-xspi-nor"; + reg = <0>; + size = ; /* 512 Mbits */ + ospi-max-frequency = ; + spi-bus-width = ; + data-rate = ; + four-byte-opcodes; + status = "okay"; + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + storage_partition: partition@3ff0000 { + label = "storage"; + reg = <0x3ff0000 DT_SIZE_K(64)>; + }; + }; + }; +}; diff --git a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_defconfig b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_defconfig new file mode 100644 index 0000000..01abfa9 --- /dev/null +++ b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_defconfig @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 STMicroelectronics + +# Enable UART driver +CONFIG_SERIAL=y + +# Enable GPIO +CONFIG_GPIO=y + +# Console +CONFIG_CONSOLE=y +CONFIG_UART_CONSOLE=y + +# Enable MPU +CONFIG_ARM_MPU=y + +# Enable HW stack protection +CONFIG_HW_STACK_PROTECTION=y + +# No internal Flash +CONFIG_XIP=n diff --git a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_sb.dts b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_sb.dts new file mode 100644 index 0000000..e7d5f34 --- /dev/null +++ b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_sb.dts @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2025 STMicroelectronics + * + * SPDX-License-Identifier: Apache-2.0 + */ + +/dts-v1/; +#include "nucleo_n657x0_q_common.dtsi" + +/ { + model = "STMicroelectronics STM32N657X0-Q-NUCLEO board"; + compatible = "st,stm32n657x0-q-nucleo-sb"; +}; diff --git a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_sb_defconfig b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_sb_defconfig new file mode 100644 index 0000000..33b0f28 --- /dev/null +++ b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_sb_defconfig @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 STMicroelectronics + +# Enable UART driver +CONFIG_SERIAL=y + +# Enable GPIO +CONFIG_GPIO=y + +# Console +CONFIG_CONSOLE=y +CONFIG_UART_CONSOLE=y + +# Enable MPU +CONFIG_ARM_MPU=y + +# Enable HW stack protection +CONFIG_HW_STACK_PROTECTION=y + +# No internal Flash +CONFIG_XIP=n + +# Use serial boot (USB) +CONFIG_STM32N6_BOOT_SERIAL=y diff --git a/u5/boards/vendor/grinreflex_n6_dk3/twister.yaml b/u5/boards/vendor/grinreflex_n6_dk3/twister.yaml new file mode 100644 index 0000000..6eb5afd --- /dev/null +++ b/u5/boards/vendor/grinreflex_n6_dk3/twister.yaml @@ -0,0 +1,18 @@ +name: grinreflex_n6_dk3 +type: mcu +arch: arm +ram: 1024 +flash: 1024 +supported: + - dma + - i2c + - spi + - uart +vendor: st +variants: + grinreflex_n6_dk3/stm32n657xx: + twister: false + grinreflex_n6_dk3/stm32n657xx/sb: + toolchain: + - zephyr + - gnuarmemb diff --git a/u5/common/CMakeLists.txt b/u5/common/CMakeLists.txt index 3fa94c7..ecb1a4b 100644 --- a/u5/common/CMakeLists.txt +++ b/u5/common/CMakeLists.txt @@ -10,11 +10,18 @@ endif() target_include_directories(app PUBLIC include) +if (CONFIG_VIDEO) target_sources(app PRIVATE - src/lvgl_utils.cpp src/video.cpp +) +endif() + +if (CONFIG_LVGL) +target_sources(app PRIVATE + src/lvgl_utils.cpp src/utils.cpp ) +endif() dt_has_chosen(USART2 PROPERTY "zephyr_usart2") diff --git a/u5/gf_n6_dk3/CMakeLists.txt b/u5/gf_n6_dk3/CMakeLists.txt new file mode 100644 index 0000000..ecb7d24 --- /dev/null +++ b/u5/gf_n6_dk3/CMakeLists.txt @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20.0) + +find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) +project(hello_world) + +target_sources(app PRIVATE src/main.c) diff --git a/u5/gf_n6_dk3/README.rst b/u5/gf_n6_dk3/README.rst new file mode 100644 index 0000000..c2533cc --- /dev/null +++ b/u5/gf_n6_dk3/README.rst @@ -0,0 +1,33 @@ +.. zephyr:code-sample:: hello_world + :name: Hello World + + Print "Hello World" to the console. + +Overview +******** + +A simple sample that can be used with any :ref:`supported board ` and +prints "Hello World" to the console. + +Building and Running +******************** + +This application can be built and executed on QEMU as follows: + +.. zephyr-app-commands:: + :zephyr-app: samples/hello_world + :host-os: unix + :board: qemu_x86 + :goals: run + :compact: + +To build for another board, change "qemu_x86" above to that board's name. + +Sample Output +============= + +.. code-block:: console + + Hello World! x86 + +Exit QEMU by pressing :kbd:`CTRL+A` :kbd:`x`. diff --git a/u5/gf_n6_dk3/prj.conf b/u5/gf_n6_dk3/prj.conf new file mode 100644 index 0000000..b2a4ba5 --- /dev/null +++ b/u5/gf_n6_dk3/prj.conf @@ -0,0 +1 @@ +# nothing here diff --git a/u5/gf_n6_dk3/sample.yaml b/u5/gf_n6_dk3/sample.yaml new file mode 100644 index 0000000..ac4dfea --- /dev/null +++ b/u5/gf_n6_dk3/sample.yaml @@ -0,0 +1,18 @@ +sample: + description: Hello World sample, the simplest Zephyr + application + name: hello world +common: + min_ram: 2 + min_flash: 16 + tags: introduction + integration_platforms: + - native_sim + harness: console + harness_config: + type: one_line + regex: + - "Hello World! (.*)" +tests: + sample.basic.helloworld: + tags: introduction diff --git a/u5/gf_n6_dk3/src/main.c b/u5/gf_n6_dk3/src/main.c new file mode 100644 index 0000000..c550ab4 --- /dev/null +++ b/u5/gf_n6_dk3/src/main.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2012-2014 Wind River Systems, Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +int main(void) +{ + printf("Hello World! %s\n", CONFIG_BOARD_TARGET); + + return 0; +} From 9d419f3fbec810beee3bcc55b62bb26b9ecdef7b Mon Sep 17 00:00:00 2001 From: Roman Pustobaiev Date: Mon, 24 Nov 2025 21:39:46 +0100 Subject: [PATCH 13/18] Adding custom stm32n6 board; derived from stm32n6 nucleo Draft commit --- .../vendor/grinreflex_n6_dk3/Kconfig.sysbuild | 3 +++ .../vendor/grinreflex_n6_dk3/board.cmake | 10 +++++-- u5/boards/vendor/grinreflex_n6_dk3/board.yml | 2 +- .../grinreflex_n6_dk3/grinreflex_n6_dk3.dts | 8 ++++-- .../grinreflex_n6_dk3_common.dtsi | 27 ++++++++++++++++--- .../grinreflex_n6_dk3_stm32n657xx_fsbl.dts | 8 ++++++ ...nreflex_n6_dk3_stm32n657xx_fsbl_defconfig} | 3 +-- .../grinreflex_n6_dk3_stm32n657xx_sb.dts | 13 --------- .../vendor/grinreflex_n6_dk3/sysbuild.cmake | 3 +++ u5/gf_n6_dk3/CMakeLists.txt | 2 +- u5/gf_n6_dk3/prj.conf | 6 +++++ u5/west.yml | 11 +++++--- 12 files changed, 69 insertions(+), 27 deletions(-) create mode 100644 u5/boards/vendor/grinreflex_n6_dk3/Kconfig.sysbuild create mode 100644 u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_fsbl.dts rename u5/boards/vendor/grinreflex_n6_dk3/{grinreflex_n6_dk3_stm32n657xx_sb_defconfig => grinreflex_n6_dk3_stm32n657xx_fsbl_defconfig} (85%) delete mode 100644 u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_sb.dts create mode 100644 u5/boards/vendor/grinreflex_n6_dk3/sysbuild.cmake diff --git a/u5/boards/vendor/grinreflex_n6_dk3/Kconfig.sysbuild b/u5/boards/vendor/grinreflex_n6_dk3/Kconfig.sysbuild new file mode 100644 index 0000000..60ae2af --- /dev/null +++ b/u5/boards/vendor/grinreflex_n6_dk3/Kconfig.sysbuild @@ -0,0 +1,3 @@ +choice BOOTLOADER + default BOOTLOADER_MCUBOOT +endchoice \ No newline at end of file diff --git a/u5/boards/vendor/grinreflex_n6_dk3/board.cmake b/u5/boards/vendor/grinreflex_n6_dk3/board.cmake index 1b42800..4a360d3 100644 --- a/u5/boards/vendor/grinreflex_n6_dk3/board.cmake +++ b/u5/boards/vendor/grinreflex_n6_dk3/board.cmake @@ -9,11 +9,17 @@ else() board_runner_args(stm32cubeprogrammer "--port=swd") board_runner_args(stm32cubeprogrammer "--tool-opt= mode=HOTPLUG ap=1") board_runner_args(stm32cubeprogrammer "--extload=MX25UM51245G_STM32N6570-NUCLEO.stldr") - board_runner_args(stm32cubeprogrammer "--download-address=0x70000000") + set(app_base_addr 0x70000000) + if(CONFIG_BOOTLOADER_MCUBOOT) + dt_nodelabel(slot0_partition NODELABEL "slot0_partition" REQUIRED) + dt_reg_addr(slot0_partition_addr PATH ${slot0_partition}) + math(EXPR app_base_addr "${app_base_addr} + ${slot0_partition_addr}") + endif() + board_runner_args(stm32cubeprogrammer "--download-address=${app_base_addr}") endif() board_runner_args(stlink_gdbserver "--apid=1") board_runner_args(stlink_gdbserver "--extload=MX25UM51245G_STM32N6570-NUCLEO.stldr") include(${ZEPHYR_BASE}/boards/common/stm32cubeprogrammer.board.cmake) -include(${ZEPHYR_BASE}/boards/common/stlink_gdbserver.board.cmake) +include(${ZEPHYR_BASE}/boards/common/stlink_gdbserver.board.cmake) \ No newline at end of file diff --git a/u5/boards/vendor/grinreflex_n6_dk3/board.yml b/u5/boards/vendor/grinreflex_n6_dk3/board.yml index 26fee19..977b21c 100644 --- a/u5/boards/vendor/grinreflex_n6_dk3/board.yml +++ b/u5/boards/vendor/grinreflex_n6_dk3/board.yml @@ -5,4 +5,4 @@ board: socs: - name: stm32n657xx variants: - - name: sb + - name: fsbl diff --git a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3.dts b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3.dts index 47c7089..d281d85 100644 --- a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3.dts +++ b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3.dts @@ -8,6 +8,10 @@ #include "grinreflex_n6_dk3_common.dtsi" / { - model = "STMicroelectronics STM32N657X0-Q-NUCLEO board"; - compatible = "st,stm32n657x0-q-nucleo"; + model = "Grinreflex N6 DK3"; + compatible = "gf,grinreflex-n6-dk3"; + + chosen { + zephyr,sram = &axisram1; + }; }; diff --git a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_common.dtsi b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_common.dtsi index 882eb51..a6c0aea 100644 --- a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_common.dtsi +++ b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_common.dtsi @@ -15,6 +15,9 @@ zephyr,shell-uart = &usart1; zephyr,sram = &axisram2; spi-flash0 = &mx25um51245g; + zephyr,flash-controller = &mx25um51245g; + zephyr,flash = &mx25um51245g; + zephyr,code-partition = &slot0_partition; }; aliases { @@ -148,7 +151,7 @@ mx25um51245g: ospi-nor-flash@0 { compatible = "st,stm32-xspi-nor"; reg = <0>; - size = ; /* 512 Mbits */ + size = ; /* 128 Mbits */ ospi-max-frequency = ; spi-bus-width = ; data-rate = ; @@ -160,9 +163,27 @@ #address-cells = <1>; #size-cells = <1>; - storage_partition: partition@3ff0000 { + /* + * Following flash partition is dedicated to the use of bootloader + */ + boot_partition: partition@0 { + label = "mcuboot"; + reg = <0x00000000 DT_SIZE_K(256)>; + }; + + slot0_partition: partition@40000 { + label = "image-0"; + reg = <0x40000 DT_SIZE_K(1536)>; + }; + + slot1_partition: partition@240000 { + label = "image-1"; + reg = <0x240000 DT_SIZE_K(1536)>; + }; + + storage_partition: partition@440000 { label = "storage"; - reg = <0x3ff0000 DT_SIZE_K(64)>; + reg = <0x440000 DT_SIZE_K(64)>; }; }; }; diff --git a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_fsbl.dts b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_fsbl.dts new file mode 100644 index 0000000..4ec8ee4 --- /dev/null +++ b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_fsbl.dts @@ -0,0 +1,8 @@ + +/dts-v1/; +#include "grinreflex_n6_dk3_common.dtsi" + +/ { + model = "Grinreflex N6 DK3"; + compatible = "gf,grinreflex-n6-dk3-fsbl"; +}; diff --git a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_sb_defconfig b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_fsbl_defconfig similarity index 85% rename from u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_sb_defconfig rename to u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_fsbl_defconfig index 33b0f28..56da754 100644 --- a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_sb_defconfig +++ b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_fsbl_defconfig @@ -20,5 +20,4 @@ CONFIG_HW_STACK_PROTECTION=y # No internal Flash CONFIG_XIP=n -# Use serial boot (USB) -CONFIG_STM32N6_BOOT_SERIAL=y +CONFIG_LOG_DEFAULT_LEVEL=4 \ No newline at end of file diff --git a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_sb.dts b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_sb.dts deleted file mode 100644 index e7d5f34..0000000 --- a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_sb.dts +++ /dev/null @@ -1,13 +0,0 @@ -/* - * Copyright (c) 2025 STMicroelectronics - * - * SPDX-License-Identifier: Apache-2.0 - */ - -/dts-v1/; -#include "nucleo_n657x0_q_common.dtsi" - -/ { - model = "STMicroelectronics STM32N657X0-Q-NUCLEO board"; - compatible = "st,stm32n657x0-q-nucleo-sb"; -}; diff --git a/u5/boards/vendor/grinreflex_n6_dk3/sysbuild.cmake b/u5/boards/vendor/grinreflex_n6_dk3/sysbuild.cmake new file mode 100644 index 0000000..c35e163 --- /dev/null +++ b/u5/boards/vendor/grinreflex_n6_dk3/sysbuild.cmake @@ -0,0 +1,3 @@ +if(SB_CONFIG_BOOTLOADER_MCUBOOT) + set_target_properties(mcuboot PROPERTIES BOARD grinreflex_n6_dk3/stm32n657xx/fsbl) +endif() \ No newline at end of file diff --git a/u5/gf_n6_dk3/CMakeLists.txt b/u5/gf_n6_dk3/CMakeLists.txt index ecb7d24..3e9efa9 100644 --- a/u5/gf_n6_dk3/CMakeLists.txt +++ b/u5/gf_n6_dk3/CMakeLists.txt @@ -3,6 +3,6 @@ cmake_minimum_required(VERSION 3.20.0) find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) -project(hello_world) +project(gf_n6_dk3) target_sources(app PRIVATE src/main.c) diff --git a/u5/gf_n6_dk3/prj.conf b/u5/gf_n6_dk3/prj.conf index b2a4ba5..6da2115 100644 --- a/u5/gf_n6_dk3/prj.conf +++ b/u5/gf_n6_dk3/prj.conf @@ -1 +1,7 @@ # nothing here +CONFIG_LOG_DEFAULT_LEVEL=4 + +#CONFIG_FAULT_DUMP=2 +CONFIG_ASSERT=y +#CONFIG_RESET_ON_FATAL_ERROR=n +#CONFIG_LOG=y diff --git a/u5/west.yml b/u5/west.yml index a2f4ecf..c52411e 100644 --- a/u5/west.yml +++ b/u5/west.yml @@ -11,12 +11,12 @@ manifest: remotes: - name: upstream url-base: https://github.com/zephyrproject-rtos/ - - name: zephyrproject-rtos + - name: origin url-base: https://github.com/cornway/ projects: - name: zephyr - remote: zephyrproject-rtos + remote: origin revision: roman/development import: # By using name-allowlist we can clone only the modules that are @@ -25,4 +25,9 @@ manifest: - cmsis_6 # required by the ARM port for Cortex-M - hal_stm32 - tflite-micro - - lvgl \ No newline at end of file + - lvgl + - mbedtls + - name: mcuboot + remote: origin + revision: roman/development + path: bootloader/mcuboot \ No newline at end of file From a9c7bc9e7a99a8a5076e9f66551eed2b92e04389 Mon Sep 17 00:00:00 2001 From: Roman Pustobaiev Date: Wed, 26 Nov 2025 12:41:37 +0100 Subject: [PATCH 14/18] More tweaks to add stm32n6 support --- .gitignore | 1 + README.md | 3 +- stm32hal.patch | 20 +++ .../Kconfig.grinreflex_n6_dk3 | 35 +++++ .../vendor/grinreflex_n6_dk3/board.cmake | 2 +- u5/boards/vendor/grinreflex_n6_dk3/board.yml | 1 + .../grinreflex_n6_dk3/grinreflex_n6_dk3.dts | 24 +++ .../grinreflex_n6_dk3_common.dtsi | 148 ++++++++++++++++++ .../grinreflex_n6_dk3_defconfig | 24 ++- .../grinreflex_n6_dk3_stm32n657xx_app.dts | 8 + ...rinreflex_n6_dk3_stm32n657xx_app_defconfig | 31 ++++ .../grinreflex_n6_dk3_stm32n657xx_fsbl.dts | 24 +++ ...grinreflex_n6_dk3_stm32n657xx_fsbl_app.dts | 8 + ...flex_n6_dk3_stm32n657xx_fsbl_app_defconfig | 31 ++++ u5/common/src/video.cpp | 3 - u5/drivers/jpeg/jpeg.c | 7 + u5/drivers/jpeg/jpeg_utils_conf.h | 8 + u5/gf_dkx/boards/grinreflex_n6_dk3.conf | 0 u5/gf_dkx/boards/grinreflex_n6_dk3.overlay | 0 u5/gf_dkx/prj.conf | 7 +- u5/gf_n6_dk3/prj.conf | 2 + u5/gf_n6_dk3/src/main.c | 12 +- 22 files changed, 385 insertions(+), 14 deletions(-) create mode 100644 u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_app.dts create mode 100644 u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_app_defconfig create mode 100644 u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_fsbl_app.dts create mode 100644 u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_fsbl_app_defconfig create mode 100644 u5/gf_dkx/boards/grinreflex_n6_dk3.conf create mode 100644 u5/gf_dkx/boards/grinreflex_n6_dk3.overlay diff --git a/.gitignore b/.gitignore index 282eb22..29f7521 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ zephyr build modules .vscode +bootloader diff --git a/README.md b/README.md index a4e04f5..8f35552 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,8 @@ For a fresh start * `git submodule init --update` * `west init -l u5` * `west update` -* `west build -b stm32u5g9j_dk2 u5/app/ -- -DBOARD_ROOT=./../` (builds for stm32u5g9j_dk2 board, you can google what that board is) +# Outdated +* `west build -b stm32u5g9j_dk2 u5/app/` (builds for stm32u5g9j_dk2 board, you can google what that board is) * `west flash` ### Important note: since stm32u5g9j_dk2 doesn't have camera sensor, you need to find out your own way how to transfer images to the board; I used uart for that purpose diff --git a/stm32hal.patch b/stm32hal.patch index 4b7fe0e..d4878f1 100644 --- a/stm32hal.patch +++ b/stm32hal.patch @@ -1,3 +1,23 @@ +diff --git a/stm32cube/stm32n6xx/drivers/src/stm32n6xx_hal_dma_ex.c b/stm32cube/stm32n6xx/drivers/src/stm32n6xx_hal_dma_ex.c +index 71c10570..df0bc157 100644 +--- a/stm32cube/stm32n6xx/drivers/src/stm32n6xx_hal_dma_ex.c ++++ b/stm32cube/stm32n6xx/drivers/src/stm32n6xx_hal_dma_ex.c +@@ -520,6 +520,9 @@ + /* Includes ----------------------------------------------------------------------------------------------------------*/ + #include "stm32n6xx_hal.h" + ++#pragma GCC push_options ++#pragma GCC optimize ("O0") ++ + /** @addtogroup STM32N6xx_HAL_Driver + * @{ + */ +@@ -4738,3 +4741,5 @@ static void DMA_List_CleanQueue(DMA_QListTypeDef *const pQList) + /** + * @} + */ ++ ++#pragma GCC pop_options diff --git a/stm32cube/stm32u5xx/drivers/src/stm32u5xx_hal_dma_ex.c b/stm32cube/stm32u5xx/drivers/src/stm32u5xx_hal_dma_ex.c index d60eca23..708ff379 100644 --- a/stm32cube/stm32u5xx/drivers/src/stm32u5xx_hal_dma_ex.c diff --git a/u5/boards/vendor/grinreflex_n6_dk3/Kconfig.grinreflex_n6_dk3 b/u5/boards/vendor/grinreflex_n6_dk3/Kconfig.grinreflex_n6_dk3 index 22ec64e..34353ff 100644 --- a/u5/boards/vendor/grinreflex_n6_dk3/Kconfig.grinreflex_n6_dk3 +++ b/u5/boards/vendor/grinreflex_n6_dk3/Kconfig.grinreflex_n6_dk3 @@ -5,3 +5,38 @@ config BOARD_GRINREFLEX_N6_DK3 select SOC_STM32N657XX + +menu "GrinReflex Video settings" + +config GRINREFLEX_JPEG_VIDEO + bool "Enable jpeg video input" + default y + +config GRINREFLEX_JPEG_VIDEO_QUALITY + int "Set jpeg quality, lower value - higher quality" + default 63 + depends on GRINREFLEX_JPEG_VIDEO + +config GRINREFLEX_VIDEO_WIDTH + int "Width of a video frame" + default 160 + +config GRINREFLEX_VIDEO_HEIGHT + int "Height of a video frame" + default 120 + +config GRINREFLEX_VIDEO_RGB565_SWAP_BYTES + bool "Swap rgb565 bytes (e.g. in ov2640 sensor)" + default n + +endmenu + +menu "GrinReflex LVGL settings" + +if GRINREFLEX_JPEG_VIDEO +config LV_COLOR_16_SWAP + bool + default y +endif + +endmenu \ No newline at end of file diff --git a/u5/boards/vendor/grinreflex_n6_dk3/board.cmake b/u5/boards/vendor/grinreflex_n6_dk3/board.cmake index 4a360d3..a61bde4 100644 --- a/u5/boards/vendor/grinreflex_n6_dk3/board.cmake +++ b/u5/boards/vendor/grinreflex_n6_dk3/board.cmake @@ -9,7 +9,7 @@ else() board_runner_args(stm32cubeprogrammer "--port=swd") board_runner_args(stm32cubeprogrammer "--tool-opt= mode=HOTPLUG ap=1") board_runner_args(stm32cubeprogrammer "--extload=MX25UM51245G_STM32N6570-NUCLEO.stldr") - set(app_base_addr 0x70000000) + set(app_base_addr 0x70000000) if(CONFIG_BOOTLOADER_MCUBOOT) dt_nodelabel(slot0_partition NODELABEL "slot0_partition" REQUIRED) dt_reg_addr(slot0_partition_addr PATH ${slot0_partition}) diff --git a/u5/boards/vendor/grinreflex_n6_dk3/board.yml b/u5/boards/vendor/grinreflex_n6_dk3/board.yml index 977b21c..f7b6acb 100644 --- a/u5/boards/vendor/grinreflex_n6_dk3/board.yml +++ b/u5/boards/vendor/grinreflex_n6_dk3/board.yml @@ -6,3 +6,4 @@ board: - name: stm32n657xx variants: - name: fsbl + - name: fsbl_app diff --git a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3.dts b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3.dts index d281d85..4a57955 100644 --- a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3.dts +++ b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3.dts @@ -15,3 +15,27 @@ zephyr,sram = &axisram1; }; }; + +&st7735r_160x80 { + status = "disabled"; +}; + +&jpeg0 { + status = "disabled"; +}; + +&gpdma1 { + status = "disabled"; +}; + +&ov2640 { + status = "disabled"; +}; + +&zephyr_camera_dvp { + status = "disabled"; +}; + +&mco1 { + status = "disabled"; +}; \ No newline at end of file diff --git a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_common.dtsi b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_common.dtsi index a6c0aea..4d4b8e8 100644 --- a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_common.dtsi +++ b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_common.dtsi @@ -18,10 +18,82 @@ zephyr,flash-controller = &mx25um51245g; zephyr,flash = &mx25um51245g; zephyr,code-partition = &slot0_partition; + zephyr,jpeg = &jpeg0; + zephyr,display = &st7735r_160x80; + zephyr,camera = &zephyr_camera_dvp; }; aliases { }; + + mipi_dbi_st7735r_160x80 { + compatible = "zephyr,mipi-dbi-spi"; + spi-dev = <&spi4>; + dc-gpios = <&gpioc 5 GPIO_ACTIVE_HIGH>; + #address-cells = <1>; + #size-cells = <0>; + + st7735r_160x80: st7735r@0 { + compatible = "sitronix,st7735r"; + mipi-max-frequency = <20000000>; + mipi-mode = "MIPI_DBI_MODE_SPI_4WIRE"; + reg = <0>; + width = <160>; + height = <80>; + inversion-on; + rgb-is-inverted; + x-offset = <1>; + y-offset = <26>; + pwctr1 = [A2 02 84]; + pwctr2 = [C5]; + pwctr3 = [0A 00]; + pwctr4 = [8A 2A]; + pwctr5 = [8A EE]; + invctr = <7>; + frmctr1 = [01 2C 2D]; + frmctr2 = [01 2C 2D]; + frmctr3 = [01 2C 2D 01 2C 2D]; + vmctr1 = <14>; + gamctrp1 = [02 1C 07 12 37 32 29 2D 29 25 2B 39 00 01 03 10]; + gamctrn1 = [03 1D 07 06 2E 2C 29 2D 2E 2E 37 3F 00 00 02 10]; + colmod = <5>; + /* Set D3 (RGB) bit to 1. LV_COLOR_16_SWAP is enabled by default */ + madctl = <184>; /* Set to <184> to rotate the image 180 degrees. */ + caset = [00 01 00 a0]; + raset = [00 1a 00 69]; + }; + }; + + dcmi_camera_connector: connector_dcmi_camera { + compatible = "weact,dcmi-camera-connector"; + #gpio-cells = <2>; + gpio-map-mask = <0xffffffff 0xffffffc0>; + gpio-map-pass-thru = <0 0x3f>; + + gpio-map = <3 0 &gpioc 1 0>, /* DVP_SDA (I2C2_SDA)*/ + <5 0 &gpioh 9 0>, /* DVP_SCL (I2C2_SCL)*/ + <7 0 &gpioe 6 0>, /* DVP_VSYNC*/ + <8 0 &gpioc 4 0>, /* DVP_PWDN NC*/ + <9 0 &gpioc 0 0>, /* DVP_HSYNC*/ + <12 0 &gpiof 1 0>, /* DVP_D7*/ + <13 0 &gpioa 8 0>, /* DVP_XCLK (RCC_MCO1)*/ + <14 0 &gpiof 5 0>, /* DVP_D6*/ + <16 0 &gpioe 4 0>, /* DVP_D5*/ + <17 0 &gpioa 6 0>, /* DVP_PCLK*/ + <18 0 &gpioc 11 0>, /* DVP_D4*/ + <19 0 &gpioa 1 0>, /* DVP_D0*/ + <20 0 &gpioc 9 0>, /* DVP_D3 */ + <21 0 &gpioa 10 0>, /* DVP_D1 */ + <22 0 &gpioc 3 0>; /* DVP_D2 */ + }; + + jpeg0: jpeg_hw { + compatible = "st,jpeg-hw"; + dmas = <&gpdma1 2 1 (STM32_DMA_PERIPH_RX | STM32_DMA_PERIPH_32BITS | STM32_DMA_MEM_32BITS) + &gpdma1 3 0 (STM32_DMA_PERIPH_TX | STM32_DMA_PERIPH_32BITS | STM32_DMA_MEM_32BITS)>; + dma-names = "tx", "rx"; + status = "okay"; + }; }; &clk_hse { @@ -102,6 +174,16 @@ timg-prescaler = <2>; }; +&gpioc { + status = "okay"; + + lcd_led { + gpio-hog; + gpios = <6 GPIO_ACTIVE_LOW>; + output-high; + }; +}; + &i2c1 { clocks = <&rcc STM32_CLOCK(APB1, 21)>, <&rcc STM32_SRC_CKPER I2C1_SEL(1)>; @@ -109,12 +191,29 @@ pinctrl-names = "default"; clock-frequency = ; status = "okay"; + + ov2640: ov2640@30 { + compatible = "ovti,ov2640"; + reg = <0x30>; + + supply-gpios = <&dcmi_camera_connector 8 GPIO_ACTIVE_HIGH>; + clock-rate-control = <0x80>; + + status = "okay"; + + port { + ov2640_ep_out: endpoint { + remote-endpoint-label = "zephyr_camera_dvp_in"; + }; + }; + }; }; &spi4 { clocks = <&rcc STM32_CLOCK(APB2, 20)>, <&rcc STM32_SRC_CKPER SPI5_SEL(1)>; pinctrl-0 = <&spi4_sck_pe2 &spi4_mosi_pb7>; + cs-gpios = <&gpioe 7 (GPIO_ACTIVE_LOW | GPIO_PULL_UP)>; pinctrl-names = "default"; status = "okay"; }; @@ -188,3 +287,52 @@ }; }; }; + +&gpdma1 { + status = "okay"; +}; + +zephyr_camera_dvp: &dcmi { + pinctrl-0 = <&dcmi_hsync_pc0 + &dcmi_pixclk_pa6 + &dcmi_vsync_pe6 + &dcmi_d0_pa1 + &dcmi_d1_pa10 + &dcmi_d2_pc3 + &dcmi_d3_pc9 + &dcmi_d4_pc11 + &dcmi_d5_pe4 + &dcmi_d6_pf5 + &dcmi_d7_pf1>; + pinctrl-names = "default"; + + port { + zephyr_camera_dvp_in: endpoint { + remote-endpoint-label = "ov2640_ep_out"; + bus-width = <8>; + hsync-active = <0>; + vsync-active = <0>; + pclk-sample = <1>; + }; + }; + status = "okay"; +}; + + +/* See reference manual (RM0486 Rev 2) page 433: + * 4: HSE48 clock selected (hse_ck) + */ +#define MCO1_SEL_HSE48 4 + + /* See reference manual (RM0486 Rev 2) page 531: + * 0011: division by 4 + */ +#define MCO1_PRE_DIV_4 3 + +&mco1 { + status = "okay"; + clocks = <&rcc MCO1_SEL_HSE48 MCO1_SEL(MCO1_SEL_HSE48)>; + prescaler = ; + pinctrl-0 = <&rcc_mco_1_pa8>; + pinctrl-names = "default"; +}; diff --git a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_defconfig b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_defconfig index 01abfa9..fa22c69 100644 --- a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_defconfig +++ b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_defconfig @@ -10,12 +10,26 @@ CONFIG_GPIO=y # Console CONFIG_CONSOLE=y CONFIG_UART_CONSOLE=y - -# Enable MPU -CONFIG_ARM_MPU=y +CONFIG_EARLY_CONSOLE=y # Enable HW stack protection CONFIG_HW_STACK_PROTECTION=y -# No internal Flash -CONFIG_XIP=n +CONFIG_ARM_MPU=n +#CONFIG_XIP=y + +#CONFIG_VIDEO=y +#CONFIG_VIDEO_LOG_LEVEL_DBG=y +#CONFIG_VIDEO_LOG_LEVEL_INF=y + +#CONFIG_STM32_JPEG_RGB_FORMAT=3 +#CONFIG_GRINREFLEX_JPEG_VIDEO=y +#CONFIG_GRINREFLEX_JPEG_VIDEO_QUALITY=10 +#CONFIG_GRINREFLEX_VIDEO_WIDTH=640 +#CONFIG_GRINREFLEX_VIDEO_HEIGHT=480 + +#CONFIG_VIDEO_BUFFER_POOL_SZ_MAX=76800 +#CONFIG_VIDEO_BUFFER_POOL_SZ_MAX=153600 + +#CONFIG_DISPLAY=y +#CONFIG_DISPLAY_LOG_LEVEL_ERR=y diff --git a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_app.dts b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_app.dts new file mode 100644 index 0000000..4ec8ee4 --- /dev/null +++ b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_app.dts @@ -0,0 +1,8 @@ + +/dts-v1/; +#include "grinreflex_n6_dk3_common.dtsi" + +/ { + model = "Grinreflex N6 DK3"; + compatible = "gf,grinreflex-n6-dk3-fsbl"; +}; diff --git a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_app_defconfig b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_app_defconfig new file mode 100644 index 0000000..7691842 --- /dev/null +++ b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_app_defconfig @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 STMicroelectronics + +# Enable UART driver +CONFIG_SERIAL=y + +# Enable GPIO +CONFIG_GPIO=y + +# Console +CONFIG_CONSOLE=y +CONFIG_UART_CONSOLE=y +#CONFIG_EARLY_CONSOLE=y + +# Enable MPU +CONFIG_ARM_MPU=y + +# Enable HW stack protection +CONFIG_HW_STACK_PROTECTION=y + +# No internal Flash +CONFIG_XIP=n + +CONFIG_LOG=y +CONFIG_LOG_DEFAULT_LEVEL=4 + +CONFIG_VIDEO=y +CONFIG_VIDEO_LOG_LEVEL_INF=y + +CONFIG_SHELL=y +#CONFIG_VIDEO_SHELL=y \ No newline at end of file diff --git a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_fsbl.dts b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_fsbl.dts index 4ec8ee4..ee80615 100644 --- a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_fsbl.dts +++ b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_fsbl.dts @@ -6,3 +6,27 @@ model = "Grinreflex N6 DK3"; compatible = "gf,grinreflex-n6-dk3-fsbl"; }; + +&st7735r_160x80 { + status = "disabled"; +}; + +&jpeg0 { + status = "disabled"; +}; + +&gpdma1 { + status = "disabled"; +}; + +&ov2640 { + status = "disabled"; +}; + +&zephyr_camera_dvp { + status = "disabled"; +}; + +&mco1 { + status = "disabled"; +}; diff --git a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_fsbl_app.dts b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_fsbl_app.dts new file mode 100644 index 0000000..4ec8ee4 --- /dev/null +++ b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_fsbl_app.dts @@ -0,0 +1,8 @@ + +/dts-v1/; +#include "grinreflex_n6_dk3_common.dtsi" + +/ { + model = "Grinreflex N6 DK3"; + compatible = "gf,grinreflex-n6-dk3-fsbl"; +}; diff --git a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_fsbl_app_defconfig b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_fsbl_app_defconfig new file mode 100644 index 0000000..7691842 --- /dev/null +++ b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_fsbl_app_defconfig @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 STMicroelectronics + +# Enable UART driver +CONFIG_SERIAL=y + +# Enable GPIO +CONFIG_GPIO=y + +# Console +CONFIG_CONSOLE=y +CONFIG_UART_CONSOLE=y +#CONFIG_EARLY_CONSOLE=y + +# Enable MPU +CONFIG_ARM_MPU=y + +# Enable HW stack protection +CONFIG_HW_STACK_PROTECTION=y + +# No internal Flash +CONFIG_XIP=n + +CONFIG_LOG=y +CONFIG_LOG_DEFAULT_LEVEL=4 + +CONFIG_VIDEO=y +CONFIG_VIDEO_LOG_LEVEL_INF=y + +CONFIG_SHELL=y +#CONFIG_VIDEO_SHELL=y \ No newline at end of file diff --git a/u5/common/src/video.cpp b/u5/common/src/video.cpp index f979b93..6f87ec4 100644 --- a/u5/common/src/video.cpp +++ b/u5/common/src/video.cpp @@ -1,7 +1,5 @@ -#include -#include #include #include "gf/video.hpp" @@ -32,7 +30,6 @@ void setup() { int i = 0; int err; - video_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_camera)); if (!device_is_ready(video_dev)) { LOG_ERR("%s device is not ready", video_dev->name); return; diff --git a/u5/drivers/jpeg/jpeg.c b/u5/drivers/jpeg/jpeg.c index d6a6b00..8ff55b9 100644 --- a/u5/drivers/jpeg/jpeg.c +++ b/u5/drivers/jpeg/jpeg.c @@ -15,7 +15,14 @@ #include #include + +#if defined(CONFIG_SOC_SERIES_STM32N6X) +#include +#elif defined(CONFIG_SOC_SERIES_STM32U5X) #include +#else +#error "JPEG: soc is not supported yet" +#endif #include "jpeg_utils.h" diff --git a/u5/drivers/jpeg/jpeg_utils_conf.h b/u5/drivers/jpeg/jpeg_utils_conf.h index b542a08..9328a18 100644 --- a/u5/drivers/jpeg/jpeg_utils_conf.h +++ b/u5/drivers/jpeg/jpeg_utils_conf.h @@ -17,8 +17,16 @@ /* Includes ------------------------------------------------------------------*/ +#if defined(CONFIG_SOC_SERIES_STM32N6X) +#include +#include +#elif defined(CONFIG_SOC_SERIES_STM32U5X) #include "stm32u5xx_hal.h" #include "stm32u5xx_hal_jpeg.h" +#else +#error "JPEG: soc is not supported yet" +#endif + /* Private define ------------------------------------------------------------*/ /** @addtogroup JPEG_Private_Defines diff --git a/u5/gf_dkx/boards/grinreflex_n6_dk3.conf b/u5/gf_dkx/boards/grinreflex_n6_dk3.conf new file mode 100644 index 0000000..e69de29 diff --git a/u5/gf_dkx/boards/grinreflex_n6_dk3.overlay b/u5/gf_dkx/boards/grinreflex_n6_dk3.overlay new file mode 100644 index 0000000..e69de29 diff --git a/u5/gf_dkx/prj.conf b/u5/gf_dkx/prj.conf index 9190576..7e27c8b 100644 --- a/u5/gf_dkx/prj.conf +++ b/u5/gf_dkx/prj.conf @@ -33,7 +33,8 @@ CONFIG_PICOLIBC=n CONFIG_COMMON_LIBC_MALLOC=y CONFIG_HEAP_MEM_POOL_SIZE=8192 #CONFIG_COMMON_LIBC_MALLOC_ARENA_SIZE=2097152 -CONFIG_COMMON_LIBC_MALLOC_ARENA_SIZE=1503232 +#CONFIG_COMMON_LIBC_MALLOC_ARENA_SIZE=1503232 +CONFIG_COMMON_LIBC_MALLOC_ARENA_SIZE=524288 CONFIG_POSIX_API=y @@ -49,9 +50,9 @@ CONFIG_ASSERT_VERBOSE=y CONFIG_SPEED_OPTIMIZATIONS=y #CONFIG_TFLITE_LIB=y -CONFIG_OPENCV_LIB=y +#CONFIG_OPENCV_LIB=y #CONFIG_QUIRC_LIB=y -CONFIG_NEMA=y +#CONFIG_NEMA=y CONFIG_FPU=y CONFIG_FP_SOFTABI=y diff --git a/u5/gf_n6_dk3/prj.conf b/u5/gf_n6_dk3/prj.conf index 6da2115..b4bda07 100644 --- a/u5/gf_n6_dk3/prj.conf +++ b/u5/gf_n6_dk3/prj.conf @@ -5,3 +5,5 @@ CONFIG_LOG_DEFAULT_LEVEL=4 CONFIG_ASSERT=y #CONFIG_RESET_ON_FATAL_ERROR=n #CONFIG_LOG=y + +CONFIG_ARM_MPU=n \ No newline at end of file diff --git a/u5/gf_n6_dk3/src/main.c b/u5/gf_n6_dk3/src/main.c index c550ab4..d150958 100644 --- a/u5/gf_n6_dk3/src/main.c +++ b/u5/gf_n6_dk3/src/main.c @@ -4,11 +4,21 @@ * SPDX-License-Identifier: Apache-2.0 */ +#include #include +// #define LOG_LEVEL CONFIG_LOG_DEFAULT_LEVEL +// #include +// LOG_MODULE_REGISTER(main); + +// static const struct device *video_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_camera)); + int main(void) { printf("Hello World! %s\n", CONFIG_BOARD_TARGET); - + // if (!device_is_ready(video_dev)) { + // LOG_ERR("%s device is not ready", video_dev->name); + // } + // LOG_DBG("%s", video_dev->name); return 0; } From 995db566a4ac9752091c94e74d5a780cd6969572 Mon Sep 17 00:00:00 2001 From: Roman Pustobaiev Date: Wed, 26 Nov 2025 14:14:15 +0100 Subject: [PATCH 15/18] More tweaks to add stm32n6 support Phase #2 I required to add CONFIG_DISPLAY_INIT_PRIORITY=100 Otherwise spi4, somehow, interfered with XSPI or something ? Causing device to brick --- u5/boards/vendor/grinreflex_n6_dk3/board.yml | 1 + .../grinreflex_n6_dk3_common.dtsi | 14 +- .../grinreflex_n6_dk3_stm32n657xx_app.dts | 11 +- ...rinreflex_n6_dk3_stm32n657xx_app_defconfig | 55 +++++++- u5/gf_dkx/src/lvgl_video_app.cpp | 7 - u5/gf_n6_dk3/src/application.cpp | 120 ++++++++++++++++++ 6 files changed, 182 insertions(+), 26 deletions(-) create mode 100644 u5/gf_n6_dk3/src/application.cpp diff --git a/u5/boards/vendor/grinreflex_n6_dk3/board.yml b/u5/boards/vendor/grinreflex_n6_dk3/board.yml index f7b6acb..4f7f97a 100644 --- a/u5/boards/vendor/grinreflex_n6_dk3/board.yml +++ b/u5/boards/vendor/grinreflex_n6_dk3/board.yml @@ -7,3 +7,4 @@ board: variants: - name: fsbl - name: fsbl_app + - name: app diff --git a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_common.dtsi b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_common.dtsi index 4d4b8e8..e67a181 100644 --- a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_common.dtsi +++ b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_common.dtsi @@ -210,11 +210,12 @@ }; &spi4 { - clocks = <&rcc STM32_CLOCK(APB2, 20)>, - <&rcc STM32_SRC_CKPER SPI5_SEL(1)>; + clocks = <&rcc STM32_CLOCK(APB2, 13)>, + <&rcc STM32_SRC_CKPER SPI4_SEL(1)>; pinctrl-0 = <&spi4_sck_pe2 &spi4_mosi_pb7>; cs-gpios = <&gpioe 7 (GPIO_ACTIVE_LOW | GPIO_PULL_UP)>; pinctrl-names = "default"; + clock-frequency = ; status = "okay"; }; @@ -227,15 +228,6 @@ status = "okay"; }; -&usart3 { - clocks = <&rcc STM32_CLOCK(APB1, 18)>, - <&rcc STM32_SRC_CKPER USART3_SEL(1)>; - pinctrl-0 = <&usart3_tx_pd8 &usart3_rx_pd9>; - pinctrl-names = "default"; - current-speed = <115200>; - status = "okay"; -}; - &xspi2 { pinctrl-0 = <&xspim_p2_ncs1_pn1 &xspim_p2_dqs0_pn0 &xspim_p2_clk_pn6 &xspim_p2_io0_pn2 &xspim_p2_io1_pn3 &xspim_p2_io2_pn4 diff --git a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_app.dts b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_app.dts index 4ec8ee4..d281d85 100644 --- a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_app.dts +++ b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_app.dts @@ -1,8 +1,17 @@ +/* + * Copyright (c) 2025 STMicroelectronics + * + * SPDX-License-Identifier: Apache-2.0 + */ /dts-v1/; #include "grinreflex_n6_dk3_common.dtsi" / { model = "Grinreflex N6 DK3"; - compatible = "gf,grinreflex-n6-dk3-fsbl"; + compatible = "gf,grinreflex-n6-dk3"; + + chosen { + zephyr,sram = &axisram1; + }; }; diff --git a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_app_defconfig b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_app_defconfig index 7691842..8607387 100644 --- a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_app_defconfig +++ b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_app_defconfig @@ -10,16 +10,13 @@ CONFIG_GPIO=y # Console CONFIG_CONSOLE=y CONFIG_UART_CONSOLE=y -#CONFIG_EARLY_CONSOLE=y - -# Enable MPU -CONFIG_ARM_MPU=y +CONFIG_EARLY_CONSOLE=y # Enable HW stack protection CONFIG_HW_STACK_PROTECTION=y -# No internal Flash -CONFIG_XIP=n +CONFIG_ARM_MPU=n +#CONFIG_XIP=y CONFIG_LOG=y CONFIG_LOG_DEFAULT_LEVEL=4 @@ -28,4 +25,48 @@ CONFIG_VIDEO=y CONFIG_VIDEO_LOG_LEVEL_INF=y CONFIG_SHELL=y -#CONFIG_VIDEO_SHELL=y \ No newline at end of file + +CONFIG_VIDEO=y +CONFIG_VIDEO_LOG_LEVEL_DBG=y +CONFIG_VIDEO_LOG_LEVEL_INF=y + +#CONFIG_STM32_JPEG_RGB_FORMAT=3 +#CONFIG_GRINREFLEX_JPEG_VIDEO=y +#CONFIG_GRINREFLEX_JPEG_VIDEO_QUALITY=10 +#CONFIG_GRINREFLEX_VIDEO_WIDTH=640 +#CONFIG_GRINREFLEX_VIDEO_HEIGHT=480 + +#CONFIG_VIDEO_BUFFER_POOL_SZ_MAX=76800 +CONFIG_VIDEO_BUFFER_POOL_SZ_MAX=153600 + +CONFIG_DISPLAY=y +CONFIG_DISPLAY_LOG_LEVEL_ERR=y +CONFIG_DISPLAY_LOG_LEVEL_DBG=y + +CONFIG_SPI_LOG_LEVEL_DBG=y +CONFIG_CLOCK_CONTROL_LOG_LEVEL_DBG=y +CONFIG_PINCTRL_LOG_LEVEL_DBG=y + +#CONFIG_LV_Z_MEM_POOL_SIZE=16384 +#CONFIG_LV_Z_SHELL=n +#CONFIG_LVGL=y +#CONFIG_LV_USE_LOG=n +#CONFIG_LV_USE_LABEL=n +#CONFIG_LV_USE_ARC=y +#CONFIG_LV_USE_IMAGE=y +#CONFIG_LV_FONT_MONTSERRAT_14=y + +#CONFIG_LV_CACHE_DEF_SIZE=4096 +#CONFIG_LV_IMAGE_HEADER_CACHE_DEF_CNT=1 +#CONFIG_LV_USE_BUILTIN_MALLOC=n + +#CONFIG_LV_USE_PERF_MONITOR=y +#CONFIG_LV_USE_DROPDOWN=y +#CONFIG_LV_USE_SYSMON=y + +#CONFIG_SPI=y +#CONFIG_SPI_SHELL=y + +CONFIG_SPI_STM32=y +CONFIG_SPI_STM32_USE_HW_SS=n +CONFIG_DISPLAY_INIT_PRIORITY=100 \ No newline at end of file diff --git a/u5/gf_dkx/src/lvgl_video_app.cpp b/u5/gf_dkx/src/lvgl_video_app.cpp index 674cd42..f9d8012 100644 --- a/u5/gf_dkx/src/lvgl_video_app.cpp +++ b/u5/gf_dkx/src/lvgl_video_app.cpp @@ -1,13 +1,6 @@ #include #include - -#if defined(CONFIG_OPENCV_LIB) -#include "cascades.h" -#include "opencv2/opencv.hpp" -#include "opencv_utils.hpp" -#endif - #include #include #include diff --git a/u5/gf_n6_dk3/src/application.cpp b/u5/gf_n6_dk3/src/application.cpp new file mode 100644 index 0000000..f9d8012 --- /dev/null +++ b/u5/gf_n6_dk3/src/application.cpp @@ -0,0 +1,120 @@ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#define LOG_LEVEL CONFIG_LOG_DEFAULT_LEVEL +#include +LOG_MODULE_REGISTER(grinreflex_app); + +#include + +#include "config.hpp" +#include "gf/lvgl_utils.hpp" +#include "gf/utils.hpp" +#include "gf/video.hpp" +#include "grinreflex.h" + +static const struct device *video_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_camera)); +static const struct device *display_dev = + DEVICE_DT_GET(DT_CHOSEN(zephyr_display)); +const struct device *jpeg_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_jpeg)); + +static uint8_t + fullFrameBuffer[FULL_FRAME_WIDTH * FULL_FRAME_HEIGHT * + LV_COLOR_FORMAT_GET_SIZE(FULL_FRAME_COLOR_FORMAT)]; +static uint8_t roiFrameBuffer[ROI_FRAME_WIDTH * ROI_FRAME_HEIGHT * + LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8)]; +static uint8_t + thumbnailFrameBuffer[THUMBNAIL_FRAME_WIDTH * THUMBNAIL_FRAME_HEIGHT * + LV_COLOR_FORMAT_GET_SIZE(LV_COLOR_FORMAT_L8)]; + +static lv_obj_t *screen_canvas; +static lv_obj_t *dummy_canvas; + +int init() { + if (!device_is_ready(display_dev)) { + LOG_ERR("Display device not ready, aborting test"); + return -ENODEV; + } + + if (!device_is_ready(video_dev)) { + LOG_ERR("Video device not ready, aborting test"); + return -ENODEV; + } + + if (!device_is_ready(jpeg_dev)) { + printf("%s JPEG device not ready", jpeg_dev->name); + return -ENODEV; + } + + Video::setup(); + + dummy_canvas = lv_canvas_create(NULL); + screen_canvas = lv_canvas_create(lv_screen_active()); + lv_canvas_fill_bg(screen_canvas, lv_color_hex3(0xccc), LV_OPA_COVER); + lv_obj_center(screen_canvas); + + display_blanking_off(display_dev); + + return 0; +} + +int loop() { + + struct video_buffer vbuf {}; + struct video_buffer *vbuf_ptr = &vbuf; + struct jpeg_out_prop jpeg_prop; + + vbuf_ptr->type = VIDEO_BUF_TYPE_OUTPUT; + + int err = video_dequeue(video_dev, &vbuf_ptr, K_FOREVER); + if (err) { + LOG_ERR("Unable to dequeue video buf"); + return 0; + } + + // jpeg_hw_decode(jpeg_dev, (uint8_t *)vbuf_ptr->buffer, vbuf_ptr->bytesused, + // jpeg_frame_buffer); + + jpeg_hw_poll(jpeg_dev, 0, &jpeg_prop); + + // printf("JPEG done w = %d, h = %d, color = %d chroma = %d\n", + // jpeg_prop.width, + // jpeg_prop.height, jpeg_prop.color_space, jpeg_prop.chroma); + + jpeg_color_convert_helper(jpeg_dev, &jpeg_prop, vbuf_ptr->buffer, + fullFrameBuffer); + + cropFullFrameToRoi(dummy_canvas, fullFrameBuffer, roiFrameBuffer, + lvgl::Size(FULL_FRAME_WIDTH, FULL_FRAME_HEIGHT), + lvgl::Size(ROI_FRAME_WIDTH, ROI_FRAME_HEIGHT), + FULL_FRAME_COLOR_FORMAT, LV_COLOR_FORMAT_L8, + FULL_FRAME_FLIP_Y); + + fitRoiFrameToThumbnail( + screen_canvas, roiFrameBuffer, thumbnailFrameBuffer, + lvgl::Size(ROI_FRAME_WIDTH, ROI_FRAME_HEIGHT), + lvgl::Size(THUMBNAIL_FRAME_WIDTH, THUMBNAIL_FRAME_HEIGHT), + LV_COLOR_FORMAT_L8, LV_COLOR_FORMAT_L8); + lv_task_handler(); + + err = video_enqueue(video_dev, vbuf_ptr); + if (err) { + LOG_ERR("Unable to requeue video buf"); + } + + k_msleep(50); + + return 0; +} \ No newline at end of file From 8b684e3facd1a35e756d766f5515422e04d3e84b Mon Sep 17 00:00:00 2001 From: Roman Pustobaiev Date: Thu, 27 Nov 2025 20:02:40 +0100 Subject: [PATCH 16/18] Adding stm32n6 camera application enabling dcmipp some tweaks --- .../grinreflex_n6_dk3_common.dtsi | 131 +++++++++++------- .../grinreflex_n6_dk3_stm32n657xx_app.dts | 4 + ...rinreflex_n6_dk3_stm32n657xx_app_defconfig | 60 ++++---- u5/common/src/video.cpp | 28 +--- u5/drivers/jpeg/Kconfig | 2 - u5/drivers/jpeg/jpeg.c | 2 +- u5/gf_n6_dk3/CMakeLists.txt | 8 +- u5/gf_n6_dk3/prj.conf | 52 ++++++- u5/gf_n6_dk3/src/application.cpp | 25 ++-- u5/gf_n6_dk3/src/application.h | 17 +++ u5/gf_n6_dk3/src/config.hpp | 37 +++++ u5/gf_n6_dk3/src/main.c | 15 +- 12 files changed, 255 insertions(+), 126 deletions(-) create mode 100644 u5/gf_n6_dk3/src/application.h create mode 100644 u5/gf_n6_dk3/src/config.hpp diff --git a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_common.dtsi b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_common.dtsi index e67a181..2466058 100644 --- a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_common.dtsi +++ b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_common.dtsi @@ -73,13 +73,13 @@ gpio-map = <3 0 &gpioc 1 0>, /* DVP_SDA (I2C2_SDA)*/ <5 0 &gpioh 9 0>, /* DVP_SCL (I2C2_SCL)*/ <7 0 &gpioe 6 0>, /* DVP_VSYNC*/ - <8 0 &gpioc 4 0>, /* DVP_PWDN NC*/ + <8 0 &gpioc 4 0>, /* DVP_PWDN*/ <9 0 &gpioc 0 0>, /* DVP_HSYNC*/ <12 0 &gpiof 1 0>, /* DVP_D7*/ <13 0 &gpioa 8 0>, /* DVP_XCLK (RCC_MCO1)*/ <14 0 &gpiof 5 0>, /* DVP_D6*/ <16 0 &gpioe 4 0>, /* DVP_D5*/ - <17 0 &gpioa 6 0>, /* DVP_PCLK*/ + <17 0 &gpiod 5 0>, /* DVP_PCLK*/ <18 0 &gpioc 11 0>, /* DVP_D4*/ <19 0 &gpioa 1 0>, /* DVP_D0*/ <20 0 &gpioc 9 0>, /* DVP_D3 */ @@ -92,7 +92,7 @@ dmas = <&gpdma1 2 1 (STM32_DMA_PERIPH_RX | STM32_DMA_PERIPH_32BITS | STM32_DMA_MEM_32BITS) &gpdma1 3 0 (STM32_DMA_PERIPH_TX | STM32_DMA_PERIPH_32BITS | STM32_DMA_MEM_32BITS)>; dma-names = "tx", "rx"; - status = "okay"; + status = "disabled"; }; }; @@ -155,6 +155,12 @@ status = "okay"; }; +&ic17 { + pll-src = <1>; + ic-div = <4>; + status = "okay"; +}; + &perck { clocks = <&rcc STM32_SRC_HSI PER_SEL(0)>; status = "okay"; @@ -182,30 +188,11 @@ gpios = <6 GPIO_ACTIVE_LOW>; output-high; }; -}; - -&i2c1 { - clocks = <&rcc STM32_CLOCK(APB1, 21)>, - <&rcc STM32_SRC_CKPER I2C1_SEL(1)>; - pinctrl-0 = <&i2c1_scl_ph9 &i2c1_sda_pc1>; - pinctrl-names = "default"; - clock-frequency = ; - status = "okay"; - ov2640: ov2640@30 { - compatible = "ovti,ov2640"; - reg = <0x30>; - - supply-gpios = <&dcmi_camera_connector 8 GPIO_ACTIVE_HIGH>; - clock-rate-control = <0x80>; - - status = "okay"; - - port { - ov2640_ep_out: endpoint { - remote-endpoint-label = "zephyr_camera_dvp_in"; - }; - }; + dvp_pwdn { + gpio-hog; + gpios = <4 GPIO_ACTIVE_LOW>; + output-high; }; }; @@ -215,7 +202,7 @@ pinctrl-0 = <&spi4_sck_pe2 &spi4_mosi_pb7>; cs-gpios = <&gpioe 7 (GPIO_ACTIVE_LOW | GPIO_PULL_UP)>; pinctrl-names = "default"; - clock-frequency = ; + clock-frequency = ; status = "okay"; }; @@ -243,7 +230,7 @@ compatible = "st,stm32-xspi-nor"; reg = <0>; size = ; /* 128 Mbits */ - ospi-max-frequency = ; + ospi-max-frequency = ; spi-bus-width = ; data-rate = ; four-byte-opcodes; @@ -284,30 +271,80 @@ status = "okay"; }; -zephyr_camera_dvp: &dcmi { - pinctrl-0 = <&dcmi_hsync_pc0 - &dcmi_pixclk_pa6 - &dcmi_vsync_pe6 - &dcmi_d0_pa1 - &dcmi_d1_pa10 - &dcmi_d2_pc3 - &dcmi_d3_pc9 - &dcmi_d4_pc11 - &dcmi_d5_pe4 - &dcmi_d6_pf5 - &dcmi_d7_pf1>; + +&i2c1 { + clocks = <&rcc STM32_CLOCK(APB1, 21)>, + <&rcc STM32_SRC_CKPER I2C1_SEL(1)>; + pinctrl-0 = <&i2c1_scl_ph9 &i2c1_sda_pc1>; pinctrl-names = "default"; + clock-frequency = ; + status = "okay"; + + ov2640: ov2640@30 { + compatible = "ovti,ov2640"; + reg = <0x30>; - port { - zephyr_camera_dvp_in: endpoint { - remote-endpoint-label = "ov2640_ep_out"; - bus-width = <8>; - hsync-active = <0>; - vsync-active = <0>; - pclk-sample = <1>; + supply-gpios = <&dcmi_camera_connector 8 GPIO_ACTIVE_HIGH>; + clock-rate-control = <0x80>; + + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + ov2640_ep_out: endpoint { + /* parallel bus */ + bus-type = ; /* MEDIA_BUS_TYPE_PARALLEL */ + bus-width = <8>; /* D0..D7 */ + hsync-active = <0>; + vsync-active = <0>; + pclk-sample = <1>; + + /* link to DCMIPP endpoint via label */ + remote-endpoint-label = "zephyr_camera_dvp_in"; + }; + }; }; }; +}; + +&dcmipp { + compatible = "st,stm32-dcmipp"; status = "okay"; + + pinctrl-0 = <&dcmipp_hsync_pc0 + &dcmipp_pixclk_pd5 + &dcmipp_vsync_pe6 + &dcmipp_d0_pa1 + &dcmipp_d1_pa10 + &dcmipp_d2_pc3 + &dcmipp_d3_pc9 + &dcmipp_d4_pc11 + &dcmipp_d5_pe4 + &dcmipp_d6_pf5 + &dcmipp_d7_pf1>; + pinctrl-names = "default"; + + ports { + port@0 { + reg = <0>; + zephyr_camera_dvp_in: endpoint { + remote-endpoint-label = "ov2640_ep_out"; + bus-type = ; + bus-width = <8>; + data-shift = <0>; + hsync-active = <0>; + vsync-active = <0>; + pclk-sample = <1>; + }; + }; + port@1 { + zephyr_camera_dvp: endpoint@0 { }; + }; + }; }; diff --git a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_app.dts b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_app.dts index d281d85..09ec311 100644 --- a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_app.dts +++ b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_app.dts @@ -15,3 +15,7 @@ zephyr,sram = &axisram1; }; }; + +&dcmi { + jpeg-dev = <&jpeg0>; +}; diff --git a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_app_defconfig b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_app_defconfig index 8607387..6c398eb 100644 --- a/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_app_defconfig +++ b/u5/boards/vendor/grinreflex_n6_dk3/grinreflex_n6_dk3_stm32n657xx_app_defconfig @@ -15,58 +15,48 @@ CONFIG_EARLY_CONSOLE=y # Enable HW stack protection CONFIG_HW_STACK_PROTECTION=y -CONFIG_ARM_MPU=n -#CONFIG_XIP=y +CONFIG_ARM_MPU=y +CONFIG_XIP=y CONFIG_LOG=y CONFIG_LOG_DEFAULT_LEVEL=4 -CONFIG_VIDEO=y CONFIG_VIDEO_LOG_LEVEL_INF=y CONFIG_SHELL=y CONFIG_VIDEO=y -CONFIG_VIDEO_LOG_LEVEL_DBG=y -CONFIG_VIDEO_LOG_LEVEL_INF=y - -#CONFIG_STM32_JPEG_RGB_FORMAT=3 -#CONFIG_GRINREFLEX_JPEG_VIDEO=y -#CONFIG_GRINREFLEX_JPEG_VIDEO_QUALITY=10 -#CONFIG_GRINREFLEX_VIDEO_WIDTH=640 -#CONFIG_GRINREFLEX_VIDEO_HEIGHT=480 +CONFIG_VIDEO_STM32_DCMIPP=y #CONFIG_VIDEO_BUFFER_POOL_SZ_MAX=76800 CONFIG_VIDEO_BUFFER_POOL_SZ_MAX=153600 +CONFIG_SPI_STM32=y +CONFIG_SPI_STM32_USE_HW_SS=n +CONFIG_DISPLAY_INIT_PRIORITY=100 + CONFIG_DISPLAY=y -CONFIG_DISPLAY_LOG_LEVEL_ERR=y -CONFIG_DISPLAY_LOG_LEVEL_DBG=y - -CONFIG_SPI_LOG_LEVEL_DBG=y -CONFIG_CLOCK_CONTROL_LOG_LEVEL_DBG=y -CONFIG_PINCTRL_LOG_LEVEL_DBG=y - -#CONFIG_LV_Z_MEM_POOL_SIZE=16384 -#CONFIG_LV_Z_SHELL=n -#CONFIG_LVGL=y -#CONFIG_LV_USE_LOG=n -#CONFIG_LV_USE_LABEL=n -#CONFIG_LV_USE_ARC=y -#CONFIG_LV_USE_IMAGE=y -#CONFIG_LV_FONT_MONTSERRAT_14=y - -#CONFIG_LV_CACHE_DEF_SIZE=4096 -#CONFIG_LV_IMAGE_HEADER_CACHE_DEF_CNT=1 -#CONFIG_LV_USE_BUILTIN_MALLOC=n + +CONFIG_LV_Z_MEM_POOL_SIZE=16384 +CONFIG_LV_Z_SHELL=n +CONFIG_LVGL=y +CONFIG_LV_USE_LOG=n +CONFIG_LV_USE_LABEL=n +CONFIG_LV_USE_ARC=y +CONFIG_LV_USE_IMAGE=y +CONFIG_LV_FONT_MONTSERRAT_14=y + +CONFIG_LV_CACHE_DEF_SIZE=4096 +CONFIG_LV_IMAGE_HEADER_CACHE_DEF_CNT=1 +CONFIG_LV_USE_BUILTIN_MALLOC=n + +CONFIG_LV_COLOR_16_SWAP=y #CONFIG_LV_USE_PERF_MONITOR=y #CONFIG_LV_USE_DROPDOWN=y #CONFIG_LV_USE_SYSMON=y -#CONFIG_SPI=y -#CONFIG_SPI_SHELL=y +#CONFIG_STM32_JPEG=y +#CONFIG_STM32_JPEG_INIT_PRIORITY=180 -CONFIG_SPI_STM32=y -CONFIG_SPI_STM32_USE_HW_SS=n -CONFIG_DISPLAY_INIT_PRIORITY=100 \ No newline at end of file +CONFIG_VIDEO_INIT_PRIORITY=200 \ No newline at end of file diff --git a/u5/common/src/video.cpp b/u5/common/src/video.cpp index 6f87ec4..6847200 100644 --- a/u5/common/src/video.cpp +++ b/u5/common/src/video.cpp @@ -1,5 +1,6 @@ +#include #include #include "gf/video.hpp" @@ -14,10 +15,14 @@ static const struct device *video_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_camera)); static struct video_buffer *buffers[2]; #if defined(CONFIG_GRINREFLEX_JPEG_VIDEO) +#define SECOND_BUFFER_SIZE (CONFIG_GRINREFLEX_VIDEO_WIDTH * CONFIG_GRINREFLEX_VIDEO_HEIGHT * 2) +#else +#define SECOND_BUFFER_SIZE (CONFIG_GRINREFLEX_VIDEO_WIDTH * CONFIG_GRINREFLEX_VIDEO_HEIGHT * 2) +#endif + static struct video_buffer second_buffer {}; __attribute__((aligned(32))) -static uint8_t video_large_buffer[CONFIG_GRINREFLEX_VIDEO_WIDTH * CONFIG_GRINREFLEX_VIDEO_HEIGHT * 2]; -#endif +static uint8_t video_large_buffer[SECOND_BUFFER_SIZE]; void setup() { struct video_format fmt; @@ -127,7 +132,6 @@ void setup() { /* Size to allocate for each buffer */ bsize = fmt.pitch * fmt.height; -#if defined(CONFIG_GRINREFLEX_JPEG_VIDEO) bsize = CONFIG_VIDEO_BUFFER_POOL_SZ_MAX; buffers[0] = video_buffer_aligned_alloc(bsize, CONFIG_VIDEO_BUFFER_POOL_ALIGN, @@ -145,25 +149,7 @@ void setup() { buffers[1]->buffer = video_large_buffer; buffers[1]->size = sizeof(video_large_buffer); - if (buffers[1]->buffer == NULL) { - LOG_ERR("Unable to alloc video buffer"); - return; - } - video_enqueue(video_dev, buffers[1]); -#else /* !defined(CONFIG_GRINREFLEX_JPEG_VIDEO) */ - /* Alloc video buffers and enqueue for capture */ - for (i = 0; i < ARRAY_SIZE(buffers); i++) { - buffers[i] = video_buffer_aligned_alloc( - bsize, CONFIG_VIDEO_BUFFER_POOL_ALIGN, K_FOREVER); - if (buffers[i] == NULL) { - LOG_ERR("Unable to alloc video buffer"); - return; - } - buffers[i]->type = type; - video_enqueue(video_dev, buffers[i]); - } -#endif /* defined(CONFIG_GRINREFLEX_JPEG_VIDEO) */ /* Set controls */ struct video_control ctrl = {.id = VIDEO_CID_HFLIP, .val = 1}; diff --git a/u5/drivers/jpeg/Kconfig b/u5/drivers/jpeg/Kconfig index b1d7339..095ec17 100644 --- a/u5/drivers/jpeg/Kconfig +++ b/u5/drivers/jpeg/Kconfig @@ -6,8 +6,6 @@ config STM32_JPEG bool "STM32 JPEG HW decoder device drivers" default n - # depends on DMA_STM32 - select USE_STM32_HAL_DMA select USE_STM32_HAL_JPEG select USE_STM32_HAL_CORTEX diff --git a/u5/drivers/jpeg/jpeg.c b/u5/drivers/jpeg/jpeg.c index 8ff55b9..a5d6bd0 100644 --- a/u5/drivers/jpeg/jpeg.c +++ b/u5/drivers/jpeg/jpeg.c @@ -231,7 +231,7 @@ static int jpeg_hw_init(const struct device *dev) { __HAL_RCC_JPEG_CLK_ENABLE(); if (HAL_JPEG_Init(&dev_data->hjpeg) != HAL_OK) { - abort(); + return -EINVAL; } int ret = 0; diff --git a/u5/gf_n6_dk3/CMakeLists.txt b/u5/gf_n6_dk3/CMakeLists.txt index 3e9efa9..9be22bf 100644 --- a/u5/gf_n6_dk3/CMakeLists.txt +++ b/u5/gf_n6_dk3/CMakeLists.txt @@ -5,4 +5,10 @@ cmake_minimum_required(VERSION 3.20.0) find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) project(gf_n6_dk3) -target_sources(app PRIVATE src/main.c) +target_include_directories(app PRIVATE +src +) + +target_sources(app PRIVATE +src/application.cpp +src/main.c) diff --git a/u5/gf_n6_dk3/prj.conf b/u5/gf_n6_dk3/prj.conf index b4bda07..3bbc9dd 100644 --- a/u5/gf_n6_dk3/prj.conf +++ b/u5/gf_n6_dk3/prj.conf @@ -1,9 +1,57 @@ # nothing here -CONFIG_LOG_DEFAULT_LEVEL=4 +CONFIG_LOG_DEFAULT_LEVEL=3 #CONFIG_FAULT_DUMP=2 CONFIG_ASSERT=y #CONFIG_RESET_ON_FATAL_ERROR=n #CONFIG_LOG=y -CONFIG_ARM_MPU=n \ No newline at end of file +CONFIG_HW_STACK_PROTECTION=y # if supported +CONFIG_STACK_SENTINEL=y +CONFIG_INIT_STACKS=y + +CONFIG_MAIN_STACK_SIZE=65536 + +CONFIG_CPP=y +CONFIG_STD_CPP20=y +CONFIG_CPP_EXCEPTIONS=y +CONFIG_CPP_RTTI=y + +CONFIG_REQUIRES_FULL_LIBC=y +CONFIG_GLIBCXX_LIBCPP=y + +CONFIG_NEWLIB_LIBC=y +CONFIG_PICOLIBC=n + +CONFIG_COMMON_LIBC_MALLOC=y +CONFIG_HEAP_MEM_POOL_SIZE=8192 + +CONFIG_POSIX_API=y + +CONFIG_ASSERT_VERBOSE=y + +CONFIG_FPU=y +CONFIG_FP_SOFTABI=y + +CONFIG_FAULT_DUMP=2 + +CONFIG_DMA_LOG_LEVEL_DBG=y + +# Video / DCMI logs +CONFIG_VIDEO_LOG_LEVEL_DBG=y + + +CONFIG_VIDEO_LOG_LEVEL_DBG=y +CONFIG_VIDEO_LOG_LEVEL_INF=y + +CONFIG_STM32_JPEG_RGB_FORMAT=3 +CONFIG_GRINREFLEX_JPEG_VIDEO=n +CONFIG_GRINREFLEX_JPEG_VIDEO_QUALITY=10 +CONFIG_GRINREFLEX_VIDEO_WIDTH=320 +CONFIG_GRINREFLEX_VIDEO_HEIGHT=240 + +CONFIG_VIDEO_STM32_DCMIPP_SENSOR_WIDTH=320 +CONFIG_VIDEO_STM32_DCMIPP_SENSOR_HEIGHT=240 +CONFIG_VIDEO_STM32_DCMIPP_SENSOR_PIXEL_FORMAT="RGBP" + +CONFIG_DCACHE=y \ No newline at end of file diff --git a/u5/gf_n6_dk3/src/application.cpp b/u5/gf_n6_dk3/src/application.cpp index f9d8012..254302b 100644 --- a/u5/gf_n6_dk3/src/application.cpp +++ b/u5/gf_n6_dk3/src/application.cpp @@ -23,12 +23,15 @@ LOG_MODULE_REGISTER(grinreflex_app); #include "gf/lvgl_utils.hpp" #include "gf/utils.hpp" #include "gf/video.hpp" -#include "grinreflex.h" +#include "application.h" static const struct device *video_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_camera)); static const struct device *display_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_display)); + +#if defined(CONFIG_STM32_JPEG) const struct device *jpeg_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_jpeg)); +#endif static uint8_t fullFrameBuffer[FULL_FRAME_WIDTH * FULL_FRAME_HEIGHT * @@ -52,12 +55,12 @@ int init() { LOG_ERR("Video device not ready, aborting test"); return -ENODEV; } - +#if defined(CONFIG_STM32_JPEG) if (!device_is_ready(jpeg_dev)) { printf("%s JPEG device not ready", jpeg_dev->name); return -ENODEV; } - +#endif Video::setup(); dummy_canvas = lv_canvas_create(NULL); @@ -66,6 +69,7 @@ int init() { lv_obj_center(screen_canvas); display_blanking_off(display_dev); + lv_task_handler(); return 0; } @@ -84,6 +88,7 @@ int loop() { return 0; } +#if defined(CONFIG_GRINREFLEX_JPEG_VIDEO) // jpeg_hw_decode(jpeg_dev, (uint8_t *)vbuf_ptr->buffer, vbuf_ptr->bytesused, // jpeg_frame_buffer); @@ -96,6 +101,15 @@ int loop() { jpeg_color_convert_helper(jpeg_dev, &jpeg_prop, vbuf_ptr->buffer, fullFrameBuffer); +#else + memcpy(fullFrameBuffer, vbuf_ptr->buffer, vbuf_ptr->bytesused); +#endif + + err = video_enqueue(video_dev, vbuf_ptr); + if (err) { + LOG_ERR("Unable to requeue video buf"); + } + cropFullFrameToRoi(dummy_canvas, fullFrameBuffer, roiFrameBuffer, lvgl::Size(FULL_FRAME_WIDTH, FULL_FRAME_HEIGHT), lvgl::Size(ROI_FRAME_WIDTH, ROI_FRAME_HEIGHT), @@ -109,11 +123,6 @@ int loop() { LV_COLOR_FORMAT_L8, LV_COLOR_FORMAT_L8); lv_task_handler(); - err = video_enqueue(video_dev, vbuf_ptr); - if (err) { - LOG_ERR("Unable to requeue video buf"); - } - k_msleep(50); return 0; diff --git a/u5/gf_n6_dk3/src/application.h b/u5/gf_n6_dk3/src/application.h new file mode 100644 index 0000000..43ceff3 --- /dev/null +++ b/u5/gf_n6_dk3/src/application.h @@ -0,0 +1,17 @@ + +#ifndef GRINREFLEX_APPLICATION_H +#define GRINREFLEX_APPLICATION_H + +#ifdef __cplusplus +extern "C" { +#endif + +int init(); + +int loop(); + +#ifdef __cplusplus +} +#endif + +#endif /* GRINREFLEX_APPLICATION_H */ \ No newline at end of file diff --git a/u5/gf_n6_dk3/src/config.hpp b/u5/gf_n6_dk3/src/config.hpp new file mode 100644 index 0000000..960f5e7 --- /dev/null +++ b/u5/gf_n6_dk3/src/config.hpp @@ -0,0 +1,37 @@ + +#pragma once + +#if defined(CONFIG_GRINREFLEX_JPEG_VIDEO) + +#if defined(CONFIG_STM32_JPEG_RGB_FORMAT) +#if CONFIG_STM32_JPEG_RGB_FORMAT == 0 +#define FULL_FRAME_COLOR_FORMAT LV_COLOR_FORMAT_ARGB8888 +#elif CONFIG_STM32_JPEG_RGB_FORMAT == 1 +#define FULL_FRAME_COLOR_FORMAT LV_COLOR_FORMAT_RGB888 +#elif CONFIG_STM32_JPEG_RGB_FORMAT == 2 +#define FULL_FRAME_COLOR_FORMAT LV_COLOR_FORMAT_RGB565 +#elif CONFIG_STM32_JPEG_RGB_FORMAT == 3 +#define FULL_FRAME_COLOR_FORMAT LV_COLOR_FORMAT_L8 +#else +#error "CONFIG_STM32_JPEG_RGB_FORMAT is not supported" +#endif + +#else +#define FULL_FRAME_COLOR_FORMAT LV_COLOR_FORMAT_RGB888 +#endif /* !defined(CONFIG_STM32_JPEG_RGB_FORMAT) */ + +#else +#define FULL_FRAME_COLOR_FORMAT LV_COLOR_FORMAT_RGB565 +#endif /* !defined(CONFIG_GRINREFLEX_JPEG_VIDEO) */ + +#define FULL_FRAME_WIDTH CONFIG_GRINREFLEX_VIDEO_WIDTH +#define FULL_FRAME_HEIGHT CONFIG_GRINREFLEX_VIDEO_HEIGHT + +#define ROI_FRAME_WIDTH 320 +#define ROI_FRAME_HEIGHT 240 + +#define THUMBNAIL_FRAME_WIDTH 90 +#define THUMBNAIL_FRAME_HEIGHT 90 + +#define INPUT_PREPROCESS_OPENCV (0) +#define FULL_FRAME_FLIP_Y (0) \ No newline at end of file diff --git a/u5/gf_n6_dk3/src/main.c b/u5/gf_n6_dk3/src/main.c index d150958..7897b4b 100644 --- a/u5/gf_n6_dk3/src/main.c +++ b/u5/gf_n6_dk3/src/main.c @@ -7,18 +7,15 @@ #include #include -// #define LOG_LEVEL CONFIG_LOG_DEFAULT_LEVEL -// #include -// LOG_MODULE_REGISTER(main); - -// static const struct device *video_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_camera)); +#include "application.h" int main(void) { printf("Hello World! %s\n", CONFIG_BOARD_TARGET); - // if (!device_is_ready(video_dev)) { - // LOG_ERR("%s device is not ready", video_dev->name); - // } - // LOG_DBG("%s", video_dev->name); + init(); + + while (true) { + loop(); + } return 0; } From 07362ca372c1c29a176205dfdc9a8e7088141bcf Mon Sep 17 00:00:00 2001 From: Roman Pustobaiev Date: Sat, 29 Nov 2025 16:09:32 +0100 Subject: [PATCH 17/18] Updating readme --- README.md | 70 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 48 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 8f35552..d9c2e12 100644 --- a/README.md +++ b/README.md @@ -1,32 +1,58 @@ -## This is a draft playground repository to check opencv capabilities with zephyr rtos in embedded world -Currently it is only tested with stm32u5xx boards, as it takes ~3mb of rom/flash +## This repository contains a bunch of projects, primarily targeting stm32 based platforms +### The idea behind it - is to creatre unified space where you can build and start your own projects targeting Computer Vision and AI on the edge + +Current version supports 3 customized boards, each board has well known predecessor \ +Boards schematics and pcb's can be found [Here](https://github.com/cornway/ComputerVisionHw)\ +Having this boards schematics you can easily figure out and adopt examples found here to your own hardware For a fresh start * Clone this repo * `git submodule init --update` * `west init -l u5` * `west update` -# Outdated -* `west build -b stm32u5g9j_dk2 u5/app/` (builds for stm32u5g9j_dk2 board, you can google what that board is) -* `west flash` - -### Important note: since stm32u5g9j_dk2 doesn't have camera sensor, you need to find out your own way how to transfer images to the board; I used uart for that purpose ### Apply stm32hal.patch, in case -O2 (CONFIG_SPEED_OPTIMIZATIONS=y) is used -* `modules/hal/stm32/` +* `cd modules/hal/stm32/` * `git apply ../../../stm32hal.patch` -## To enable OpenCV library and example code -* in `u5/app/prj.conf` : CONFIG_OPENCV_LIB=y - -Note: there are maybe more dependencies, so watch out - -## Also possible to use tflite micro -* in `u5/app/prj.conf` : CONFIG_TFLITE_LIB=y - -### Planning to add support for stm32n6 boards -### there are more custom boards based on stm32u5xx family, PCB repository is coming -If you are willing to roll out your own board, please don't hesitate to contact me - - -### Note: this project is in it's very very early phase +## Applicatons: +* **u5/uvc_gf_dk1/** - Simple one that utilizes UVC USB class to given board into a USB camera; only works for **grinreflex_dk1** board +* **u5/gf_dkx/** - unified application, can be used for all boards (so far), has two targets - face detection, and face + smile detection using opencv +* **uvc_gf_dk1/** - example video streaming application, available only for **grinreflex_n6_dk1** board + +## Boards supported: +* **grinreflex_dk1** - uses schematic from [HW](https://github.com/cornway/ComputerVisionHw) repo **stm32u5_v1.0** + * Extrnal flash + * USB + * OV2640 compatible camera + * LCD serial display + * A few GPIOS + * ... +* **grinreflex_dk2** - counterpart of **stm32u5_v2.0** in [HW](https://github.com/cornway/ComputerVisionHw) repo + * Simplified version of v1.0, unfortunatelly I put there MCU that seems to went out of stock + * All hardware fetures from V1.0, except - there is no external flash and USB +* **grinreflex_n6_dk1** - most complicated so far, uses **stm32n6_v1.0** schematic in [HW] (https://github.com/cornway/ComputerVisionHw) + * Hw Features + * LCD serial display + * OV2640 compatible camera + * Eternal flash + * External vddcore regulator + * Boot switch + * GPIO extension connector + * Build variants (Just provide this option after **-board** to west command) + * **grinreflex_n6_dk3//app** - XIP application, requires **--sysbuild** option, loaded by fsbl + * **grinreflex_n6_dk3//fsbl_app** - application that is a part of fsbl + * **grinreflex_n6_dk3//fsbl** - fsbl, usually required when you specify **--sysbuild**, that builds mcuboot + * Example: build XIP application \ + `west build -b grinreflex_n6_dk3//app --sysbuild u5/gf_n6_dk3/` \ +That board pretty much derived from [Nucleo](https://docs.zephyrproject.org/latest/boards/st/nucleo_n657x0_q/doc/index.html), so you can use their tutorial to obtain information on how to build/run/debug + +## Configuration options +* **CONFIG_OPENCV_LIB** - if **y**, adds OpenCv library, please see `u5/lib/cv/CMakeLists.txt` to see which modules are actually built +* **CONFIG_TFLITE_LIB** - if **y**, adds TfLite framework, the performance remains poor as no optimizations are done yet +* **CONFIG_QUIRC_LIB** - say **y** to Add [this](https://github.com/dlbeer/quirc) +* **CONFIG_NEMA** - enables **NEMA** or stm32's **GPU2.5**, no modifications required to **lvgl** (Yes, that works only as a part of lvgl so far), that sometimes significantly speeds up graphic manipulations +* **CONFIG_STM32_JPEG** - pretty much **rough** stm32's jpeg driver, tested only on **u5** family so far + + +### Note: while playing yo may face weird issues, where configuration is not suppoorted or missing, please feel free to submit an issue; Maintainers are very welcome \ No newline at end of file From a7830b349b97d2be00f2782405828ddc90c83e48 Mon Sep 17 00:00:00 2001 From: Roman <31817791+cornway@users.noreply.github.com> Date: Sat, 29 Nov 2025 16:20:00 +0100 Subject: [PATCH 18/18] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d9c2e12..5a95833 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ For a fresh start * **grinreflex_dk2** - counterpart of **stm32u5_v2.0** in [HW](https://github.com/cornway/ComputerVisionHw) repo * Simplified version of v1.0, unfortunatelly I put there MCU that seems to went out of stock * All hardware fetures from V1.0, except - there is no external flash and USB -* **grinreflex_n6_dk1** - most complicated so far, uses **stm32n6_v1.0** schematic in [HW] (https://github.com/cornway/ComputerVisionHw) +* **grinreflex_n6_dk1** - most complicated so far, uses **stm32n6_v1.0** schematic in [HW](https://github.com/cornway/ComputerVisionHw) * Hw Features * LCD serial display * OV2640 compatible camera @@ -55,4 +55,4 @@ That board pretty much derived from [Nucleo](https://docs.zephyrproject.org/late * **CONFIG_STM32_JPEG** - pretty much **rough** stm32's jpeg driver, tested only on **u5** family so far -### Note: while playing yo may face weird issues, where configuration is not suppoorted or missing, please feel free to submit an issue; Maintainers are very welcome \ No newline at end of file +### Note: while playing yo may face weird issues, where configuration is not suppoorted or missing, please feel free to submit an issue; Maintainers are very welcome