add nmea2000 dependency. Fix CSV parsing

lely-core CANopen dependency, SPN bit highlighting

CMakeLists.txt

@@ -15,9 +15,23 @@ option(BUILD_SHARED_LIBS "Build using shared libraries" OFF)
 include(cmake/dependencies.cmake)
 
 project(canscope)
-find_package(Boost REQUIRED COMPONENTS regex)
+include(cmake/lely.cmake)
-file(GLOB_RECURSE SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/*.hpp ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp)
 
+if(NOT BUILD_SHARED_LIBS)
+  set(Boost_USE_STATIC_LIBS ON)
+endif()
+
+find_package(Boost REQUIRED COMPONENTS regex)
+
+# Strip ICU dependencies from Boost::regex for static builds — ICU static libs
+# are not available on Manjaro and we don't use ICU-dependent regex features.
+if(NOT BUILD_SHARED_LIBS AND TARGET Boost::regex)
+  set_target_properties(Boost::regex PROPERTIES
+    INTERFACE_LINK_LIBRARIES ""
+  )
+endif()
+
+file(GLOB_RECURSE SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/*.hpp ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp)
 add_executable(${CMAKE_PROJECT_NAME} ${SOURCES})
 target_include_directories(${CMAKE_PROJECT_NAME} PRIVATE
   ${CMAKE_CURRENT_SOURCE_DIR}
@@ -31,5 +45,9 @@ target_include_directories(${CMAKE_PROJECT_NAME} PRIVATE
   ${clipp_SOURCE_DIR}/include
 )
 
-target_link_libraries(${CMAKE_PROJECT_NAME} PRIVATE ftxui::component ftxui::screen ftxui::dom tiny-process-library xlnt sqlite3_lib z Boost::regex ${Boost_LIBRARIES})
+target_compile_definitions(${CMAKE_PROJECT_NAME} PRIVATE
+  CANBOAT_JSON_PATH="${canboat_SOURCE_DIR}/docs/canboat.json"
+)
+
+target_link_libraries(${CMAKE_PROJECT_NAME} PRIVATE ftxui::component ftxui::screen ftxui::dom tiny-process-library xlnt sqlite3_lib z Boost::regex ${Boost_LIBRARIES} lely::coapp lely::co lely::can lely::io2 lely::ev lely::util)
 

README.md

@@ -1,6 +1,6 @@
 # {canscope}
 
-CAN bus sniffer and SAE J1939 protocol analyzer. Reads CAN frames in `candump` format, decodes them using a J1939 Digital Annex (xlsx), and presents results in an interactive terminal UI or as JSON output.
+CAN bus sniffer and SAE J1939 protocol analyzer. Reads CAN frames in `candump` format, decodes them using a J1939 Digital Annex (xlsx or csv), and presents results in an interactive terminal UI or as JSON output.
 
 ![demo](canscope-demo.gif)
 
@@ -9,7 +9,7 @@ CAN bus sniffer and SAE J1939 protocol analyzer. Reads CAN frames in `candump` f
 - **TUI mode** - full-screen interactive terminal interface (FTXUI). Multiple display modes per CAN ID: deployed, brief, verbose, manual, little-endian
 - **Headless mode** - JSON output to stdout or file, for scripting and automation
 - **Recording** - decoded J1939 SPN values saved to SQLite database with gzip compression and batch flushing
-- **J1939 decoding** - PGN/SPN lookup, bit-level value extraction from payload
+- **J1939 decoding** - PGN/SPN lookup, bit-level value extraction from payload. Supports xlsx and csv input formats
 - **CAN playback** - replay recorded CAN frames
 - **Custom SPN configuration** - per-parameter settings, parameter export
 - **Real-time** - 30 fps UI refresh
@@ -20,7 +20,7 @@ CAN bus sniffer and SAE J1939 protocol analyzer. Reads CAN frames in `candump` f
 - clang++ with C++23 support
 - CMake >= 3.13
 - Ninja
-- System libraries: boost (signals2, spirit, phoenix), sqlite3, zlib
+- System libraries: boost (signals2, spirit, phoenix, regex), sqlite3, zlib, icu
 
 Dependencies fetched automatically via CMake FetchContent:
 
@@ -28,6 +28,7 @@ Dependencies fetched automatically via CMake FetchContent:
 - [tiny-process-library](https://gitlab.com/eidheim/tiny-process-library) - subprocess management
 - [sqlite_modern_cpp](https://github.com/SqliteModernCpp/sqlite_modern_cpp) - modern C++ SQLite wrapper
 - [xlnt](https://github.com/xlnt-community/xlnt) - xlsx reading
+- [lely-core](https://gitlab.com/lely_industries/lely-core) - CANopen protocol stack
 - [fmt](https://github.com/fmtlib/fmt) - text formatting
 - [nlohmann/json](https://github.com/nlohmann/json) - JSON library
 - [clipp](https://github.com/muellan/clipp) - CLI argument parsing
@@ -53,7 +54,9 @@ make clean             # Remove all build artifacts
 
 ```bash
 make build
-./build/native/canscope -e "candump can0" -j1939 thirdparty/j1939da_2018.xlsx
+./build/native/canscope -e "candump can0" -j1939-xlsx thirdparty/j1939da_2018.xlsx
+# or with CSV (faster parsing)
+./build/native/canscope -e "candump can0" -j1939-csv thirdparty/j1939da_2018.csv
 ```
 
 ### Docker (cross-platform)
@@ -62,13 +65,13 @@ Works on Linux, macOS (?), and Windows (?). Requires only Docker and Make.
 
 ```bash
 # TUI mode - local CAN interface
-make docker-run ARGS='-e "candump can0" -j1939 thirdparty/j1939da_2018.xlsx'
+make docker-run ARGS='-e "candump can0" -j1939-xlsx thirdparty/j1939da_2018.xlsx'
 
 # TUI mode - remote CAN interface via SSH (no data if will ask password - use public key access or sshpass utility)
-make docker-run ARGS='-e "ssh user@remote candump can0" -j1939 thirdparty/j1939da_2018.xlsx'
+make docker-run ARGS='-e "ssh user@remote candump can0" -j1939-xlsx thirdparty/j1939da_2018.xlsx'
 
 # Headless mode - create report about collected PGNs and SPNs
-make docker-run ARGS='-hl -e "candump can0" -j1939 thirdparty/j1939da_2018.xlsx -of output.json'
+make docker-run ARGS='-hl -e "candump can0" -j1939-xlsx thirdparty/j1939da_2018.xlsx -of output.json'
 ```
 
 ### Cross-compile for arm64
@@ -84,22 +87,22 @@ Requires Docker. SSH keys from `~/.ssh` and `/etc/hosts` are forwarded into the
 
 ```bash
 # TUI mode (default)
-canscope -e "candump can0" -j1939 thirdparty/j1939da_2018.xlsx
+canscope -e "candump can0" -j1939-xlsx thirdparty/j1939da_2018.xlsx
 
 # Headless - JSON to stdout
-canscope -hl -e "candump can0" -j1939 thirdparty/j1939da_2018.xlsx
+canscope -hl -e "candump can0" -j1939-xlsx thirdparty/j1939da_2018.xlsx
 
 # Headless - JSON to file
-canscope -hl -e "candump can0" -j1939 thirdparty/j1939da_2018.xlsx -of output.json
+canscope -hl -e "candump can0" -j1939-xlsx thirdparty/j1939da_2018.xlsx -of output.json
 
 # Read from stdin (pipe)
-candump can0 | canscope -j1939 thirdparty/j1939da_2018.xlsx
+candump can0 | canscope -j1939-xlsx thirdparty/j1939da_2018.xlsx
 
 # Record to SQLite database
-canscope -rec -db recording.db -e "candump can0" -j1939 thirdparty/j1939da_2018.xlsx
+canscope -rec -db recording.db -e "candump can0" -j1939-xlsx thirdparty/j1939da_2018.xlsx
 
 # Record + TUI
-canscope -rec -db recording.db -tui -e "candump can0" -j1939 thirdparty/j1939da_2018.xlsx
+canscope -rec -db recording.db -tui -e "candump can0" -j1939-xlsx thirdparty/j1939da_2018.xlsx
 ```
 
 > **Note:** J1939 decoding has only been tested with the Digital Annex 2018 edition. Other editions may work but are not guaranteed.
@@ -108,7 +111,8 @@ canscope -rec -db recording.db -tui -e "candump can0" -j1939 thirdparty/j1939da_
 
 | Flag | Long form | Description |
 |------|-----------|-------------|
-| `-j1939` | `--j1939-document` | **(required)** J1939 Digital Annex xlsx file |
+| `-j1939-xlsx` | | J1939 Digital Annex xlsx file |
+| `-j1939-csv` | | J1939 Digital Annex csv file (faster parsing) |
 | `-e` | `--execute-command` | Command to read CAN frames from (e.g. `"candump can0"`) |
 | `-hl` | `--headless` | Headless mode (no TUI) |
 | `-of` | `--output-file` | Output file path (headless mode) |
@@ -119,5 +123,6 @@ canscope -rec -db recording.db -tui -e "candump can0" -j1939 thirdparty/j1939da_
 
 ## Roadmap
 
-- **CANopen protocol support** - CANopen decoding alongside J1939
+- **NMEA 2000 protocol support** - NMEA 2000 decoding using [canboat](https://github.com/canboat/canboat) PGN database (JSON). Same 29-bit CAN ID as J1939, requires Fast Packet protocol implementation
+- **CANopen protocol support** - CANopen decoding alongside J1939 (11-bit CAN ID, SDO/PDO/NMT)
 - **Other small features and enhancements** - UI improvements, performance optimizations, additional export formats

cmake/dependencies.cmake

@@ -112,3 +112,29 @@ FetchContent_GetProperties(clipp)
 if(NOT clipp_POPULATED)
   FetchContent_Populate(clipp)
 endif()
+
+FetchContent_Declare(
+  lely_core
+  GIT_REPOSITORY https://gitlab.com/lely_industries/lely-core.git
+  GIT_TAG        v2.3.5
+  GIT_SHALLOW    TRUE
+  GIT_PROGRESS   TRUE
+)
+
+FetchContent_Declare(
+  canboat
+  GIT_REPOSITORY https://github.com/canboat/canboat.git
+  GIT_TAG        v6.1.6
+  GIT_SHALLOW    TRUE
+  GIT_PROGRESS   TRUE
+)
+
+FetchContent_GetProperties(canboat)
+if(NOT canboat_POPULATED)
+  FetchContent_Populate(canboat)
+endif()
+
+FetchContent_GetProperties(lely_core)
+if(NOT lely_core_POPULATED)
+  FetchContent_Populate(lely_core)
+endif()

cmake/lely.cmake

@@ -0,0 +1,69 @@
+set(LELY_INSTALL_DIR ${CMAKE_BINARY_DIR}/lely-install)
+file(MAKE_DIRECTORY ${LELY_INSTALL_DIR}/include)
+
+if(BUILD_SHARED_LIBS)
+  set(LELY_SHARED_FLAG "--enable-shared;--disable-static")
+  set(LELY_LIB_TYPE SHARED)
+  set(LELY_LIB_SUFFIX .so)
+else()
+  set(LELY_SHARED_FLAG "--disable-shared;--enable-static")
+  set(LELY_LIB_TYPE STATIC)
+  set(LELY_LIB_SUFFIX .a)
+endif()
+
+set(LELY_CFLAGS "-Wno-error -Wno-implicit-function-declaration -Wno-unterminated-string-initialization -DATOMIC_VAR_INIT=")
+set(LELY_CXXFLAGS "-Wno-error")
+
+if(CMAKE_CROSSCOMPILING)
+  set(LELY_HOST_FLAG "--host=${CMAKE_C_COMPILER_TARGET}")
+  set(LELY_CROSS_FLAGS "--target=${CMAKE_C_COMPILER_TARGET}")
+  if(CMAKE_SYSROOT)
+    string(APPEND LELY_CROSS_FLAGS " --sysroot=${CMAKE_SYSROOT}")
+  endif()
+  set(LELY_CC "${CMAKE_C_COMPILER} ${LELY_CROSS_FLAGS}")
+  set(LELY_CXX "${CMAKE_CXX_COMPILER} ${LELY_CROSS_FLAGS}")
+else()
+  set(LELY_HOST_FLAG "")
+  set(LELY_CC "")
+  set(LELY_CXX "")
+endif()
+
+include(ExternalProject)
+set(LELY_BYPRODUCTS)
+foreach(_lib can co coapp util ev io2)
+  list(APPEND LELY_BYPRODUCTS ${LELY_INSTALL_DIR}/lib/liblely-${_lib}${LELY_LIB_SUFFIX})
+endforeach()
+
+set(LELY_ENV_VARS "CFLAGS=${LELY_CFLAGS}" "CXXFLAGS=${LELY_CXXFLAGS}")
+if(LELY_CC)
+  list(APPEND LELY_ENV_VARS "CC=${LELY_CC}" "CXX=${LELY_CXX}" "LDFLAGS=-fuse-ld=lld")
+endif()
+
+ExternalProject_Add(lely_core_ext
+  SOURCE_DIR        ${lely_core_SOURCE_DIR}
+  BUILD_IN_SOURCE   TRUE
+  CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${LELY_ENV_VARS}
+                    autoreconf -i ${lely_core_SOURCE_DIR}
+            COMMAND ${CMAKE_COMMAND} -E env ${LELY_ENV_VARS}
+                    ${lely_core_SOURCE_DIR}/configure
+                    --prefix=${LELY_INSTALL_DIR}
+                    ${LELY_HOST_FLAG}
+                    ${LELY_SHARED_FLAG}
+                    --disable-dependency-tracking
+                    --disable-python
+                    --disable-tests
+                    --disable-doc
+  BUILD_COMMAND     ${CMAKE_COMMAND} -E env ${LELY_ENV_VARS}
+                    make -j
+  INSTALL_COMMAND   make install
+  BUILD_BYPRODUCTS  ${LELY_BYPRODUCTS}
+)
+
+foreach(_lib can co coapp util ev io2)
+  add_library(lely::${_lib} ${LELY_LIB_TYPE} IMPORTED GLOBAL)
+  set_target_properties(lely::${_lib} PROPERTIES
+    IMPORTED_LOCATION ${LELY_INSTALL_DIR}/lib/liblely-${_lib}${LELY_LIB_SUFFIX}
+    INTERFACE_INCLUDE_DIRECTORIES ${LELY_INSTALL_DIR}/include
+  )
+  add_dependencies(lely::${_lib} lely_core_ext)
+endforeach()

docker/Dockerfile

@@ -5,8 +5,12 @@ RUN pacman -Syu --noconfirm && \
       clang \
       cmake \
       make \
+      autoconf \
+      automake \
+      libtool \
       git \
       boost \
+      icu \
       zlib && \
     pacman -Scc --noconfirm
 

docker/Dockerfile.cross

@@ -8,6 +8,10 @@ RUN pacman -Syu --noconfirm && \
       ninja \
       git \
       openssh \
+      autoconf \
+      automake \
+      libtool \
+      make \
     && pacman -Scc --noconfirm
 
 WORKDIR /app

docker/Dockerfile.dev

@@ -9,7 +9,12 @@ RUN pacman -Syu --noconfirm && \
       git \
       openssh \
       sshpass \
+      autoconf \
+      automake \
+      libtool \
+      make \
       boost \
+      icu \
       sqlite3 \
       zlib && \
     pacman -Scc --noconfirm

docker/Dockerfile.sysroot

@@ -3,6 +3,7 @@ FROM --platform=linux/arm64 manjarolinux/base:latest
 RUN pacman -Syu --noconfirm && \
     pacman -S --noconfirm \
       boost \
+      icu \
       zlib \
       gcc && \
     pacman -Scc --noconfirm

src/can_player_dialog.cpp

@@ -29,6 +29,8 @@
 #include <linux/can.h>
 #include <linux/can/raw.h>
 
+#include <boost/regex.hpp>
+
 #define FMT_HEADER_ONLY
 #include <fmt/format.h>
 
@@ -45,6 +47,7 @@ ftxui::Component makeCanPlayerDialog(ftxui::ScreenInteractive *scr, signals_map_
     Impl(ftxui::ScreenInteractive *scr, signals_map_t &smap, bool &is_ready) {
       static sqlite::database *database = nullptr;
       static float canbus_player_focus_relative = 0;
+      static std::string player_filter_text;
 
       auto pgnContainer = ftxui::Container::Vertical({});
 
@@ -192,12 +195,14 @@ ftxui::Component makeCanPlayerDialog(ftxui::ScreenInteractive *scr, signals_map_
                 ->connect([](const std::vector<can_frame_update_s> &batch) {
                   for (const auto &entry : batch) {
                     uint32_t pgn_num = 0;
+
                     if (entry.canid.size() >= 6) {
                       auto pgn_str = entry.canid.substr(entry.canid.size() >= 8 ? entry.canid.size() - 6 : 2, 4);
                       std::stringstream ss;
                       ss << std::hex << pgn_str;
                       ss >> pgn_num;
                     }
+
                     received_pgns.insert(pgn_num);
                     if (pgs.contains(pgn_num) && pgs[pgn_num].forward && pgs[pgn_num].is_running) {
                       auto &pg = pgs[pgn_num];
@@ -405,6 +410,7 @@ ftxui::Component makeCanPlayerDialog(ftxui::ScreenInteractive *scr, signals_map_
                                                                   {"offset", spn_params.offset},
                                                               };
                                                             }(),
+
                                                             false, -100, ExpanderImpl::Root()) |
                                                             ftxui::Renderer([](ftxui::Element inner) {
                                                               return ftxui::hbox({
@@ -433,21 +439,52 @@ ftxui::Component makeCanPlayerDialog(ftxui::ScreenInteractive *scr, signals_map_
                                                                       fmt::format("{0:#b}", spn_params.raw))),
                                                               }),
 
-                                                              ftxui::hbox({
+                                                              [&]() -> ftxui::Element {
-                                                                  ftxui::text("PG payload: ") | ftxui::bold |
+                                                                // Build bit mask for this SPN's fragments
-                                                                      ftxui::color(ftxui::Color::Cyan),
+                                                                const auto &payload = spn_params.pg_ref->payload;
-                                                                  ftxui::text(fmt::format(
+                                                                std::vector<bool> highlight(payload.size() * 8, false);
-                                                                      "[{}]",
+                                                                for (const auto &frag : spn_params.fragments) {
-                                                                      [&]() {
+                                                                  int32_t start_bit =
-                                                                        std::string ret;
+                                                                      frag.byte_offset * 8 + frag.bit_offset;
-                                                                        for (const auto &byte :
-                                                                             spn_params.pg_ref->payload) {
-                                                                          ret += fmt::format("{0:#010b} ", byte);
-                                                                        }
 
-                                                                        return ret;
+                                                                  for (int32_t b = 0; b < frag.size; ++b) {
-                                                                      }())),
+                                                                    auto idx = static_cast<size_t>(start_bit + b);
-                                                              }),
+
+                                                                    if (idx < highlight.size()) {
+                                                                      highlight[idx] = true;
+                                                                    }
+                                                                  }
+                                                                }
+
+                                                                ftxui::Elements parts;
+                                                                parts.push_back(ftxui::text("PG payload: ") |
+                                                                                ftxui::bold |
+                                                                                ftxui::color(ftxui::Color::Cyan));
+                                                                parts.push_back(ftxui::text("["));
+
+                                                                for (size_t i = 0; i < payload.size(); ++i) {
+                                                                  parts.push_back(ftxui::text("0b"));
+
+                                                                  for (int32_t bit = 7; bit >= 0; --bit) {
+                                                                    bool is_set = (payload[i] >> bit) & 1;
+                                                                    bool is_spn = highlight[i * 8 + bit];
+                                                                    auto ch = ftxui::text(is_set ? "1" : "0");
+
+                                                                    if (is_spn) {
+                                                                      ch = ch | ftxui::color(ftxui::Color::Red) |
+                                                                           ftxui::bold;
+                                                                    }
+
+                                                                    parts.push_back(ch);
+                                                                  }
+
+                                                                  parts.push_back(ftxui::text(" "));
+                                                                }
+
+                                                                parts.push_back(ftxui::text("]"));
+
+                                                                return ftxui::hbox(std::move(parts));
+                                                              }(),
 
                                                               ftxui::separatorEmpty(),
                                                           });
@@ -461,7 +498,7 @@ ftxui::Component makeCanPlayerDialog(ftxui::ScreenInteractive *scr, signals_map_
                                 });
                               };
 
-                          pgnContainer->Add(ftxui::Container::Vertical({
+                          auto pgn_entry = ftxui::Container::Vertical({
                               ftxui::Container::Horizontal({
                                   ftxui::Checkbox({
                                       .checked = &pg_ref.selected,
@@ -621,6 +658,18 @@ ftxui::Component makeCanPlayerDialog(ftxui::ScreenInteractive *scr, signals_map_
                                       spnContainer,
                                   }) | ftxui::border,
                                   &pg_ref.selected),
+                          });
+
+                          pgnContainer->Add(ftxui::Maybe(pgn_entry, [&pg_ref = pgs[pgn]]() -> bool {
+                            if (player_filter_text.empty())
+                              return true;
+                            try {
+                              boost::regex re(player_filter_text, boost::regex_constants::icase);
+                              std::string subject = fmt::format("0x{:x} {}", pg_ref.pgn, pg_ref.label);
+                              return boost::regex_search(subject, re);
+                            } catch (...) {
+                              return true;
+                            }
                           }));
                         };
 
@@ -634,57 +683,106 @@ ftxui::Component makeCanPlayerDialog(ftxui::ScreenInteractive *scr, signals_map_
       }
 
       auto main = ftxui::Container::Vertical({
+          ftxui::Input({
+              .content = &player_filter_text,
+              .placeholder = "regex filter ...",
+              .transform = [](ftxui::InputState state) -> ftxui::Element {
+                bool valid = true;
+                if (!player_filter_text.empty()) {
+                  try {
+                    boost::regex(player_filter_text, boost::regex_constants::icase);
+                  } catch (...) {
+                    valid = false;
+                  }
+                }
+
+                state.element |= (!valid ? ftxui::color(ftxui::Color::Red) : ftxui::nothing) |
+                                 (state.focused ? ftxui::color(ftxui::Color::Cyan) : ftxui::nothing) |
+                                 (state.hovered ? ftxui::bold : ftxui::nothing);
+
+                return ftxui::hbox({
+                    ftxui::text(" Search: [ "),
+                    state.element |
+                        (state.hovered || state.focused ? ftxui::bgcolor(ftxui::Color::Grey11) : ftxui::nothing) |
+                        ftxui::xflex,
+                    ftxui::text(" ]"),
+                });
+              },
+              .multiline = false,
+          }),
+
+          ftxui::Renderer([]() { return ftxui::separator(); }),
+
           (pgnContainer | ftxui::Renderer([](ftxui::Element inner) {
              return inner | ftxui::focusPositionRelative(0, canbus_player_focus_relative) | ftxui::vscroll_indicator |
                     ftxui::frame | ftxui::flex;
-           })) |
+           })),
-              ftxui::CatchEvent([pgnContainer](ftxui::Event event) {
-                static const auto increment_focus = [pgnContainer = pgnContainer]() {
-                  if (auto n = pgnContainer->ChildCount(); n > 0)
-                    canbus_player_focus_relative =
-                        std::clamp(canbus_player_focus_relative + 1.0f / static_cast<float>(n), 0.0f, 1.0f);
-                };
-
-                static const auto decrement_focus = [pgnContainer = pgnContainer]() {
-                  if (auto n = pgnContainer->ChildCount(); n > 0)
-                    canbus_player_focus_relative =
-                        std::clamp(canbus_player_focus_relative - 1.0f / static_cast<float>(n), 0.0f, 1.0f);
-                };
-
-                if (!database) {
-                  return true;
-                }
-
-                if (event.is_mouse()) {
-                  switch (static_cast<enum ftxui::Mouse::Button>(event.mouse().button)) {
-                  case ftxui::Mouse::Button::WheelDown: {
-                    increment_focus();
-                    return true;
-                  } break;
-
-                  case ftxui::Mouse::Button::WheelUp: {
-                    decrement_focus();
-                    return true;
-                  } break;
-
-                  default:
-                    break;
-                  }
-                } else if (!event.is_character()) {
-                  if (event == ftxui::Event::ArrowDown) {
-                    increment_focus();
-                    return true;
-                  } else if (event == ftxui::Event::ArrowUp) {
-                    decrement_focus();
-                    return true;
-                  }
-                }
-
-                return false;
-              }),
       });
 
-      Add({main});
+      Add({main | ftxui::CatchEvent([pgnContainer](ftxui::Event event) {
+             if (!database)
+               return true;
+
+             const auto scroll_step = []() -> float {
+               size_t visible_lines = 0;
+               boost::regex re;
+               bool has_filter = !player_filter_text.empty();
+
+               if (has_filter) {
+                 try {
+                   re = boost::regex(player_filter_text, boost::regex_constants::icase);
+                 } catch (...) {
+                   has_filter = false;
+                 }
+               }
+
+               for (const auto &[_, pg] : pgs) {
+                 if (has_filter) {
+                   std::string subject = fmt::format("0x{:x} {}", pg.pgn, pg.label);
+                   if (!boost::regex_search(subject, re)) {
+                     continue;
+                   }
+                 }
+
+                 ++visible_lines;
+
+                 if (pg.selected) {
+                   visible_lines += 15;
+                 }
+               }
+
+               return visible_lines > 0 ? 1.0f / static_cast<float>(visible_lines) : 0.03f;
+             };
+
+             if (event.is_mouse()) {
+               switch (static_cast<enum ftxui::Mouse::Button>(event.mouse().button)) {
+               case ftxui::Mouse::Button::WheelDown: {
+                 canbus_player_focus_relative = std::clamp(canbus_player_focus_relative + scroll_step(), 0.0f, 1.0f);
+                 return true;
+               }
+
+               case ftxui::Mouse::Button::WheelUp: {
+                 canbus_player_focus_relative = std::clamp(canbus_player_focus_relative - scroll_step(), 0.0f, 1.0f);
+                 return true;
+               }
+
+               default:
+                 break;
+               }
+             } else if (!event.is_character()) {
+               if (event == ftxui::Event::ArrowDown) {
+
+                 canbus_player_focus_relative = std::clamp(canbus_player_focus_relative + scroll_step(), 0.0f, 1.0f);
+                 return true;
+               } else if (event == ftxui::Event::ArrowUp) {
+
+                 canbus_player_focus_relative = std::clamp(canbus_player_focus_relative - scroll_step(), 0.0f, 1.0f);
+                 return true;
+               }
+             }
+
+             return false;
+           })});
     }
   };
 

src/canid_unit.cpp

@@ -1,5 +1,6 @@
 // #include <algorithm>
 // #include <cmath>
+#include <boost/regex.hpp>
 #include <cstdint>
 #include <ftxui/component/component.hpp>
 #include <ftxui/component/component_options.hpp>
@@ -330,16 +331,19 @@ void CanIDUnit::update(const can_frame_data_s &data, const can_frame_diff_s &dif
           total_bits += bc;
         }
       }
+
       if (settings.big_endian && total_bits > 8) {
         int64_t swapped = 0;
         size_t total_bytes = (total_bits + 7) / 8;
+
         for (size_t i = 0; i < total_bytes; ++i)
           swapped |= ((result >> (i * 8)) & 0xFF) << ((total_bytes - 1 - i) * 8);
         result = swapped;
       }
-      double spn_val = static_cast<double>(result) * resolution + offset_val;
 
+      double spn_val = static_cast<double>(result) * resolution + offset_val;
       nlohmann::json::array_t frags_json;
+
       for (size_t fi = 0; fi < settings.fragments.size(); ++fi) {
         const auto &f = settings.fragments[fi];
         int32_t bo = 0, bi = 0, bc = 0;
@@ -424,8 +428,10 @@ void CanIDUnit::update(const can_frame_data_s &data, const can_frame_diff_s &dif
             for (const auto &[spn_k, spn_v] : spns_arr_v.items()) {
               if (spn_k == "SPN name") {
                 std::string spn_name = spn_v.get<std::string>();
-                if (spn_name.find("(custom)") != std::string::npos)
+                if (spn_name.find("(custom)") != std::string::npos) {
                   continue;
+                }
+
                 auto &spn_map = std::get<2u>(s_canbus_parameters_export_map_[m_canid_]);
 
                 spn_map.insert_or_assign(spn_name, std::make_tuple(false, false, spns_arr_v));
@@ -529,7 +535,19 @@ void CanIDUnit::update(const can_frame_data_s &data, const can_frame_diff_s &dif
                                            !selectors_container->ChildCount();
                                   }));
 
-      m_cansettings_dialog_->Add(container);
+      m_cansettings_dialog_->Add(ftxui::Maybe(container, [this]() -> bool {
+        if (!s_export_filter_text_ || s_export_filter_text_->empty()) {
+          return true;
+        }
+
+        try {
+          boost::regex re(*s_export_filter_text_, boost::regex_constants::icase);
+          std::string subject = m_canid_ + " " + getLabel();
+          return boost::regex_search(subject, re);
+        } catch (...) {
+          return true;
+        }
+      }));
     }
 
     // Add custom SPNs to export dialog if not already present (tracked by tag_id)
@@ -579,6 +597,7 @@ void CanIDUnit::update(const can_frame_data_s &data, const can_frame_diff_s &dif
                                           .transform =
                                               [this, tag_id](const ftxui::EntryState state) {
                                                 std::string display_name = "custom";
+
                                                 if (m_spnSettingsMap_ && m_spnSettingsMap_->contains(m_canid_) &&
                                                     (*m_spnSettingsMap_)[m_canid_].contains(tag_id)) {
                                                   auto &s = (*m_spnSettingsMap_)[m_canid_][tag_id];
@@ -619,26 +638,33 @@ void CanIDUnit::update(const can_frame_data_s &data, const can_frame_diff_s &dif
     }
 
     // Rebuild FromLive for custom SPN export entries when fragment count changes
-    if (m_spnSettingsMap_ && m_spnSettingsMap_->contains(m_canid_) && !m_data_verbose_->is_null() && m_data_verbose_->contains("SPNs")) {
+    if (m_spnSettingsMap_ && m_spnSettingsMap_->contains(m_canid_) && !m_data_verbose_->is_null() &&
+        m_data_verbose_->contains("SPNs")) {
       for (auto &[tag_id, pair] : m_export_custom_containers_) {
         auto &[wrapper, prev_frag_count] = pair;
+
         size_t cur_frag_count = 0;
         if ((*m_spnSettingsMap_)[m_canid_].contains(tag_id)) {
           cur_frag_count = (*m_spnSettingsMap_)[m_canid_][tag_id].fragments.size();
         }
+
         if (cur_frag_count != prev_frag_count) {
           wrapper->DetachAllChildren();
           std::string search_name;
+
           if ((*m_spnSettingsMap_)[m_canid_].contains(tag_id)) {
             search_name = (*m_spnSettingsMap_)[m_canid_][tag_id].spn_name + " (custom)";
           }
+
           const auto &spns = (*m_data_verbose_)["SPNs"];
           for (size_t i = 0; i < spns.size(); ++i) {
             if (spns[i].contains("SPN name") && spns[i]["SPN name"].get<std::string>() == search_name) {
-              wrapper->Add(FromLive(m_data_verbose_, nlohmann::json::json_pointer("/SPNs/" + std::to_string(i)), false, -100, ExpanderImpl::Root()));
+              wrapper->Add(FromLive(m_data_verbose_, nlohmann::json::json_pointer("/SPNs/" + std::to_string(i)), false,
+                                    -100, ExpanderImpl::Root()));
               break;
             }
           }
+
           prev_frag_count = cur_frag_count;
         }
       }

src/canid_unit.hpp

@@ -48,6 +48,16 @@ public:
   inline ftxui::Component getSpnSettingsForm() { return m_spnSettingsForm_; }
   inline const auto &getParametersExportMap() const { return s_canbus_parameters_export_map_; }
 
+  static inline std::string *s_export_filter_text_ = nullptr;
+
+  static inline std::map<
+      /* canid */ std::string,
+      std::tuple</* deployed flag */ bool, /* has data flag */ bool,
+                 /* Selected spns to export  */
+                 std::map</* spn name */ std::string,
+                          std::tuple</* deployed  */ bool, /* selected */ bool, /* data */ nlohmann::json>>>>
+      s_canbus_parameters_export_map_ = {};
+
   void update(const can_frame_data_s &data, const can_frame_diff_s &diff, std::shared_ptr<nlohmann::json> verbose,
               std::shared_ptr<nlohmann::json> brief);
 
@@ -71,12 +81,4 @@ private:
   size_t m_last_brief_spn_count_ = 0;
   ftxui::Component m_export_selectors_;
   std::map<int32_t, std::pair<ftxui::Component, size_t>> m_export_custom_containers_;
-
-  static inline std::map<
-      /* canid */ std::string,
-      std::tuple</* deployed flag */ bool, /* has data flag */ bool,
-                 /* Selected spns to export  */
-                 std::map</* spn name */ std::string,
-                          std::tuple</* deployed  */ bool, /* selected */ bool, /* data */ nlohmann::json>>>>
-      s_canbus_parameters_export_map_ = {};
 };

src/csv.cpp

@@ -0,0 +1,125 @@
+#include <fstream>
+#include <map>
+#include <memory>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+#define FMT_HEADER_ONLY
+#include <fmt/format.h>
+
+#include "j1939_db.hpp"
+#include "sqlite_modern_cpp.h"
+
+static std::vector<std::string> parseCsvLine(const std::string &line) {
+  std::vector<std::string> fields;
+  std::string field;
+  bool in_quotes = false;
+
+  for (size_t i = 0; i < line.size(); ++i) {
+    char c = line[i];
+
+    if (in_quotes) {
+      if (c == '"') {
+        if (i + 1 < line.size() && line[i + 1] == '"') {
+          field += '"';
+          ++i;
+        } else {
+          in_quotes = false;
+        }
+      } else {
+        field += c;
+      }
+    } else {
+
+      if (c == '"') {
+        in_quotes = true;
+      } else if (c == ',') {
+        fields.push_back(std::move(field));
+        field.clear();
+      } else {
+        field += c;
+      }
+    }
+  }
+
+  fields.push_back(std::move(field));
+  return fields;
+}
+
+std::unique_ptr<sqlite::database> parseCsv(const std::string &file) {
+  std::ifstream ifs(file);
+  if (!ifs.is_open()) {
+    throw std::runtime_error(fmt::format("Cannot open file {}", file));
+  }
+
+  auto db_ptr = std::make_unique<sqlite::database>(":memory:");
+  auto &db = *db_ptr;
+
+  initJ1939Database(db);
+
+  const auto &pgn_mapping = J1939MappingTables::pgn();
+  const auto &spn_mapping = J1939MappingTables::spn();
+
+  // Read header line and build column_index -> db_column_name maps
+  std::map<size_t, std::string> pgn_col_to_db, spn_col_to_db;
+
+  std::string header_line;
+  if (!std::getline(ifs, header_line)) {
+    throw std::runtime_error("CSV file is empty");
+  }
+
+  auto headers = parseCsvLine(header_line);
+  for (size_t i = 0; i < headers.size(); ++i) {
+    const auto &h = headers[i];
+    if (auto it = pgn_mapping.find(h); it != pgn_mapping.end()) {
+
+      pgn_col_to_db[i] = std::string(std::get<1u>(it->second));
+    } else if (auto it2 = spn_mapping.find(h); it2 != spn_mapping.end()) {
+
+      spn_col_to_db[i] = std::string(std::get<1u>(it2->second));
+    }
+  }
+
+  const auto pgn_insert_sql = buildPgnInsertSql();
+  const auto spn_insert_sql = buildSpnInsertSql();
+
+  db << "BEGIN TRANSACTION";
+
+  std::string line;
+  while (std::getline(ifs, line)) {
+    if (line.empty())
+      continue;
+
+    while (std::count(line.begin(), line.end(), '"') % 2 != 0) {
+      std::string next;
+      if (!std::getline(ifs, next))
+        break;
+      line += '\n';
+      line += next;
+    }
+
+    auto fields = parseCsvLine(line);
+    std::map<std::string, std::string> pgn_row_map, spn_row_map;
+
+    for (size_t i = 0; i < fields.size(); ++i) {
+      if (fields[i].empty())
+        continue;
+
+      if (auto it = pgn_col_to_db.find(i); it != pgn_col_to_db.end()) {
+        pgn_row_map[it->second] = std::move(fields[i]);
+      } else if (auto it2 = spn_col_to_db.find(i); it2 != spn_col_to_db.end()) {
+        spn_row_map[it2->second] = std::move(fields[i]);
+      }
+    }
+
+    if (!pgn_row_map.empty() && !spn_row_map.empty()) {
+      insertJ1939Row(db, pgn_insert_sql, spn_insert_sql, pgn_row_map, spn_row_map);
+    }
+  }
+
+  db << "COMMIT";
+
+  return db_ptr;
+}

src/filedialog.cpp

@@ -77,6 +77,7 @@ ftxui::Component makeFileDialog(ftxui::ScreenInteractive *scr, signals_map_t &sm
           ftxui::Container::Vertical({
               ftxui::Renderer([&]() -> ftxui::Element { return ftxui::text("Export file"); }) |
                   ftxui::color(ftxui::Color::Red) | ftxui::hcenter,
+              ftxui::Renderer([]() { return ftxui::separator(); }),
 
               ftxui::Container::Horizontal({
                   ftxui::Renderer([]() -> ftxui::Element {
@@ -90,9 +91,9 @@ ftxui::Component makeFileDialog(ftxui::ScreenInteractive *scr, signals_map_t &sm
                   }),
               }),
 
-              ftxui::Renderer([]() { return ftxui::separatorEmpty(); }),
+              ftxui::Renderer([]() { return ftxui::separator(); }),
-
               entryList | ftxui::vscroll_indicator | ftxui::frame | ftxui::flex,
+              ftxui::Renderer([]() { return ftxui::separator(); }),
 
               ftxui::Container::Horizontal({
                   ftxui::Button({
@@ -104,7 +105,7 @@ ftxui::Component makeFileDialog(ftxui::ScreenInteractive *scr, signals_map_t &sm
                           },
 
                       .transform = [this](const ftxui::EntryState &state) -> ftxui::Element {
-                        return ftxui::text(" >[Export]< ") | (state.focused ? ftxui::bold : ftxui::nothing) |
+                        return ftxui::text(" >[export]< ") | (state.focused ? ftxui::bold : ftxui::nothing) |
                                ftxui::color(ftxui::Color::Cyan) |
                                (state.focused ? ftxui::bgcolor(ftxui::Color::Grey11) : ftxui::nothing);
                       },
@@ -113,7 +114,7 @@ ftxui::Component makeFileDialog(ftxui::ScreenInteractive *scr, signals_map_t &sm
                   ftxui::Button({
                       .on_click = [&]() { shown = false; },
                       .transform = [this](const ftxui::EntryState &state) -> ftxui::Element {
-                        return ftxui::text(" >[Cancel]< ") | (state.focused ? ftxui::bold : ftxui::nothing) |
+                        return ftxui::text(" >[cancel]< ") | (state.focused ? ftxui::bold : ftxui::nothing) |
                                ftxui::color(ftxui::Color::Cyan) |
                                (state.focused ? ftxui::bgcolor(ftxui::Color::Grey11) : ftxui::nothing);
                       },

src/j1939_db.cpp

@@ -0,0 +1,183 @@
+#include "j1939_db.hpp"
+#include "parsers.hpp"
+
+#include <cmath>
+
+#define FMT_HEADER_ONLY
+#include <fmt/format.h>
+
+void initJ1939Database(sqlite::database &db) {
+  db << R"(
+    CREATE TABLE IF NOT EXISTS pgns (
+      id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+      pgn INTEGER UNIQUE,
+      pg_label TEXT,
+      pg_acronym TEXT,
+      pg_descr TEXT,
+      edp INTEGER,
+      dp INTEGER,
+      pf INTEGER,
+      ps INTEGER,
+      pg_datalen INTEGER,
+      pg_priority INTEGER
+    );
+  )";
+
+  db << R"(
+    CREATE TABLE IF NOT EXISTS spns (
+      id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+      pgn INTEGER,
+      spn INTEGER UNIQUE,
+      spn_name TEXT,
+      spn_pos TEXT,
+      spn_length INTEGER,
+      resolution REAL,
+      offset REAL,
+      data_range TEXT,
+      min_value REAL,
+      max_value REAL,
+      units TEXT,
+      slot_id TEXT,
+      slot_name TEXT,
+      spn_type TEXT
+    );
+  )";
+
+  db << R"(
+    CREATE TABLE IF NOT EXISTS spn_fragments (
+      id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+      spn INTEGER,
+      pgn INTEGER,
+      byte_offset INTEGER,
+      bit_offset INTEGER,
+      size INTEGER
+    );
+  )";
+
+  db << "PRAGMA journal_mode = OFF";
+  db << "PRAGMA synchronous = OFF";
+  db << "PRAGMA foreign_keys = ON;";
+
+  db << "CREATE INDEX IF NOT EXISTS idx_spns_pgn ON spns(pgn);";
+  db << "CREATE INDEX IF NOT EXISTS idx_spn_fragments_spn ON spn_fragments(spn);";
+  db << "CREATE INDEX IF NOT EXISTS idx_spn_fragments_spn_pgn ON spn_fragments(spn, pgn);";
+}
+
+std::string buildPgnInsertSql() {
+  const auto &pgn_mapping = J1939MappingTables::pgn();
+  std::string cols, placeholders;
+  for (const auto &[k, v] : pgn_mapping) {
+    if (!cols.empty()) { cols += ", "; placeholders += ", "; }
+    cols += std::get<1u>(v);
+    placeholders += "?";
+  }
+  return fmt::format("INSERT OR REPLACE INTO pgns ({}) VALUES ({})", cols, placeholders);
+}
+
+std::string buildSpnInsertSql() {
+  const auto &spn_mapping = J1939MappingTables::spn();
+  std::string cols = "pgn", placeholders = "?";
+  for (const auto &[k, v] : spn_mapping) {
+    cols += ", ";
+    placeholders += ", ";
+    if (std::get<1u>(v) == "data_range") {
+      cols += "min_value, max_value";
+      placeholders += "?, ?";
+    } else {
+      cols += std::get<1u>(v);
+      placeholders += "?";
+    }
+  }
+  return fmt::format("INSERT OR REPLACE INTO spns ({}) VALUES ({})", cols, placeholders);
+}
+
+void insertJ1939Row(sqlite::database &db, const std::string &pgn_insert_sql, const std::string &spn_insert_sql,
+                    std::map<std::string, std::string> &pgn_row_map, std::map<std::string, std::string> &spn_row_map) {
+  const auto &pgn_mapping = J1939MappingTables::pgn();
+  const auto &spn_mapping = J1939MappingTables::spn();
+
+  // Insert PGN
+  try {
+    auto ps = db << pgn_insert_sql;
+    for (const auto &[k, v] : pgn_mapping) {
+      ps << pgn_row_map[std::get<1u>(v).data()];
+    }
+    ps.execute();
+  } catch (const sqlite::sqlite_exception &e) {
+    if (e.get_extended_code() != SQLITE_CONSTRAINT_UNIQUE) throw;
+  }
+
+  // Insert SPN
+  struct {
+    bool parts_inserted_flag = false;
+    double min = 0.0, max = 0.0;
+    size_t size_bits = 0u;
+  } calc;
+
+  // Pre-compute size_bits
+  {
+    auto it = spn_mapping.find("SPN Length");
+    if (it != spn_mapping.end()) {
+      auto size = parsers::parseSpnSize(spn_row_map[std::get<1u>(it->second).data()]);
+      calc.size_bits = size.has_value() ? size.value().size_bits : 0u;
+    }
+  }
+
+  for (const auto &[k, v] : spn_row_map) {
+    try {
+      auto ps = db << spn_insert_sql;
+      ps << pgn_row_map["pgn"];
+
+      for (const auto &[k, v] : spn_mapping) {
+        if (std::get<1u>(v) == "data_range") {
+          auto range = parsers::parseSpnDataRange(spn_row_map[std::get<1u>(v).data()]);
+          if (range.has_value()) {
+            calc.min = range.value().min;
+            calc.max = range.value().max;
+            ps << range.value().min << range.value().max;
+          } else {
+            ps << nullptr << nullptr;
+          }
+        } else if (std::get<1u>(v) == "offset") {
+          auto offset = parsers::parseSpnOffset(spn_row_map[std::get<1u>(v).data()]);
+          ps << (offset.has_value() ? offset.value().offset : 0.0);
+        } else if (std::get<1u>(v) == "spn_length") {
+          auto size = parsers::parseSpnSize(spn_row_map[std::get<1u>(v).data()]);
+          calc.size_bits = size.has_value() ? size.value().size_bits : 0u;
+          ps << calc.size_bits;
+        } else if (std::get<1u>(v) == "resolution") {
+          double calculated = (calc.max - calc.min) / (std::pow(2.0, calc.size_bits) - 1.0);
+          if (std::fabs(calculated - 1.0) < 1e-9) {
+            ps << calculated;
+          } else {
+            auto resolution = parsers::parseSpnResolution(spn_row_map[std::get<1u>(v).data()]);
+            ps << (resolution.has_value() ? resolution.value().resolution : 1.0);
+          }
+        } else {
+          ps << spn_row_map[std::get<1u>(v).data()];
+        }
+      }
+
+      ps.execute();
+
+      // Insert SPN fragments
+      if (!calc.parts_inserted_flag) {
+        auto size = parsers::parseSpnSize(spn_row_map[std::get<1u>(spn_mapping.at("SPN Length")).data()]);
+        if (size.has_value()) {
+          auto spn_fragments = parsers::parseSpnPosition(
+              size.value().size_bits, spn_row_map[std::get<1u>(spn_mapping.at("SPN Position in PG")).data()]);
+          if (spn_fragments.has_value()) {
+            auto spn = std::stoll(spn_row_map["spn"]);
+            for (const auto &part : spn_fragments.value().spn_fragments) {
+              db << R"(INSERT OR REPLACE INTO spn_fragments (spn, pgn, byte_offset, bit_offset, size) VALUES (?, ?, ?, ?, ?);)"
+                 << spn << std::stoll(pgn_row_map["pgn"]) << part.byte_offset << part.bit_offset << part.size;
+            }
+            calc.parts_inserted_flag = true;
+          }
+        }
+      }
+    } catch (const sqlite::sqlite_exception &e) {
+      if (e.get_extended_code() != SQLITE_CONSTRAINT_UNIQUE) throw;
+    }
+  }
+}

src/j1939_db.hpp

@@ -0,0 +1,56 @@
+#pragma once
+
+#include <map>
+#include <memory>
+#include <string>
+#include <string_view>
+#include <tuple>
+
+#include "sqlite_modern_cpp.h"
+
+struct J1939MappingTables {
+  static const std::map<std::string_view, std::tuple<size_t, std::string_view>> &pgn() {
+    static const std::map<std::string_view, std::tuple<size_t, std::string_view>> t = {
+        {"PGN", {5u, "pgn"}},
+        {"Parameter Group Label", {6u, "pg_label"}},
+        {"PG Acronym", {7u, "pg_acronym"}},
+        {"PG Description", {8u, "pg_descr"}},
+        {"EDP", {9u, "edp"}},
+        {"DP", {10u, "dp"}},
+        {"PF", {11u, "pf"}},
+        {"PS", {12u, "ps"}},
+        {"PG Data Length", {15u, "pg_datalen"}},
+        {"Default Priority", {16u, "pg_priority"}},
+    };
+    return t;
+  }
+
+  static const std::map<std::string_view, std::tuple<size_t, std::string_view>> &spn() {
+    static const std::map<std::string_view, std::tuple<size_t, std::string_view>> t = {
+        {"SPN", {19u, "spn"}},
+        {"SPN Name", {20u, "spn_name"}},
+        {"SPN Position in PG", {18u, "spn_pos"}},
+        {"SPN Length", {22u, "spn_length"}},
+        {"Offset", {24u, "offset"}},
+        {"Data Range", {25u, "data_range"}},
+        {"Resolution", {23u, "resolution"}},
+        {"Units", {27u, "units"}},
+        {"SLOT Identifier", {28u, "slot_id"}},
+        {"SLOT Name", {29u, "slot_name"}},
+        {"SPN Type", {30u, "spn_type"}},
+    };
+    return t;
+  }
+};
+
+void initJ1939Database(sqlite::database &db);
+
+// Build INSERT SQL for pgns table (call once, reuse)
+std::string buildPgnInsertSql();
+
+// Build INSERT SQL for spns table (call once, reuse)
+std::string buildSpnInsertSql();
+
+// Insert a row into the database. pgn_row_map/spn_row_map are column_name -> value.
+void insertJ1939Row(sqlite::database &db, const std::string &pgn_insert_sql, const std::string &spn_insert_sql,
+                    std::map<std::string, std::string> &pgn_row_map, std::map<std::string, std::string> &spn_row_map);

src/main.cpp

@@ -46,14 +46,15 @@ int32_t main(int32_t argc, char *argv[]) {
   static std::atomic<sqlite::database *> j1939_db{nullptr};
   static std::stop_source aggregator_task_stop, refresh_task_stop, headless_task_stop;
   static TinyProcessLib::Process *p = nullptr;
-  std::future<void> xlsx_parser_task, headless_task;
+  std::future<void> j1939_parser_task, headless_task;
   extern std::unique_ptr<sqlite::database> parseXlsx(const std::string &file);
+  extern std::unique_ptr<sqlite::database> parseCsv(const std::string &file);
   static std::unique_ptr<sqlite::database> j1939_db_owner;
   static std::unique_ptr<Recorder> recorder;
   static std::unique_ptr<HeadlessHandler> headless_handler;
 
   static struct {
-    std::string docfile, command = "", output_file = "", record_db_path = "";
+    std::string xlsx_file, csv_file, command = "", output_file = "", record_db_path = "";
     bool show_help = false, headless_mode = false, sync_to_server = false, record_mode = false, tui_mode = false;
   } cli_opts;
 
@@ -90,16 +91,26 @@ int32_t main(int32_t argc, char *argv[]) {
         clipp::option("-e", "--execute-command") &
             clipp::value("command", cli_opts.command).call([&]() {}).doc("execute cli command to read its output"),
 
-        clipp::required("-j1939", "--j1939-document") &
+        (clipp::option("-j1939-xlsx") &
-            clipp::value("J1939 Document file", cli_opts.docfile)
+             clipp::value("J1939 XLSX file", cli_opts.xlsx_file)
-                .call([&]() {
+                 .call([&]() {
-                  xlsx_parser_task = std::async(std::launch::async, [&]() {
+                   j1939_parser_task = std::async(std::launch::async, [&]() {
-                    j1939_db_owner = parseXlsx(cli_opts.docfile);
+                     j1939_db_owner = parseXlsx(cli_opts.xlsx_file);
-                    j1939_db.store(j1939_db_owner.get());
+                     j1939_db.store(j1939_db_owner.get());
-                    signals.map.get<void(sqlite::database &)>("j1939_database_ready")->operator()(*j1939_db_owner);
+                     signals.map.get<void(sqlite::database &)>("j1939_database_ready")->operator()(*j1939_db_owner);
-                  });
+                   });
-                })
+                 })
-                .doc("provide .xlsx document file for J1939 processing"));
+                 .doc("J1939 Digital Annex .xlsx file")) |
+        (clipp::option("-j1939-csv") &
+             clipp::value("J1939 CSV file", cli_opts.csv_file)
+                 .call([&]() {
+                   j1939_parser_task = std::async(std::launch::async, [&]() {
+                     j1939_db_owner = parseCsv(cli_opts.csv_file);
+                     j1939_db.store(j1939_db_owner.get());
+                     signals.map.get<void(sqlite::database &)>("j1939_database_ready")->operator()(*j1939_db_owner);
+                   });
+                 })
+                 .doc("J1939 Digital Annex .csv file")));
 
     auto man = clipp::make_man_page(cli, argv[0]);
     auto cli_with_help = (cli | clipp::option("-h", "--help").set(cli_opts.show_help).doc("show this help").call([&]() {
@@ -415,7 +426,7 @@ int32_t main(int32_t argc, char *argv[]) {
   {
     const char *names[] = {"xlsx_parser", "aggregator", "refresh", "headless"};
     int idx = 0;
-    for (auto *task : {&xlsx_parser_task, &aggregator_task, &refresh_task, &headless_task}) {
+    for (auto *task : {&j1939_parser_task, &aggregator_task, &refresh_task, &headless_task}) {
       if (task && task->valid()) {
         task->wait_for(std::chrono::seconds(3));
       }

src/mainform.cpp

@@ -30,6 +30,7 @@ ftxui::Component makeMainForm(ftxui::ScreenInteractive *screen, signals_map_t &s
       static float canbus_params_focus_relative = 0;
       static std::string canid_active;
       static std::string filter_text;
+      static std::string export_filter_text;
       static size_t tags_count = 0u;
       static std::unordered_map<std::string, std::shared_ptr<CanIDUnit>> canid_lookup;
       static std::atomic<sqlite::database *> database_atomic{nullptr};
@@ -47,6 +48,7 @@ ftxui::Component makeMainForm(ftxui::ScreenInteractive *screen, signals_map_t &s
       static std::map<std::string, std::map<int32_t, ftxui::Component>> spnSettingsFormMap;
       static auto spn_export_dialog = ftxui::Container::Vertical({});
       static auto canbus_params_export_dialog = ftxui::Container::Vertical({});
+      CanIDUnit::s_export_filter_text_ = &export_filter_text;
       static auto canbus_player_dialog = ftxui::Container::Vertical({});
 
       static const auto convertParametersMapToJson = []() {
@@ -314,6 +316,37 @@ ftxui::Component makeMainForm(ftxui::ScreenInteractive *screen, signals_map_t &s
                           }) | ftxui::hcenter,
                           ftxui::Renderer([]() { return ftxui::separator(); }),
 
+                          ftxui::Input({
+                              .content = &export_filter_text,
+                              .placeholder = "regex filter ...",
+                              .transform = [](ftxui::InputState state) -> ftxui::Element {
+                                bool valid = true;
+                                if (!export_filter_text.empty()) {
+                                  try {
+                                    boost::regex(export_filter_text, boost::regex_constants::icase);
+                                  } catch (...) {
+                                    valid = false;
+                                  }
+                                }
+
+                                state.element |= (!valid ? ftxui::color(ftxui::Color::Red) : ftxui::nothing) |
+                                                 (state.focused ? ftxui::color(ftxui::Color::Cyan) : ftxui::nothing) |
+                                                 (state.hovered ? ftxui::bold : ftxui::nothing);
+
+                                return ftxui::hbox({
+                                           ftxui::text(" Search: [ "),
+                                           state.element |
+                                               (state.hovered || state.focused ? ftxui::bgcolor(ftxui::Color::Grey11)
+                                                                               : ftxui::nothing) |
+                                               ftxui::xflex,
+                                           ftxui::text(" ]"),
+                                       });
+                              },
+                              .multiline = false,
+                          }),
+
+                          ftxui::Renderer([]() { return ftxui::separator(); }),
+
                           (canbus_params_export_dialog | ftxui::Renderer([](ftxui::Element inner) {
                              return inner | ftxui::focusPositionRelative(0, canbus_params_focus_relative) |
                                     ftxui::vscroll_indicator | ftxui::frame | ftxui::flex;
@@ -362,48 +395,48 @@ ftxui::Component makeMainForm(ftxui::ScreenInteractive *screen, signals_map_t &s
                       }) | ftxui::size(ftxui::WIDTH, ftxui::EQUAL, 96) |
                           ftxui::size(ftxui::HEIGHT, ftxui::EQUAL, 48) | ftxui::border |
                           ftxui::CatchEvent([](ftxui::Event event) {
-                            static constexpr float scroll_step = 0.03f;
+                            const auto scroll_step = []() -> float {
-                            static const auto increment_focus = []() {
+                              size_t visible = 0;
-                              canbus_params_focus_relative =
+                              for (const auto &[_, entry] : CanIDUnit::s_canbus_parameters_export_map_) {
-                                  std::clamp(canbus_params_focus_relative + scroll_step, 0.0f, 1.0f);
+                                if (export_filter_text.empty()) {
-                            };
+                                  ++visible;
-                            static const auto decrement_focus = []() {
+                                } else {
-                              canbus_params_focus_relative =
+                                  // Filtering is handled by Maybe in canid_unit.cpp,
-                                  std::clamp(canbus_params_focus_relative - scroll_step, 0.0f, 1.0f);
+                                  // just count all for scroll step estimation
+                                  ++visible;
+                                }
+                                if (std::get<0u>(entry))
+                                  visible += 5;
+                              }
+                              return visible > 0 ? 1.0f / static_cast<float>(visible) : 0.03f;
                             };
 
                             if (event.is_mouse()) {
                               switch (static_cast<enum ftxui::Mouse::Button>(event.mouse().button)) {
-                              case ftxui::Mouse::Button::WheelDown: {
+                              case ftxui::Mouse::Button::WheelDown:
-                                increment_focus();
+                                canbus_params_focus_relative =
-                                goto done;
+                                    std::clamp(canbus_params_focus_relative + scroll_step(), 0.0f, 1.0f);
-                              } break;
+                                return true;
-
+                              case ftxui::Mouse::Button::WheelUp:
-                              case ftxui::Mouse::Button::WheelUp: {
+                                canbus_params_focus_relative =
-                                decrement_focus();
+                                    std::clamp(canbus_params_focus_relative - scroll_step(), 0.0f, 1.0f);
-                                goto done;
+                                return true;
-                              } break;
-
                               default:
                                 break;
                               }
                             } else if (!event.is_character()) {
                               if (event == ftxui::Event::ArrowDown) {
-
+                                canbus_params_focus_relative =
-                                increment_focus();
+                                    std::clamp(canbus_params_focus_relative + scroll_step(), 0.0f, 1.0f);
-                                goto done;
+                                return true;
                               } else if (event == ftxui::Event::ArrowUp) {
-
+                                canbus_params_focus_relative =
-                                decrement_focus();
+                                    std::clamp(canbus_params_focus_relative - scroll_step(), 0.0f, 1.0f);
-                                goto done;
+                                return true;
                               }
                             }
 
-                          forward:
                             return false;
-
-                          done:
-                            return true;
                           }),
                       &canbus_params_export_dialog_shown) |
                   ftxui::Modal(makeFileDialog(screen, smap, file_dialog_shown), &file_dialog_shown) |

src/xlsx.cpp

@@ -1,117 +1,28 @@
-#include <filesystem>
-#include <fstream>
-#include <limits>
 #include <map>
 #include <memory>
-#include <nlohmann/json.hpp>
-#include <sstream>
 #include <stdexcept>
-#include <unistd.h>
+#include <string>
 
 #define FMT_HEADER_ONLY
 #include <fmt/format.h>
-#include <fmt/ranges.h>
 
-// for XLSX files
 #include <xlnt/xlnt.hpp>
 
-// For sqlite
+#include "j1939_db.hpp"
-#include "parsers.hpp"
 #include "sqlite_modern_cpp.h"
 
-static void init_db(sqlite::database &db) {
-  try {
-    db << R"(
-      CREATE TABLE IF NOT EXISTS pgns (
-        id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
-        pgn INTEGER UNIQUE,
-        pg_label TEXT,
-        pg_acronym TEXT,
-        pg_descr TEXT,
-        edp INTEGER,
-        dp INTEGER,
-        pf INTEGER,
-        ps INTEGER,
-        pg_datalen INTEGER,
-        pg_priority INTEGER
-      );
-    )";
-
-    db << R"(
-      CREATE TABLE IF NOT EXISTS spns (
-        id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
-        pgn INTEGER,
-        spn INTEGER UNIQUE,
-        spn_name TEXT,
-        spn_pos TEXT,
-        spn_length INTEGER,
-        resolution REAL,
-        offset REAL,
-        data_range TEXT,
-        min_value REAL,
-        max_value REAL,
-        units TEXT,
-        slot_id TEXT,
-        slot_name TEXT,
-        spn_type TEXT
-      );
-    )";
-
-    db << R"(
-      CREATE TABLE IF NOT EXISTS spn_fragments (
-        id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
-        spn INTEGER,
-        pgn INTEGER,
-        byte_offset INTEGER,
-        bit_offset INTEGER,
-        size INTEGER
-      );
-    )";
-
-    db << "PRAGMA journal_mode = OFF";
-    db << "PRAGMA synchronous = OFF";
-    db << "PRAGMA foreign_keys = ON;";
-  } catch (const sqlite::sqlite_exception &e) {
-    throw;
-  }
-}
-
 std::unique_ptr<sqlite::database> parseXlsx(const std::string &file) {
   auto db_ptr = std::make_unique<sqlite::database>(":memory:");
   auto &db = *db_ptr;
 
-  init_db(db);
+  initJ1939Database(db);
 
-  static const std::map<std::string_view, std::tuple<size_t, std::string_view>>
+  const auto &pgn_mapping = J1939MappingTables::pgn();
-      pgn_mapping_table =
+  const auto &spn_mapping = J1939MappingTables::spn();
-          {
-              {"PGN", {5u, "pgn"}},
-              {"Parameter Group Label", {6u, "pg_label"}},
-              {"PG Acronym", {7u, "pg_acronym"}},
-              {"PG Description", {8u, "pg_descr"}},
-              {"EDP", {9u, "edp"}},
-              {"DP", {10u, "dp"}},
-              {"PF", {11u, "pf"}},
-              {"PS", {12u, "ps"}},
-              {"PG Data Length", {15u, "pg_datalen"}},
-              {"Default Priority", {16u, "pg_priority"}},
-          },
 
-      spn_mapping_table = {
+  // Pre-build reverse lookup: column_index -> db_column_name
-          {"SPN", {19u, "spn"}},
+  std::map<size_t, std::string> pgn_col_to_db, spn_col_to_db;
-          {"SPN Name", {20u, "spn_name"}},
-          {"SPN Position in PG", {18u, "spn_pos"}},
-          {"SPN Length", {22u, "spn_length"}},
-          {"Offset", {24u, "offset"}},
-          {"Data Range", {25u, "data_range"}},
-          {"Resolution", {23u, "resolution"}},
-          {"Units", {27u, "units"}},
-          {"SLOT Identifier", {28u, "slot_id"}},
-          {"SLOT Name", {29u, "slot_name"}},
-          {"SPN Type", {30u, "spn_type"}},
-      };
 
-  std::map<size_t, std::string> pgn_headers, spn_headers;
   xlnt::workbook wb;
 
   if (xlnt::path(file).exists()) {
@@ -123,249 +34,48 @@ std::unique_ptr<sqlite::database> parseXlsx(const std::string &file) {
 
     wb.active_sheet(0);
   } else {
-
     throw std::runtime_error(fmt::format("File {} does not exists", file));
   }
 
   xlnt::worksheet ws = wb.active_sheet();
   for (const auto &col : ws.columns()) {
     if (auto [cell, value] = std::pair{col[0], col[0].to_string()}; !value.empty()) {
-      if (pgn_mapping_table.contains(value)) {
+      auto col_idx = cell.column_index();
-
+      if (auto it = pgn_mapping.find(value); it != pgn_mapping.end() && col_idx == std::get<0u>(it->second)) {
-        auto it = pgn_headers.insert_or_assign(cell.column_index(), value);
+        pgn_col_to_db[col_idx] = std::string(std::get<1u>(it->second));
-      } else if (spn_mapping_table.contains(value)) {
+      } else if (auto it2 = spn_mapping.find(value); it2 != spn_mapping.end() && col_idx == std::get<0u>(it2->second)) {
-
+        spn_col_to_db[col_idx] = std::string(std::get<1u>(it2->second));
-        auto it = spn_headers.insert_or_assign(cell.column_index(), value);
       }
     }
   }
 
+  const auto pgn_insert_sql = buildPgnInsertSql();
+  const auto spn_insert_sql = buildSpnInsertSql();
+
+  db << "BEGIN TRANSACTION";
+
   for (const auto &row : ws.rows()) {
     std::map<std::string, std::string> pgn_row_map, spn_row_map;
     for (const auto &cell : row) {
       if (cell.row() != 1u) {
+        auto col_idx = cell.column_index();
+        auto value = cell.to_string();
+        if (value.empty()) continue;
 
-        if (pgn_headers.contains(cell.column_index())) {
+        if (auto it = pgn_col_to_db.find(col_idx); it != pgn_col_to_db.end()) {
-          if (!cell.to_string().empty()) {
+          pgn_row_map[it->second] = std::move(value);
-            pgn_row_map.insert_or_assign(
+        } else if (auto it2 = spn_col_to_db.find(col_idx); it2 != spn_col_to_db.end()) {
-                [&]() {
+          spn_row_map[it2->second] = std::move(value);
-                  std::string ret;
-
-                  for (const auto &[k, v] : pgn_mapping_table) {
-                    if (k == pgn_headers[cell.column_index()] && cell.column_index() == std::get<0u>(v)) {
-
-                      ret = std::get<1u>(v);
-                    }
-                  }
-
-                  return ret;
-                }(),
-
-                cell.to_string());
-          }
-        } else if (spn_headers.contains(cell.column_index())) {
-          if (!cell.to_string().empty()) {
-            spn_row_map.insert_or_assign(
-                [&]() {
-                  std::string ret;
-
-                  for (const auto &[k, v] : spn_mapping_table) {
-                    if (k == spn_headers[cell.column_index()] && cell.column_index() == std::get<0u>(v)) {
-
-                      ret = std::get<1u>(v);
-                    }
-                  }
-
-                  return ret;
-                }(),
-
-                cell.to_string());
-          }
         }
       }
     }
 
-    // Add rows to database
     if (!pgn_row_map.empty() && !spn_row_map.empty()) {
-      for (const auto &[k, v] : pgn_row_map) {
+      insertJ1939Row(db, pgn_insert_sql, spn_insert_sql, pgn_row_map, spn_row_map);
-        try {
-          auto ps = db << fmt::format(
-                        "INSERT OR REPLACE INTO pgns ({}) VALUES ({})",
-                        [&]() {
-                          std::string ret;
-
-                          for (const auto &[k, v] : pgn_mapping_table) {
-                            auto end = pgn_mapping_table.end();
-                            bool is_last = k == (--end)->first;
-                            ret += fmt::format("{}{}", std::get<1u>(v), !is_last ? ", " : "");
-                          }
-
-                          return ret;
-                        }(),
-
-                        [&]() {
-                          std::string ret;
-                          for (const auto &[k, _] : pgn_mapping_table) {
-                            auto end = pgn_mapping_table.end();
-                            bool is_last = k == (--end)->first;
-                            ret += !is_last ? "?, " : "?";
-                          }
-
-                          return ret;
-                        }());
-
-          for (const auto &[k, v] : pgn_mapping_table) {
-            ps << pgn_row_map[std::get<1u>(v).data()];
-          }
-
-          ps.execute();
-        } catch (const sqlite::sqlite_exception &e) {
-          if (e.get_extended_code() == SQLITE_CONSTRAINT_UNIQUE) {
-            continue;
-          }
-
-          throw;
-        }
-      }
-
-      struct {
-        bool parts_inserted_flag = false;
-        double min = 0.0, max = 0.0;
-        size_t size_bits = 0u;
-      } spn_settings_calculated;
-
-      // Pre-compute size_bits before the main loop (resolution calculation depends on it)
-      {
-        auto it = spn_mapping_table.find("SPN Length");
-        if (it != spn_mapping_table.end()) {
-          auto size = parsers::parseSpnSize(spn_row_map[std::get<1u>(it->second).data()]);
-          spn_settings_calculated.size_bits = size.has_value() ? size.value().size_bits : 0u;
-        }
-      }
-
-      for (const auto &[k, v] : spn_row_map) {
-        try {
-          auto ps = db << fmt::format(
-                        "INSERT OR REPLACE INTO spns ({}) VALUES ({})",
-                        [&]() {
-                          std::string ret;
-
-                          ret += "pgn, ";
-                          for (const auto &[k, v] : spn_mapping_table) {
-                            auto end = spn_mapping_table.end();
-                            bool is_last = k == (--end)->first;
-
-                            // Split 'data range' to two columns 'min' and 'max'
-                            if (std::get<1u>(v) == "data_range") {
-
-                              ret += is_last ? ", min_value, max_value" : "min_value, max_value, ";
-                            } else {
-
-                              ret += fmt::format("{}{}", std::get<1u>(v), !is_last ? ", " : "");
-                            }
-                          }
-
-                          return ret;
-                        }(),
-
-                        [&]() {
-                          std::string ret;
-
-                          ret += "?, ";
-                          for (const auto &[k, v] : spn_mapping_table) {
-                            auto end = spn_mapping_table.end();
-                            bool is_last = k == (--end)->first;
-
-                            // Split 'data range' to two columns 'min' and 'max'
-                            if (std::get<1u>(v) == "data_range") {
-
-                              ret += is_last ? ", ?, ?" : "?, ?, ";
-                            } else {
-
-                              ret += !is_last ? "?, " : "?";
-                            }
-                          }
-
-                          return ret;
-                        }());
-
-          ps << pgn_row_map["pgn"];
-          for (const auto &[k, v] : spn_mapping_table) {
-            if (std::get<1u>(v) == "data_range") { // Parse 'data range' string and split it to two columns 'min' and 'max'
-
-              auto range = parsers::parseSpnDataRange(spn_row_map[std::get<1u>(v).data()]);
-              if (range.has_value()) {
-                (spn_settings_calculated.min = range.value().min, spn_settings_calculated.max = range.value().max);
-
-                for (const auto &val : {range.value().min, range.value().max}) {
-                  ps << val;
-                }
-              } else {
-                ps << nullptr << nullptr;
-              }
-            } else if (std::get<1u>(v) == "offset") {
-
-              auto offset = parsers::parseSpnOffset(spn_row_map[std::get<1u>(v).data()]);
-              if (offset.has_value()) {
-                ps << offset.value().offset;
-              } else {
-                ps << nullptr;
-              }
-            } else if (std::get<1u>(v) == "spn_length") {
-
-              auto size = parsers::parseSpnSize(spn_row_map[std::get<1u>(v).data()]);
-              spn_settings_calculated.size_bits = size.has_value() ? size.value().size_bits : 0u;
-              ps << (size.has_value() ? size.value().size_bits : 0u);
-            } else if (std::get<1u>(v) == "resolution") {
-              double calculated = (spn_settings_calculated.max - spn_settings_calculated.min) / (std::pow(2.0f, spn_settings_calculated.size_bits) - 1u);
-
-              // If is discrete -- use calculated resolution
-              if (std::fabs(calculated - 1.0) < 1e-9) {
-
-                ps << calculated;
-              } else {
-
-                auto resolution = parsers::parseSpnResolution(spn_row_map[std::get<1u>(v).data()]);
-                ps << (resolution.has_value() ? resolution.value().resolution : 1.0f);
-              }
-            } else if (std::get<1u>(v) == "spn_pos") {
-              ps << spn_row_map[std::get<1u>(v).data()];
-            } else {
-
-              ps << spn_row_map[std::get<1u>(v).data()];
-            }
-          }
-
-          ps.execute();
-
-          // Insert SPN fragments after successful spns INSERT
-          if (!spn_settings_calculated.parts_inserted_flag) {
-            auto size = parsers::parseSpnSize(spn_row_map[std::get<1u>(spn_mapping_table.at("SPN Length")).data()]);
-
-            if (size.has_value()) {
-              auto spn_fragments = parsers::parseSpnPosition(size.value().size_bits, spn_row_map[std::get<1u>(spn_mapping_table.at("SPN Position in PG")).data()]);
-
-              if (spn_fragments.has_value()) {
-                auto spn = std::stoll(spn_row_map["spn"]);
-                for (const auto &part : spn_fragments.value().spn_fragments) {
-                  db << R"(INSERT OR REPLACE INTO spn_fragments (spn, pgn, byte_offset, bit_offset, size) VALUES (?, ?, ?, ?, ?);)" << spn << std::stoll(pgn_row_map["pgn"])
-                     << part.byte_offset << part.bit_offset << part.size;
-                }
-
-                spn_settings_calculated.parts_inserted_flag = true;
-              }
-            }
-          }
-        } catch (const sqlite::sqlite_exception &e) {
-          if (e.get_extended_code() == SQLITE_CONSTRAINT_UNIQUE) {
-            continue;
-          }
-
-          throw;
-        }
-      }
     }
   }
 
+  db << "COMMIT";
+
   return db_ptr;
 }