harden candump parser, detect bus errors, classify SPN value encoding

lely-core CANopen dependency, SPN bit highlighting

CMakeLists.txt

@@ -15,9 +15,23 @@ option(BUILD_SHARED_LIBS "Build using shared libraries" OFF)
 include(cmake/dependencies.cmake)
 
 project(canscope)
-find_package(Boost REQUIRED COMPONENTS regex)
-file(GLOB_RECURSE SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/*.hpp ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp)
+include(cmake/lely.cmake)
 
+if(NOT BUILD_SHARED_LIBS)
+  set(Boost_USE_STATIC_LIBS ON)
+endif()
+
+find_package(Boost REQUIRED COMPONENTS regex)
+
+# Strip ICU dependencies from Boost::regex for static builds — ICU static libs
+# are not available on Manjaro and we don't use ICU-dependent regex features.
+if(NOT BUILD_SHARED_LIBS AND TARGET Boost::regex)
+  set_target_properties(Boost::regex PROPERTIES
+    INTERFACE_LINK_LIBRARIES ""
+  )
+endif()
+
+file(GLOB_RECURSE SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/*.hpp ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp)
 add_executable(${CMAKE_PROJECT_NAME} ${SOURCES})
 target_include_directories(${CMAKE_PROJECT_NAME} PRIVATE
   ${CMAKE_CURRENT_SOURCE_DIR}
@@ -31,5 +45,9 @@ target_include_directories(${CMAKE_PROJECT_NAME} PRIVATE
   ${clipp_SOURCE_DIR}/include
 )
 
-target_link_libraries(${CMAKE_PROJECT_NAME} PRIVATE ftxui::component ftxui::screen ftxui::dom tiny-process-library xlnt sqlite3_lib z Boost::regex ${Boost_LIBRARIES})
+target_compile_definitions(${CMAKE_PROJECT_NAME} PRIVATE
+  CANBOAT_JSON_PATH="${canboat_SOURCE_DIR}/docs/canboat.json"
+)
+
+target_link_libraries(${CMAKE_PROJECT_NAME} PRIVATE ftxui::component ftxui::screen ftxui::dom tiny-process-library xlnt sqlite3_lib z Boost::regex ${Boost_LIBRARIES} lely::coapp lely::co lely::can lely::io2 lely::ev lely::util)
 

README.md

@@ -1,6 +1,6 @@
 # {canscope}
 
-CAN bus sniffer and SAE J1939 protocol analyzer. Reads CAN frames in `candump` format, decodes them using a J1939 Digital Annex (xlsx), and presents results in an interactive terminal UI or as JSON output.
+CAN bus sniffer and SAE J1939 protocol analyzer. Reads CAN frames in `candump` format, decodes them using a J1939 Digital Annex (xlsx or csv), and presents results in an interactive terminal UI or as JSON output.
 
 ![demo](canscope-demo.gif)
 
@@ -9,7 +9,7 @@ CAN bus sniffer and SAE J1939 protocol analyzer. Reads CAN frames in `candump` f
 - **TUI mode** - full-screen interactive terminal interface (FTXUI). Multiple display modes per CAN ID: deployed, brief, verbose, manual, little-endian
 - **Headless mode** - JSON output to stdout or file, for scripting and automation
 - **Recording** - decoded J1939 SPN values saved to SQLite database with gzip compression and batch flushing
-- **J1939 decoding** - PGN/SPN lookup, bit-level value extraction from payload
+- **J1939 decoding** - PGN/SPN lookup, bit-level value extraction from payload. Supports xlsx and csv input formats
 - **CAN playback** - replay recorded CAN frames
 - **Custom SPN configuration** - per-parameter settings, parameter export
 - **Real-time** - 30 fps UI refresh
@@ -20,7 +20,7 @@ CAN bus sniffer and SAE J1939 protocol analyzer. Reads CAN frames in `candump` f
 - clang++ with C++23 support
 - CMake >= 3.13
 - Ninja
-- System libraries: boost (signals2, spirit, phoenix), sqlite3, zlib
+- System libraries: boost (signals2, spirit, phoenix, regex), sqlite3, zlib, icu
 
 Dependencies fetched automatically via CMake FetchContent:
 
@@ -28,6 +28,7 @@ Dependencies fetched automatically via CMake FetchContent:
 - [tiny-process-library](https://gitlab.com/eidheim/tiny-process-library) - subprocess management
 - [sqlite_modern_cpp](https://github.com/SqliteModernCpp/sqlite_modern_cpp) - modern C++ SQLite wrapper
 - [xlnt](https://github.com/xlnt-community/xlnt) - xlsx reading
+- [lely-core](https://gitlab.com/lely_industries/lely-core) - CANopen protocol stack
 - [fmt](https://github.com/fmtlib/fmt) - text formatting
 - [nlohmann/json](https://github.com/nlohmann/json) - JSON library
 - [clipp](https://github.com/muellan/clipp) - CLI argument parsing
@@ -53,7 +54,9 @@ make clean             # Remove all build artifacts
 
 ```bash
 make build
-./build/native/canscope -e "candump can0" -j1939 thirdparty/j1939da_2018.xlsx
+./build/native/canscope -e "candump can0" -j1939-xlsx thirdparty/j1939da_2018.xlsx
+# or with CSV (faster parsing)
+./build/native/canscope -e "candump can0" -j1939-csv thirdparty/j1939da_2018.csv
 ```
 
 ### Docker (cross-platform)
@@ -62,13 +65,16 @@ Works on Linux, macOS (?), and Windows (?). Requires only Docker and Make.
 
 ```bash
 # TUI mode - local CAN interface
-make docker-run ARGS='-e "candump can0" -j1939 thirdparty/j1939da_2018.xlsx'
+make docker-run ARGS='-e "candump can0" -j1939-xlsx thirdparty/j1939da_2018.xlsx'
 
 # TUI mode - remote CAN interface via SSH (no data if will ask password - use public key access or sshpass utility)
-make docker-run ARGS='-e "ssh user@remote candump can0" -j1939 thirdparty/j1939da_2018.xlsx'
+make docker-run ARGS='-e "ssh user@remote candump can0" -j1939-xlsx thirdparty/j1939da_2018.xlsx'
 
-# Headless mode - create report about collected PGNs and SPNs
-make docker-run ARGS='-hl -e "candump can0" -j1939 thirdparty/j1939da_2018.xlsx -of output.json'
+# Discover mode - find out what PGNs/SPNs are on the bus
+make docker-run ARGS='-discover -e "candump can0" -j1939-xlsx thirdparty/j1939da_2018.xlsx'
+
+# Headless mode - stream decoded values
+make docker-run ARGS='-hl -e "candump can0" -j1939-xlsx thirdparty/j1939da_2018.xlsx'
 ```
 
 ### Cross-compile for arm64
@@ -82,42 +88,53 @@ Requires Docker. SSH keys from `~/.ssh` and `/etc/hosts` are forwarded into the
 
 ## Usage
 
+All operating modes are mutually exclusive. If none is specified, TUI mode is used.
+
 ```bash
-# TUI mode (default)
-canscope -e "candump can0" -j1939 thirdparty/j1939da_2018.xlsx
+# TUI mode (default) — interactive terminal interface
+canscope -e "candump can0" -j1939-xlsx thirdparty/j1939da_2018.xlsx
 
-# Headless - JSON to stdout
-canscope -hl -e "candump can0" -j1939 thirdparty/j1939da_2018.xlsx
+# Discover mode — output PGN/SPN structure (no values) to stdout or file
+canscope -discover -e "candump can0" -j1939-xlsx thirdparty/j1939da_2018.xlsx
+canscope -discover -of discovered.json -e "candump can0" -j1939-csv thirdparty/j1939da_2018.csv
 
-# Headless - JSON to file
-canscope -hl -e "candump can0" -j1939 thirdparty/j1939da_2018.xlsx -of output.json
+# Headless mode — stream all decoded values (NDJSON) to stdout
+canscope -hl -e "candump can0" -j1939-xlsx thirdparty/j1939da_2018.xlsx
+
+# Record mode — write all decoded values + timestamps to SQLite
+canscope -rec -db recording.db -e "candump can0" -j1939-xlsx thirdparty/j1939da_2018.xlsx
 
 # Read from stdin (pipe)
-candump can0 | canscope -j1939 thirdparty/j1939da_2018.xlsx
-
-# Record to SQLite database
-canscope -rec -db recording.db -e "candump can0" -j1939 thirdparty/j1939da_2018.xlsx
-
-# Record + TUI
-canscope -rec -db recording.db -tui -e "candump can0" -j1939 thirdparty/j1939da_2018.xlsx
+candump can0 | canscope -j1939-csv thirdparty/j1939da_2018.csv
 ```
 
 > **Note:** J1939 decoding has only been tested with the Digital Annex 2018 edition. Other editions may work but are not guaranteed.
 
+### Modes
+
+| Mode | Flag | Description |
+|------|------|-------------|
+| TUI | *(default)* | Interactive full-screen terminal UI |
+| Discover | `-discover` | Output PGN/SPN structure (no values) to stdout or file (`-of`) |
+| Headless | `-hl` | Stream all decoded PGN/SPN values as NDJSON to stdout |
+| Record | `-rec` | Write all decoded PGN/SPN values + timestamps to SQLite (`-db`) |
+
 ### CLI flags
 
 | Flag | Long form | Description |
 |------|-----------|-------------|
-| `-j1939` | `--j1939-document` | **(required)** J1939 Digital Annex xlsx file |
+| `-j1939-xlsx` | | J1939 Digital Annex xlsx file |
+| `-j1939-csv` | | J1939 Digital Annex csv file (faster parsing) |
 | `-e` | `--execute-command` | Command to read CAN frames from (e.g. `"candump can0"`) |
-| `-hl` | `--headless` | Headless mode (no TUI) |
-| `-of` | `--output-file` | Output file path (headless mode) |
-| `-rec` | `--record` | Record decoded values to SQLite |
+| `-discover` | | Discover mode |
+| `-hl` | `--headless` | Headless mode |
+| `-rec` | `--record` | Record mode |
+| `-of` | `--output-file` | Output file path (used with `-discover`) |
 | `-db` | `--database` | SQLite database path (required with `-rec`) |
-| `-tui` | | Show TUI alongside recording |
 | `-h` | `--help` | Show help |
 
 ## Roadmap
 
-- **CANopen protocol support** - CANopen decoding alongside J1939
+- **NMEA 2000 protocol support** - NMEA 2000 decoding using [canboat](https://github.com/canboat/canboat) PGN database (JSON). Same 29-bit CAN ID as J1939, requires Fast Packet protocol implementation
+- **CANopen protocol support** - CANopen decoding alongside J1939 (11-bit CAN ID, SDO/PDO/NMT)
 - **Other small features and enhancements** - UI improvements, performance optimizations, additional export formats

cmake/dependencies.cmake

@@ -112,3 +112,29 @@ FetchContent_GetProperties(clipp)
 if(NOT clipp_POPULATED)
   FetchContent_Populate(clipp)
 endif()
+
+FetchContent_Declare(
+  lely_core
+  GIT_REPOSITORY https://gitlab.com/lely_industries/lely-core.git
+  GIT_TAG        v2.3.5
+  GIT_SHALLOW    TRUE
+  GIT_PROGRESS   TRUE
+)
+
+FetchContent_Declare(
+  canboat
+  GIT_REPOSITORY https://github.com/canboat/canboat.git
+  GIT_TAG        v6.1.6
+  GIT_SHALLOW    TRUE
+  GIT_PROGRESS   TRUE
+)
+
+FetchContent_GetProperties(canboat)
+if(NOT canboat_POPULATED)
+  FetchContent_Populate(canboat)
+endif()
+
+FetchContent_GetProperties(lely_core)
+if(NOT lely_core_POPULATED)
+  FetchContent_Populate(lely_core)
+endif()

cmake/lely.cmake

@@ -0,0 +1,69 @@
+set(LELY_INSTALL_DIR ${CMAKE_BINARY_DIR}/lely-install)
+file(MAKE_DIRECTORY ${LELY_INSTALL_DIR}/include)
+
+if(BUILD_SHARED_LIBS)
+  set(LELY_SHARED_FLAG "--enable-shared;--disable-static")
+  set(LELY_LIB_TYPE SHARED)
+  set(LELY_LIB_SUFFIX .so)
+else()
+  set(LELY_SHARED_FLAG "--disable-shared;--enable-static")
+  set(LELY_LIB_TYPE STATIC)
+  set(LELY_LIB_SUFFIX .a)
+endif()
+
+set(LELY_CFLAGS "-Wno-error -Wno-implicit-function-declaration -Wno-unterminated-string-initialization -DATOMIC_VAR_INIT=")
+set(LELY_CXXFLAGS "-Wno-error")
+
+if(CMAKE_CROSSCOMPILING)
+  set(LELY_HOST_FLAG "--host=${CMAKE_C_COMPILER_TARGET}")
+  set(LELY_CROSS_FLAGS "--target=${CMAKE_C_COMPILER_TARGET}")
+  if(CMAKE_SYSROOT)
+    string(APPEND LELY_CROSS_FLAGS " --sysroot=${CMAKE_SYSROOT}")
+  endif()
+  set(LELY_CC "${CMAKE_C_COMPILER} ${LELY_CROSS_FLAGS}")
+  set(LELY_CXX "${CMAKE_CXX_COMPILER} ${LELY_CROSS_FLAGS}")
+else()
+  set(LELY_HOST_FLAG "")
+  set(LELY_CC "")
+  set(LELY_CXX "")
+endif()
+
+include(ExternalProject)
+set(LELY_BYPRODUCTS)
+foreach(_lib can co coapp util ev io2)
+  list(APPEND LELY_BYPRODUCTS ${LELY_INSTALL_DIR}/lib/liblely-${_lib}${LELY_LIB_SUFFIX})
+endforeach()
+
+set(LELY_ENV_VARS "CFLAGS=${LELY_CFLAGS}" "CXXFLAGS=${LELY_CXXFLAGS}")
+if(LELY_CC)
+  list(APPEND LELY_ENV_VARS "CC=${LELY_CC}" "CXX=${LELY_CXX}" "LDFLAGS=-fuse-ld=lld")
+endif()
+
+ExternalProject_Add(lely_core_ext
+  SOURCE_DIR        ${lely_core_SOURCE_DIR}
+  BUILD_IN_SOURCE   TRUE
+  CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${LELY_ENV_VARS}
+                    autoreconf -i ${lely_core_SOURCE_DIR}
+            COMMAND ${CMAKE_COMMAND} -E env ${LELY_ENV_VARS}
+                    ${lely_core_SOURCE_DIR}/configure
+                    --prefix=${LELY_INSTALL_DIR}
+                    ${LELY_HOST_FLAG}
+                    ${LELY_SHARED_FLAG}
+                    --disable-dependency-tracking
+                    --disable-python
+                    --disable-tests
+                    --disable-doc
+  BUILD_COMMAND     ${CMAKE_COMMAND} -E env ${LELY_ENV_VARS}
+                    make -j
+  INSTALL_COMMAND   make install
+  BUILD_BYPRODUCTS  ${LELY_BYPRODUCTS}
+)
+
+foreach(_lib can co coapp util ev io2)
+  add_library(lely::${_lib} ${LELY_LIB_TYPE} IMPORTED GLOBAL)
+  set_target_properties(lely::${_lib} PROPERTIES
+    IMPORTED_LOCATION ${LELY_INSTALL_DIR}/lib/liblely-${_lib}${LELY_LIB_SUFFIX}
+    INTERFACE_INCLUDE_DIRECTORIES ${LELY_INSTALL_DIR}/include
+  )
+  add_dependencies(lely::${_lib} lely_core_ext)
+endforeach()

docker/Dockerfile

@@ -5,8 +5,12 @@ RUN pacman -Syu --noconfirm && \
       clang \
       cmake \
       make \
+      autoconf \
+      automake \
+      libtool \
       git \
       boost \
+      icu \
       zlib && \
     pacman -Scc --noconfirm
 

docker/Dockerfile.cross

@@ -8,6 +8,10 @@ RUN pacman -Syu --noconfirm && \
       ninja \
       git \
       openssh \
+      autoconf \
+      automake \
+      libtool \
+      make \
     && pacman -Scc --noconfirm
 
 WORKDIR /app

docker/Dockerfile.dev

@@ -9,7 +9,12 @@ RUN pacman -Syu --noconfirm && \
       git \
       openssh \
       sshpass \
+      autoconf \
+      automake \
+      libtool \
+      make \
       boost \
+      icu \
       sqlite3 \
       zlib && \
     pacman -Scc --noconfirm

docker/Dockerfile.sysroot

@@ -3,6 +3,7 @@ FROM --platform=linux/arm64 manjarolinux/base:latest
 RUN pacman -Syu --noconfirm && \
     pacman -S --noconfirm \
       boost \
+      icu \
       zlib \
       gcc && \
     pacman -Scc --noconfirm

src/can_data.hpp

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <atomic>
 #include <cstdint>
 #include <memory>
 #include <mutex>
@@ -10,9 +11,14 @@
 // Mutex protecting the J1939 SQLite database from concurrent access
 extern std::mutex g_j1939_db_mtx;
 
+// Count of CAN error frames seen on the bus (SocketCAN ERRORFRAME markers)
+extern std::atomic<uint64_t> g_error_frame_count;
+
 struct can_frame_data_s {
   std::vector<uint8_t> payload;
   int32_t size = 0;
+  bool is_error_frame = false;
+  bool is_remote_request = false;
 };
 
 struct can_frame_diff_s {

src/can_frame.cpp

@@ -11,6 +11,7 @@
 std::pair<nlohmann::json, nlohmann::json> processFrame(sqlite::database &db, const std::string &iface,
                                                        const std::string &canid, const std::vector<uint8_t> &data) {
   nlohmann::json verbose, brief;
+
   db << R"(SELECT pgn, pg_label, pg_acronym, pg_descr, edp, dp, pf, ps, pg_datalen, pg_priority FROM pgns WHERE pgn = ?;)"
      << [&]() -> int32_t {
     int32_t ret;
@@ -29,11 +30,12 @@ std::pair<nlohmann::json, nlohmann::json> processFrame(sqlite::database &db, con
 
     // Get SPNs
     nlohmann::json::array_t spns_array;
-    db << R"(SELECT spn, spn_name, spn_pos, spn_length, resolution, offset, data_range, min_value, max_value, units, slot_id, slot_name, spn_type FROM spns WHERE pgn = ?;)"
+    db << R"(SELECT spn, spn_name, spn_pos, spn_length, resolution, offset, data_range, min_value, max_value, units, slot_id, slot_name, spn_type, value_encoding FROM spns WHERE pgn = ?;)"
        << pgn_int >>
         [&](int32_t spn, const std::string &spn_name, const std::string &spn_pos, int32_t spn_length, double resolution,
             int32_t offset, const std::string &data_range, double min, double max, const std::string &unit,
-            const std::string &slot_id, const std::string &slot_name, const std::string &spn_type) {
+            const std::string &slot_id, const std::string &slot_name, const std::string &spn_type,
+            const std::string &value_encoding) {
           nlohmann::json spn_json = {
               {"SPN (integer)", spn},
               {"SPN name", spn_name},
@@ -46,15 +48,33 @@ std::pair<nlohmann::json, nlohmann::json> processFrame(sqlite::database &db, con
               {"Unit", unit},
               {"SLOT id", slot_id},
               {"SPN type", spn_type},
+              {"Encoding", value_encoding},
           };
 
           // Get parts
           size_t result = 0u, iter = 0u, total_size_bits = 0u;
+          std::vector<uint8_t> ascii_bytes;
           nlohmann::json::array_t parts_array;
+
           db << "SELECT byte_offset,bit_offset,size FROM spn_fragments WHERE spn = ? AND pgn = ?" << spn << pgn >>
               [&, byte_array = data](int32_t byte_offset, int32_t bit_offset, int32_t size_bits) {
+                if (value_encoding == "ascii") {
+                  // ASCII SPNs are byte-aligned; collect raw bytes directly — size may exceed 64 bits.
+                  const size_t nbytes = size_bits / UINT8_WIDTH;
+
+                  for (size_t i = 0; i < nbytes && (byte_offset + i) < byte_array.size(); ++i) {
+                    ascii_bytes.push_back(byte_array[byte_offset + i]);
+                  }
+
+                  parts_array.push_back(nlohmann::json::parse(fmt::format(
+                      R"({{"{}":{{"byte_offset":{},"bit_offset":{},"size_bits":{},"parse_result":"ascii"}}}})",
+                      fmt::format("Fragment#{}", iter++), byte_offset, bit_offset, size_bits)));
+                  return;
+                }
+
                 result <<= size_bits;
                 total_size_bits += size_bits;
+
                 for (uint32_t i = 0; i < ((size_bits / UINT8_WIDTH) + (size_bits % UINT8_WIDTH ? 1 : 0)); ++i) {
                   const uint8_t &byte = byte_array[byte_offset + i];
                   result |= (((byte << (i * UINT8_WIDTH)) &
@@ -72,9 +92,20 @@ std::pair<nlohmann::json, nlohmann::json> processFrame(sqlite::database &db, con
                                 fmt::format("{:#x}", result))));
               };
 
-          double result_real = result * resolution + offset;
           spn_json["Fragments"] = parts_array;
-          spn_json["Value"] = result_real;
+          if (value_encoding == "ascii") {
+            std::string text;
+            text.reserve(ascii_bytes.size());
+
+            for (uint8_t b : ascii_bytes) {
+              text += (b >= 0x20 && b < 0x7f) ? static_cast<char>(b) : '.';
+            }
+
+            spn_json["Value"] = text;
+          } else {
+            spn_json["Value"] = result * resolution + offset;
+          }
+
           spns_array.push_back(spn_json);
         };
 
@@ -90,8 +121,17 @@ std::pair<nlohmann::json, nlohmann::json> processFrame(sqlite::database &db, con
 
     nlohmann::json::array_t spns_array;
     for (const auto &spn : verbose["SPNs"]) {
-      spns_array.push_back(fmt::format("{}: {:.6g} {}", spn["SPN name"].get<std::string>(), spn["Value"].get<double>(),
-                                       spn["Unit"].get<std::string>()));
+      const auto name = spn["SPN name"].get<std::string>();
+      const auto unit = spn["Unit"].get<std::string>();
+      const auto encoding = spn.value("Encoding", std::string{"numeric"});
+
+      if (encoding == "ascii") {
+        spns_array.push_back(fmt::format("{}: \"{}\"", name, spn["Value"].get<std::string>()));
+      } else if (encoding == "binary") {
+        spns_array.push_back(fmt::format("{}: {}", name, spn["Value"].get<double>() != 0.0 ? "yes" : "no"));
+      } else {
+        spns_array.push_back(fmt::format("{}: {:.6g} {}", name, spn["Value"].get<double>(), unit));
+      }
     }
 
     brief["SPNs"] = spns_array;
@@ -103,26 +143,25 @@ std::pair<nlohmann::json, nlohmann::json> processFrame(sqlite::database &db, con
 nlohmann::json verboseToExportJson(const nlohmann::json &verbose) {
   nlohmann::json::array_t spns;
 
-  if (verbose.is_null() || !verbose.contains("SPNs")) return spns;
+  if (verbose.is_null() || !verbose.contains("SPNs"))
+    return spns;
 
   std::string pgn = verbose.contains("PGN") ? verbose["PGN"].get<std::string>() : "";
 
   for (const auto &v : verbose["SPNs"]) {
     nlohmann::json spn = {
-        {"name", v.value("SPN name", "")},
-        {"offset", v.value("Offset", 0)},
-        {"resolution", v.value("Resolution", 0.0)},
-        {"max", v.value("Maximum value", 0.0)},
-        {"min", v.value("Minimum value", 0.0)},
-        {"pgn", pgn},
-        {"value", v.value("Value", 0.0)},
-        {"unit", v.value("Unit", "")},
+        {"name", v.value("SPN name", "")},          {"offset", v.value("Offset", 0)},
+        {"resolution", v.value("Resolution", 0.0)}, {"max", v.value("Maximum value", 0.0)},
+        {"min", v.value("Minimum value", 0.0)},     {"pgn", pgn},
+        {"value", v.value("Value", 0.0)},           {"unit", v.value("Unit", "")},
     };
 
     nlohmann::json::array_t frags;
+
     if (v.contains("Fragments")) {
       for (const auto &[k, frag] : v["Fragments"].items()) {
         auto frag_key = fmt::format("Fragment#{}", k);
+
         if (frag.contains(frag_key)) {
           frags.push_back({{
               fmt::format("fragment#{}", k),

src/can_player_dialog.cpp

@@ -29,6 +29,8 @@
 #include <linux/can.h>
 #include <linux/can/raw.h>
 
+#include <boost/regex.hpp>
+
 #define FMT_HEADER_ONLY
 #include <fmt/format.h>
 
@@ -45,6 +47,7 @@ ftxui::Component makeCanPlayerDialog(ftxui::ScreenInteractive *scr, signals_map_
     Impl(ftxui::ScreenInteractive *scr, signals_map_t &smap, bool &is_ready) {
       static sqlite::database *database = nullptr;
       static float canbus_player_focus_relative = 0;
+      static std::string player_filter_text;
 
       auto pgnContainer = ftxui::Container::Vertical({});
 
@@ -192,12 +195,14 @@ ftxui::Component makeCanPlayerDialog(ftxui::ScreenInteractive *scr, signals_map_
                 ->connect([](const std::vector<can_frame_update_s> &batch) {
                   for (const auto &entry : batch) {
                     uint32_t pgn_num = 0;
+
                     if (entry.canid.size() >= 6) {
                       auto pgn_str = entry.canid.substr(entry.canid.size() >= 8 ? entry.canid.size() - 6 : 2, 4);
                       std::stringstream ss;
                       ss << std::hex << pgn_str;
                       ss >> pgn_num;
                     }
+
                     received_pgns.insert(pgn_num);
                     if (pgs.contains(pgn_num) && pgs[pgn_num].forward && pgs[pgn_num].is_running) {
                       auto &pg = pgs[pgn_num];
@@ -405,6 +410,7 @@ ftxui::Component makeCanPlayerDialog(ftxui::ScreenInteractive *scr, signals_map_
                                                                   {"offset", spn_params.offset},
                                                               };
                                                             }(),
+
                                                             false, -100, ExpanderImpl::Root()) |
                                                             ftxui::Renderer([](ftxui::Element inner) {
                                                               return ftxui::hbox({
@@ -433,21 +439,52 @@ ftxui::Component makeCanPlayerDialog(ftxui::ScreenInteractive *scr, signals_map_
                                                                       fmt::format("{0:#b}", spn_params.raw))),
                                                               }),
 
-                                                              ftxui::hbox({
-                                                                  ftxui::text("PG payload: ") | ftxui::bold |
-                                                                      ftxui::color(ftxui::Color::Cyan),
-                                                                  ftxui::text(fmt::format(
-                                                                      "[{}]",
-                                                                      [&]() {
-                                                                        std::string ret;
-                                                                        for (const auto &byte :
-                                                                             spn_params.pg_ref->payload) {
-                                                                          ret += fmt::format("{0:#010b} ", byte);
-                                                                        }
+                                                              [&]() -> ftxui::Element {
+                                                                // Build bit mask for this SPN's fragments
+                                                                const auto &payload = spn_params.pg_ref->payload;
+                                                                std::vector<bool> highlight(payload.size() * 8, false);
+                                                                for (const auto &frag : spn_params.fragments) {
+                                                                  int32_t start_bit =
+                                                                      frag.byte_offset * 8 + frag.bit_offset;
 
-                                                                        return ret;
-                                                                      }())),
-                                                              }),
+                                                                  for (int32_t b = 0; b < frag.size; ++b) {
+                                                                    auto idx = static_cast<size_t>(start_bit + b);
+
+                                                                    if (idx < highlight.size()) {
+                                                                      highlight[idx] = true;
+                                                                    }
+                                                                  }
+                                                                }
+
+                                                                ftxui::Elements parts;
+                                                                parts.push_back(ftxui::text("PG payload: ") |
+                                                                                ftxui::bold |
+                                                                                ftxui::color(ftxui::Color::Cyan));
+                                                                parts.push_back(ftxui::text("["));
+
+                                                                for (size_t i = 0; i < payload.size(); ++i) {
+                                                                  parts.push_back(ftxui::text("0b"));
+
+                                                                  for (int32_t bit = 7; bit >= 0; --bit) {
+                                                                    bool is_set = (payload[i] >> bit) & 1;
+                                                                    bool is_spn = highlight[i * 8 + bit];
+                                                                    auto ch = ftxui::text(is_set ? "1" : "0");
+
+                                                                    if (is_spn) {
+                                                                      ch = ch | ftxui::color(ftxui::Color::Red) |
+                                                                           ftxui::bold;
+                                                                    }
+
+                                                                    parts.push_back(ch);
+                                                                  }
+
+                                                                  parts.push_back(ftxui::text(" "));
+                                                                }
+
+                                                                parts.push_back(ftxui::text("]"));
+
+                                                                return ftxui::hbox(std::move(parts));
+                                                              }(),
 
                                                               ftxui::separatorEmpty(),
                                                           });
@@ -461,7 +498,7 @@ ftxui::Component makeCanPlayerDialog(ftxui::ScreenInteractive *scr, signals_map_
                                 });
                               };
 
-                          pgnContainer->Add(ftxui::Container::Vertical({
+                          auto pgn_entry = ftxui::Container::Vertical({
                               ftxui::Container::Horizontal({
                                   ftxui::Checkbox({
                                       .checked = &pg_ref.selected,
@@ -621,6 +658,18 @@ ftxui::Component makeCanPlayerDialog(ftxui::ScreenInteractive *scr, signals_map_
                                       spnContainer,
                                   }) | ftxui::border,
                                   &pg_ref.selected),
+                          });
+
+                          pgnContainer->Add(ftxui::Maybe(pgn_entry, [&pg_ref = pgs[pgn]]() -> bool {
+                            if (player_filter_text.empty())
+                              return true;
+                            try {
+                              boost::regex re(player_filter_text, boost::regex_constants::icase);
+                              std::string subject = fmt::format("0x{:x} {}", pg_ref.pgn, pg_ref.label);
+                              return boost::regex_search(subject, re);
+                            } catch (...) {
+                              return true;
+                            }
                           }));
                         };
 
@@ -634,57 +683,106 @@ ftxui::Component makeCanPlayerDialog(ftxui::ScreenInteractive *scr, signals_map_
       }
 
       auto main = ftxui::Container::Vertical({
+          ftxui::Input({
+              .content = &player_filter_text,
+              .placeholder = "regex filter ...",
+              .transform = [](ftxui::InputState state) -> ftxui::Element {
+                bool valid = true;
+                if (!player_filter_text.empty()) {
+                  try {
+                    boost::regex(player_filter_text, boost::regex_constants::icase);
+                  } catch (...) {
+                    valid = false;
+                  }
+                }
+
+                state.element |= (!valid ? ftxui::color(ftxui::Color::Red) : ftxui::nothing) |
+                                 (state.focused ? ftxui::color(ftxui::Color::Cyan) : ftxui::nothing) |
+                                 (state.hovered ? ftxui::bold : ftxui::nothing);
+
+                return ftxui::hbox({
+                    ftxui::text(" Search: [ "),
+                    state.element |
+                        (state.hovered || state.focused ? ftxui::bgcolor(ftxui::Color::Grey11) : ftxui::nothing) |
+                        ftxui::xflex,
+                    ftxui::text(" ]"),
+                });
+              },
+              .multiline = false,
+          }),
+
+          ftxui::Renderer([]() { return ftxui::separator(); }),
+
           (pgnContainer | ftxui::Renderer([](ftxui::Element inner) {
              return inner | ftxui::focusPositionRelative(0, canbus_player_focus_relative) | ftxui::vscroll_indicator |
                     ftxui::frame | ftxui::flex;
-           })) |
-              ftxui::CatchEvent([pgnContainer](ftxui::Event event) {
-                static const auto increment_focus = [pgnContainer = pgnContainer]() {
-                  if (auto n = pgnContainer->ChildCount(); n > 0)
-                    canbus_player_focus_relative =
-                        std::clamp(canbus_player_focus_relative + 1.0f / static_cast<float>(n), 0.0f, 1.0f);
-                };
-
-                static const auto decrement_focus = [pgnContainer = pgnContainer]() {
-                  if (auto n = pgnContainer->ChildCount(); n > 0)
-                    canbus_player_focus_relative =
-                        std::clamp(canbus_player_focus_relative - 1.0f / static_cast<float>(n), 0.0f, 1.0f);
-                };
-
-                if (!database) {
-                  return true;
-                }
-
-                if (event.is_mouse()) {
-                  switch (static_cast<enum ftxui::Mouse::Button>(event.mouse().button)) {
-                  case ftxui::Mouse::Button::WheelDown: {
-                    increment_focus();
-                    return true;
-                  } break;
-
-                  case ftxui::Mouse::Button::WheelUp: {
-                    decrement_focus();
-                    return true;
-                  } break;
-
-                  default:
-                    break;
-                  }
-                } else if (!event.is_character()) {
-                  if (event == ftxui::Event::ArrowDown) {
-                    increment_focus();
-                    return true;
-                  } else if (event == ftxui::Event::ArrowUp) {
-                    decrement_focus();
-                    return true;
-                  }
-                }
-
-                return false;
-              }),
+           })),
       });
 
-      Add({main});
+      Add({main | ftxui::CatchEvent([pgnContainer](ftxui::Event event) {
+             if (!database)
+               return true;
+
+             const auto scroll_step = []() -> float {
+               size_t visible_lines = 0;
+               boost::regex re;
+               bool has_filter = !player_filter_text.empty();
+
+               if (has_filter) {
+                 try {
+                   re = boost::regex(player_filter_text, boost::regex_constants::icase);
+                 } catch (...) {
+                   has_filter = false;
+                 }
+               }
+
+               for (const auto &[_, pg] : pgs) {
+                 if (has_filter) {
+                   std::string subject = fmt::format("0x{:x} {}", pg.pgn, pg.label);
+                   if (!boost::regex_search(subject, re)) {
+                     continue;
+                   }
+                 }
+
+                 ++visible_lines;
+
+                 if (pg.selected) {
+                   visible_lines += 15;
+                 }
+               }
+
+               return visible_lines > 0 ? 1.0f / static_cast<float>(visible_lines) : 0.03f;
+             };
+
+             if (event.is_mouse()) {
+               switch (static_cast<enum ftxui::Mouse::Button>(event.mouse().button)) {
+               case ftxui::Mouse::Button::WheelDown: {
+                 canbus_player_focus_relative = std::clamp(canbus_player_focus_relative + scroll_step(), 0.0f, 1.0f);
+                 return true;
+               }
+
+               case ftxui::Mouse::Button::WheelUp: {
+                 canbus_player_focus_relative = std::clamp(canbus_player_focus_relative - scroll_step(), 0.0f, 1.0f);
+                 return true;
+               }
+
+               default:
+                 break;
+               }
+             } else if (!event.is_character()) {
+               if (event == ftxui::Event::ArrowDown) {
+
+                 canbus_player_focus_relative = std::clamp(canbus_player_focus_relative + scroll_step(), 0.0f, 1.0f);
+                 return true;
+               } else if (event == ftxui::Event::ArrowUp) {
+
+                 canbus_player_focus_relative = std::clamp(canbus_player_focus_relative - scroll_step(), 0.0f, 1.0f);
+                 return true;
+               }
+             }
+
+             return false;
+           })});
     }
   };
 

src/canid_unit.cpp

@@ -1,5 +1,6 @@
 // #include <algorithm>
 // #include <cmath>
+#include <boost/regex.hpp>
 #include <cstdint>
 #include <ftxui/component/component.hpp>
 #include <ftxui/component/component_options.hpp>
@@ -330,16 +331,19 @@ void CanIDUnit::update(const can_frame_data_s &data, const can_frame_diff_s &dif
           total_bits += bc;
         }
       }
+
       if (settings.big_endian && total_bits > 8) {
         int64_t swapped = 0;
         size_t total_bytes = (total_bits + 7) / 8;
+
         for (size_t i = 0; i < total_bytes; ++i)
           swapped |= ((result >> (i * 8)) & 0xFF) << ((total_bytes - 1 - i) * 8);
         result = swapped;
       }
-      double spn_val = static_cast<double>(result) * resolution + offset_val;
 
+      double spn_val = static_cast<double>(result) * resolution + offset_val;
       nlohmann::json::array_t frags_json;
+
       for (size_t fi = 0; fi < settings.fragments.size(); ++fi) {
         const auto &f = settings.fragments[fi];
         int32_t bo = 0, bi = 0, bc = 0;
@@ -424,8 +428,10 @@ void CanIDUnit::update(const can_frame_data_s &data, const can_frame_diff_s &dif
             for (const auto &[spn_k, spn_v] : spns_arr_v.items()) {
               if (spn_k == "SPN name") {
                 std::string spn_name = spn_v.get<std::string>();
-                if (spn_name.find("(custom)") != std::string::npos)
+                if (spn_name.find("(custom)") != std::string::npos) {
                   continue;
+                }
+
                 auto &spn_map = std::get<2u>(s_canbus_parameters_export_map_[m_canid_]);
 
                 spn_map.insert_or_assign(spn_name, std::make_tuple(false, false, spns_arr_v));
@@ -529,7 +535,19 @@ void CanIDUnit::update(const can_frame_data_s &data, const can_frame_diff_s &dif
                                            !selectors_container->ChildCount();
                                   }));
 
-      m_cansettings_dialog_->Add(container);
+      m_cansettings_dialog_->Add(ftxui::Maybe(container, [this]() -> bool {
+        if (!s_export_filter_text_ || s_export_filter_text_->empty()) {
+          return true;
+        }
+
+        try {
+          boost::regex re(*s_export_filter_text_, boost::regex_constants::icase);
+          std::string subject = m_canid_ + " " + getLabel();
+          return boost::regex_search(subject, re);
+        } catch (...) {
+          return true;
+        }
+      }));
     }
 
     // Add custom SPNs to export dialog if not already present (tracked by tag_id)
@@ -579,6 +597,7 @@ void CanIDUnit::update(const can_frame_data_s &data, const can_frame_diff_s &dif
                                           .transform =
                                               [this, tag_id](const ftxui::EntryState state) {
                                                 std::string display_name = "custom";
+
                                                 if (m_spnSettingsMap_ && m_spnSettingsMap_->contains(m_canid_) &&
                                                     (*m_spnSettingsMap_)[m_canid_].contains(tag_id)) {
                                                   auto &s = (*m_spnSettingsMap_)[m_canid_][tag_id];
@@ -619,26 +638,33 @@ void CanIDUnit::update(const can_frame_data_s &data, const can_frame_diff_s &dif
     }
 
     // Rebuild FromLive for custom SPN export entries when fragment count changes
-    if (m_spnSettingsMap_ && m_spnSettingsMap_->contains(m_canid_) && !m_data_verbose_->is_null() && m_data_verbose_->contains("SPNs")) {
+    if (m_spnSettingsMap_ && m_spnSettingsMap_->contains(m_canid_) && !m_data_verbose_->is_null() &&
+        m_data_verbose_->contains("SPNs")) {
       for (auto &[tag_id, pair] : m_export_custom_containers_) {
         auto &[wrapper, prev_frag_count] = pair;
+
         size_t cur_frag_count = 0;
         if ((*m_spnSettingsMap_)[m_canid_].contains(tag_id)) {
           cur_frag_count = (*m_spnSettingsMap_)[m_canid_][tag_id].fragments.size();
         }
+
         if (cur_frag_count != prev_frag_count) {
           wrapper->DetachAllChildren();
           std::string search_name;
+
           if ((*m_spnSettingsMap_)[m_canid_].contains(tag_id)) {
             search_name = (*m_spnSettingsMap_)[m_canid_][tag_id].spn_name + " (custom)";
           }
+
           const auto &spns = (*m_data_verbose_)["SPNs"];
           for (size_t i = 0; i < spns.size(); ++i) {
             if (spns[i].contains("SPN name") && spns[i]["SPN name"].get<std::string>() == search_name) {
-              wrapper->Add(FromLive(m_data_verbose_, nlohmann::json::json_pointer("/SPNs/" + std::to_string(i)), false, -100, ExpanderImpl::Root()));
+              wrapper->Add(FromLive(m_data_verbose_, nlohmann::json::json_pointer("/SPNs/" + std::to_string(i)), false,
+                                    -100, ExpanderImpl::Root()));
               break;
             }
           }
+
           prev_frag_count = cur_frag_count;
         }
       }

src/canid_unit.hpp

@@ -48,6 +48,16 @@ public:
   inline ftxui::Component getSpnSettingsForm() { return m_spnSettingsForm_; }
   inline const auto &getParametersExportMap() const { return s_canbus_parameters_export_map_; }
 
+  static inline std::string *s_export_filter_text_ = nullptr;
+
+  static inline std::map<
+      /* canid */ std::string,
+      std::tuple</* deployed flag */ bool, /* has data flag */ bool,
+                 /* Selected spns to export  */
+                 std::map</* spn name */ std::string,
+                          std::tuple</* deployed  */ bool, /* selected */ bool, /* data */ nlohmann::json>>>>
+      s_canbus_parameters_export_map_ = {};
+
   void update(const can_frame_data_s &data, const can_frame_diff_s &diff, std::shared_ptr<nlohmann::json> verbose,
               std::shared_ptr<nlohmann::json> brief);
 
@@ -71,12 +81,4 @@ private:
   size_t m_last_brief_spn_count_ = 0;
   ftxui::Component m_export_selectors_;
   std::map<int32_t, std::pair<ftxui::Component, size_t>> m_export_custom_containers_;
-
-  static inline std::map<
-      /* canid */ std::string,
-      std::tuple</* deployed flag */ bool, /* has data flag */ bool,
-                 /* Selected spns to export  */
-                 std::map</* spn name */ std::string,
-                          std::tuple</* deployed  */ bool, /* selected */ bool, /* data */ nlohmann::json>>>>
-      s_canbus_parameters_export_map_ = {};
 };

src/csv.cpp

@@ -0,0 +1,125 @@
+#include <fstream>
+#include <map>
+#include <memory>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+#define FMT_HEADER_ONLY
+#include <fmt/format.h>
+
+#include "j1939_db.hpp"
+#include "sqlite_modern_cpp.h"
+
+static std::vector<std::string> parseCsvLine(const std::string &line) {
+  std::vector<std::string> fields;
+  std::string field;
+  bool in_quotes = false;
+
+  for (size_t i = 0; i < line.size(); ++i) {
+    char c = line[i];
+
+    if (in_quotes) {
+      if (c == '"') {
+        if (i + 1 < line.size() && line[i + 1] == '"') {
+          field += '"';
+          ++i;
+        } else {
+          in_quotes = false;
+        }
+      } else {
+        field += c;
+      }
+    } else {
+
+      if (c == '"') {
+        in_quotes = true;
+      } else if (c == ',') {
+        fields.push_back(std::move(field));
+        field.clear();
+      } else {
+        field += c;
+      }
+    }
+  }
+
+  fields.push_back(std::move(field));
+  return fields;
+}
+
+std::unique_ptr<sqlite::database> parseCsv(const std::string &file) {
+  std::ifstream ifs(file);
+  if (!ifs.is_open()) {
+    throw std::runtime_error(fmt::format("Cannot open file {}", file));
+  }
+
+  auto db_ptr = std::make_unique<sqlite::database>(":memory:");
+  auto &db = *db_ptr;
+
+  initJ1939Database(db);
+
+  const auto &pgn_mapping = J1939MappingTables::pgn();
+  const auto &spn_mapping = J1939MappingTables::spn();
+
+  // Read header line and build column_index -> db_column_name maps
+  std::map<size_t, std::string> pgn_col_to_db, spn_col_to_db;
+
+  std::string header_line;
+  if (!std::getline(ifs, header_line)) {
+    throw std::runtime_error("CSV file is empty");
+  }
+
+  auto headers = parseCsvLine(header_line);
+  for (size_t i = 0; i < headers.size(); ++i) {
+    const auto &h = headers[i];
+    if (auto it = pgn_mapping.find(h); it != pgn_mapping.end()) {
+
+      pgn_col_to_db[i] = std::string(std::get<1u>(it->second));
+    } else if (auto it2 = spn_mapping.find(h); it2 != spn_mapping.end()) {
+
+      spn_col_to_db[i] = std::string(std::get<1u>(it2->second));
+    }
+  }
+
+  const auto pgn_insert_sql = buildPgnInsertSql();
+  const auto spn_insert_sql = buildSpnInsertSql();
+
+  db << "BEGIN TRANSACTION";
+
+  std::string line;
+  while (std::getline(ifs, line)) {
+    if (line.empty())
+      continue;
+
+    while (std::count(line.begin(), line.end(), '"') % 2 != 0) {
+      std::string next;
+      if (!std::getline(ifs, next))
+        break;
+      line += '\n';
+      line += next;
+    }
+
+    auto fields = parseCsvLine(line);
+    std::map<std::string, std::string> pgn_row_map, spn_row_map;
+
+    for (size_t i = 0; i < fields.size(); ++i) {
+      if (fields[i].empty())
+        continue;
+
+      if (auto it = pgn_col_to_db.find(i); it != pgn_col_to_db.end()) {
+        pgn_row_map[it->second] = std::move(fields[i]);
+      } else if (auto it2 = spn_col_to_db.find(i); it2 != spn_col_to_db.end()) {
+        spn_row_map[it2->second] = std::move(fields[i]);
+      }
+    }
+
+    if (!pgn_row_map.empty() && !spn_row_map.empty()) {
+      insertJ1939Row(db, pgn_insert_sql, spn_insert_sql, pgn_row_map, spn_row_map);
+    }
+  }
+
+  db << "COMMIT";
+
+  return db_ptr;
+}

src/filedialog.cpp

@@ -77,6 +77,7 @@ ftxui::Component makeFileDialog(ftxui::ScreenInteractive *scr, signals_map_t &sm
           ftxui::Container::Vertical({
               ftxui::Renderer([&]() -> ftxui::Element { return ftxui::text("Export file"); }) |
                   ftxui::color(ftxui::Color::Red) | ftxui::hcenter,
+              ftxui::Renderer([]() { return ftxui::separator(); }),
 
               ftxui::Container::Horizontal({
                   ftxui::Renderer([]() -> ftxui::Element {
@@ -90,9 +91,9 @@ ftxui::Component makeFileDialog(ftxui::ScreenInteractive *scr, signals_map_t &sm
                   }),
               }),
 
-              ftxui::Renderer([]() { return ftxui::separatorEmpty(); }),
-
+              ftxui::Renderer([]() { return ftxui::separator(); }),
               entryList | ftxui::vscroll_indicator | ftxui::frame | ftxui::flex,
+              ftxui::Renderer([]() { return ftxui::separator(); }),
 
               ftxui::Container::Horizontal({
                   ftxui::Button({
@@ -104,7 +105,7 @@ ftxui::Component makeFileDialog(ftxui::ScreenInteractive *scr, signals_map_t &sm
                           },
 
                       .transform = [this](const ftxui::EntryState &state) -> ftxui::Element {
-                        return ftxui::text(" >[Export]< ") | (state.focused ? ftxui::bold : ftxui::nothing) |
+                        return ftxui::text(" >[export]< ") | (state.focused ? ftxui::bold : ftxui::nothing) |
                                ftxui::color(ftxui::Color::Cyan) |
                                (state.focused ? ftxui::bgcolor(ftxui::Color::Grey11) : ftxui::nothing);
                       },
@@ -113,7 +114,7 @@ ftxui::Component makeFileDialog(ftxui::ScreenInteractive *scr, signals_map_t &sm
                   ftxui::Button({
                       .on_click = [&]() { shown = false; },
                       .transform = [this](const ftxui::EntryState &state) -> ftxui::Element {
-                        return ftxui::text(" >[Cancel]< ") | (state.focused ? ftxui::bold : ftxui::nothing) |
+                        return ftxui::text(" >[cancel]< ") | (state.focused ? ftxui::bold : ftxui::nothing) |
                                ftxui::color(ftxui::Color::Cyan) |
                                (state.focused ? ftxui::bgcolor(ftxui::Color::Grey11) : ftxui::nothing);
                       },

src/headless.cpp

@@ -8,24 +8,23 @@
 #include <fmt/format.h>
 #include <fmt/ranges.h>
 
-HeadlessHandler::HeadlessHandler(const std::string &output_file)
-    : output_file_(output_file) {}
-
-void HeadlessHandler::onDatabaseReady(sqlite::database &db) {
-  database_ = &db;
-}
+HeadlessHandler::HeadlessHandler(const std::string &output_file) : output_file_(output_file) {}
+void HeadlessHandler::onDatabaseReady(sqlite::database &db) { database_ = &db; }
 
 void HeadlessHandler::onBatch(const std::vector<can_frame_update_s> &batch) {
-  if (!database_) return;
+  if (!database_)
+    return;
 
-  extern std::pair<nlohmann::json, nlohmann::json> processFrame(sqlite::database &db, const std::string &iface, const std::string &canid, const std::vector<uint8_t> &data);
+  extern std::pair<nlohmann::json, nlohmann::json> processFrame(
+      sqlite::database & db, const std::string &iface, const std::string &canid, const std::vector<uint8_t> &data);
 
   for (const auto &entry : batch) {
     const auto &iface = entry.iface;
     const auto &canid = entry.canid;
     const auto &frame_data = entry.data;
 
-    if (configuration_map_.contains(canid)) continue;
+    if (configuration_map_.contains(canid))
+      continue;
 
     auto [verbose, brief] = processFrame(*database_, iface, canid, frame_data.payload);
     configuration_map_.insert({

src/j1939_db.cpp

@@ -0,0 +1,212 @@
+#include "j1939_db.hpp"
+#include "parsers.hpp"
+
+#include <cmath>
+
+#define FMT_HEADER_ONLY
+#include <fmt/format.h>
+
+void initJ1939Database(sqlite::database &db) {
+  db << R"(
+    CREATE TABLE IF NOT EXISTS pgns (
+      id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+      pgn INTEGER UNIQUE,
+      pg_label TEXT,
+      pg_acronym TEXT,
+      pg_descr TEXT,
+      edp INTEGER,
+      dp INTEGER,
+      pf INTEGER,
+      ps INTEGER,
+      pg_datalen INTEGER,
+      pg_priority INTEGER
+    );
+  )";
+
+  db << R"(
+    CREATE TABLE IF NOT EXISTS spns (
+      id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+      pgn INTEGER,
+      spn INTEGER UNIQUE,
+      spn_name TEXT,
+      spn_pos TEXT,
+      spn_length INTEGER,
+      resolution REAL,
+      offset REAL,
+      data_range TEXT,
+      min_value REAL,
+      max_value REAL,
+      units TEXT,
+      slot_id TEXT,
+      slot_name TEXT,
+      spn_type TEXT,
+      value_encoding TEXT
+    );
+  )";
+
+  db << R"(
+    CREATE TABLE IF NOT EXISTS spn_fragments (
+      id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+      spn INTEGER,
+      pgn INTEGER,
+      byte_offset INTEGER,
+      bit_offset INTEGER,
+      size INTEGER
+    );
+  )";
+
+  db << "PRAGMA journal_mode = OFF";
+  db << "PRAGMA synchronous = OFF";
+  db << "PRAGMA foreign_keys = ON;";
+
+  db << "CREATE INDEX IF NOT EXISTS idx_spns_pgn ON spns(pgn);";
+  db << "CREATE INDEX IF NOT EXISTS idx_spn_fragments_spn ON spn_fragments(spn);";
+  db << "CREATE INDEX IF NOT EXISTS idx_spn_fragments_spn_pgn ON spn_fragments(spn, pgn);";
+}
+
+std::string buildPgnInsertSql() {
+  const auto &pgn_mapping = J1939MappingTables::pgn();
+  std::string cols, placeholders;
+  for (const auto &[k, v] : pgn_mapping) {
+    if (!cols.empty()) {
+      cols += ", ";
+      placeholders += ", ";
+    }
+
+    cols += std::get<1u>(v);
+    placeholders += "?";
+  }
+
+  return fmt::format("INSERT OR REPLACE INTO pgns ({}) VALUES ({})", cols, placeholders);
+}
+
+std::string buildSpnInsertSql() {
+  const auto &spn_mapping = J1939MappingTables::spn();
+  std::string cols = "pgn", placeholders = "?";
+
+  for (const auto &[k, v] : spn_mapping) {
+    cols += ", ";
+    placeholders += ", ";
+
+    if (std::get<1u>(v) == "data_range") {
+      cols += "min_value, max_value";
+      placeholders += "?, ?";
+    } else {
+      cols += std::get<1u>(v);
+      placeholders += "?";
+    }
+  }
+
+  // value_encoding is derived from Resolution format, not from a mapped CSV column.
+  cols += ", value_encoding";
+  placeholders += ", ?";
+  return fmt::format("INSERT OR REPLACE INTO spns ({}) VALUES ({})", cols, placeholders);
+}
+
+void insertJ1939Row(sqlite::database &db, const std::string &pgn_insert_sql, const std::string &spn_insert_sql,
+                    std::map<std::string, std::string> &pgn_row_map, std::map<std::string, std::string> &spn_row_map) {
+  const auto &pgn_mapping = J1939MappingTables::pgn();
+  const auto &spn_mapping = J1939MappingTables::spn();
+
+  // Insert PGN
+  try {
+    auto ps = db << pgn_insert_sql;
+    for (const auto &[k, v] : pgn_mapping) {
+      ps << pgn_row_map[std::get<1u>(v).data()];
+    }
+
+    ps.execute();
+  } catch (const sqlite::sqlite_exception &e) {
+    if (e.get_extended_code() != SQLITE_CONSTRAINT_UNIQUE) {
+      throw;
+    }
+  }
+
+  // Insert SPN
+  struct {
+    bool parts_inserted_flag = false;
+    double min = 0.0, max = 0.0;
+    size_t size_bits = 0u;
+    parsers::resolution_s::type_e encoding = parsers::resolution_s::type_e::numeric;
+  } calc;
+
+  // Pre-compute size_bits
+  {
+    auto it = spn_mapping.find("SPN Length");
+    if (it != spn_mapping.end()) {
+      auto size = parsers::parseSpnSize(spn_row_map[std::get<1u>(it->second).data()]);
+      calc.size_bits = size.has_value() ? size.value().size_bits : 0u;
+    }
+  }
+
+  for (const auto &[k, v] : spn_row_map) {
+    try {
+      auto ps = db << spn_insert_sql;
+      ps << pgn_row_map["pgn"];
+
+      for (const auto &[k, v] : spn_mapping) {
+        if (std::get<1u>(v) == "data_range") {
+          auto range = parsers::parseSpnDataRange(spn_row_map[std::get<1u>(v).data()]);
+
+          if (range.has_value()) {
+            calc.min = range.value().min;
+            calc.max = range.value().max;
+            ps << range.value().min << range.value().max;
+          } else {
+            ps << nullptr << nullptr;
+          }
+        } else if (std::get<1u>(v) == "offset") {
+
+          auto offset = parsers::parseSpnOffset(spn_row_map[std::get<1u>(v).data()]);
+          ps << (offset.has_value() ? offset.value().offset : 0.0);
+        } else if (std::get<1u>(v) == "spn_length") {
+
+          auto size = parsers::parseSpnSize(spn_row_map[std::get<1u>(v).data()]);
+          calc.size_bits = size.has_value() ? size.value().size_bits : 0u;
+          ps << calc.size_bits;
+        } else if (std::get<1u>(v) == "resolution") {
+
+          auto resolution = parsers::parseSpnResolution(spn_row_map[std::get<1u>(v).data()]);
+          calc.encoding = resolution.has_value() ? resolution.value().type : parsers::resolution_s::type_e::numeric;
+          double calculated = (calc.max - calc.min) / (std::pow(2.0, calc.size_bits) - 1.0);
+
+          if (std::fabs(calculated - 1.0) < 1e-9) {
+            ps << calculated;
+          } else {
+            ps << (resolution.has_value() ? resolution.value().resolution : 1.0);
+          }
+        } else {
+          ps << spn_row_map[std::get<1u>(v).data()];
+        }
+      }
+
+      ps << parsers::resolutionTypeName(calc.encoding);
+      ps.execute();
+
+      // Insert SPN fragments
+      if (!calc.parts_inserted_flag) {
+        auto size = parsers::parseSpnSize(spn_row_map[std::get<1u>(spn_mapping.at("SPN Length")).data()]);
+
+        if (size.has_value()) {
+          auto spn_fragments = parsers::parseSpnPosition(
+              size.value().size_bits, spn_row_map[std::get<1u>(spn_mapping.at("SPN Position in PG")).data()]);
+
+          if (spn_fragments.has_value()) {
+            auto spn = std::stoll(spn_row_map["spn"]);
+
+            for (const auto &part : spn_fragments.value().spn_fragments) {
+              db << R"(INSERT OR REPLACE INTO spn_fragments (spn, pgn, byte_offset, bit_offset, size) VALUES (?, ?, ?, ?, ?);)"
+                 << spn << std::stoll(pgn_row_map["pgn"]) << part.byte_offset << part.bit_offset << part.size;
+            }
+
+            calc.parts_inserted_flag = true;
+          }
+        }
+      }
+    } catch (const sqlite::sqlite_exception &e) {
+      if (e.get_extended_code() != SQLITE_CONSTRAINT_UNIQUE) {
+        throw;
+      }
+    }
+  }
+}

src/j1939_db.hpp

@@ -0,0 +1,56 @@
+#pragma once
+
+#include <map>
+#include <memory>
+#include <string>
+#include <string_view>
+#include <tuple>
+
+#include "sqlite_modern_cpp.h"
+
+struct J1939MappingTables {
+  static const std::map<std::string_view, std::tuple<size_t, std::string_view>> &pgn() {
+    static const std::map<std::string_view, std::tuple<size_t, std::string_view>> t = {
+        {"PGN", {5u, "pgn"}},
+        {"Parameter Group Label", {6u, "pg_label"}},
+        {"PG Acronym", {7u, "pg_acronym"}},
+        {"PG Description", {8u, "pg_descr"}},
+        {"EDP", {9u, "edp"}},
+        {"DP", {10u, "dp"}},
+        {"PF", {11u, "pf"}},
+        {"PS", {12u, "ps"}},
+        {"PG Data Length", {15u, "pg_datalen"}},
+        {"Default Priority", {16u, "pg_priority"}},
+    };
+    return t;
+  }
+
+  static const std::map<std::string_view, std::tuple<size_t, std::string_view>> &spn() {
+    static const std::map<std::string_view, std::tuple<size_t, std::string_view>> t = {
+        {"SPN", {19u, "spn"}},
+        {"SPN Name", {20u, "spn_name"}},
+        {"SPN Position in PG", {18u, "spn_pos"}},
+        {"SPN Length", {22u, "spn_length"}},
+        {"Offset", {24u, "offset"}},
+        {"Data Range", {25u, "data_range"}},
+        {"Resolution", {23u, "resolution"}},
+        {"Units", {27u, "units"}},
+        {"SLOT Identifier", {28u, "slot_id"}},
+        {"SLOT Name", {29u, "slot_name"}},
+        {"SPN Type", {30u, "spn_type"}},
+    };
+    return t;
+  }
+};
+
+void initJ1939Database(sqlite::database &db);
+
+// Build INSERT SQL for pgns table (call once, reuse)
+std::string buildPgnInsertSql();
+
+// Build INSERT SQL for spns table (call once, reuse)
+std::string buildSpnInsertSql();
+
+// Insert a row into the database. pgn_row_map/spn_row_map are column_name -> value.
+void insertJ1939Row(sqlite::database &db, const std::string &pgn_insert_sql, const std::string &spn_insert_sql,
+                    std::map<std::string, std::string> &pgn_row_map, std::map<std::string, std::string> &spn_row_map);

src/main.cpp

@@ -1,5 +1,6 @@
 #include <atomic>
 #include <boost/signals2.hpp>
+#include <cctype>
 #include <charconv>
 #include <cstdint>
 #include <cstdio>
@@ -7,7 +8,7 @@
 #include <map>
 #include <memory>
 #include <nlohmann/json.hpp>
-#include <sstream>
+#include <ranges>
 #include <stop_token>
 #include <string>
 #include <sys/epoll.h>
@@ -31,12 +32,14 @@
 #include "ftxui/component/screen_interactive.hpp"
 #include "ftxui/dom/elements.hpp"
 #include "headless.hpp"
+#include "headless_streamer.hpp"
 #include "process.hpp"
 #include "recorder.hpp"
 #include "signals.hpp"
 #include <clipp.h>
 
 std::mutex g_j1939_db_mtx;
+std::atomic<uint64_t> g_error_frame_count{0};
 
 int32_t main(int32_t argc, char *argv[]) {
   static auto screen = ftxui::ScreenInteractive::Fullscreen();
@@ -46,15 +49,19 @@ int32_t main(int32_t argc, char *argv[]) {
   static std::atomic<sqlite::database *> j1939_db{nullptr};
   static std::stop_source aggregator_task_stop, refresh_task_stop, headless_task_stop;
   static TinyProcessLib::Process *p = nullptr;
-  std::future<void> xlsx_parser_task, headless_task;
+  std::future<void> j1939_parser_task, headless_task;
   extern std::unique_ptr<sqlite::database> parseXlsx(const std::string &file);
+  extern std::unique_ptr<sqlite::database> parseCsv(const std::string &file);
   static std::unique_ptr<sqlite::database> j1939_db_owner;
   static std::unique_ptr<Recorder> recorder;
   static std::unique_ptr<HeadlessHandler> headless_handler;
+  static std::unique_ptr<HeadlessStreamer> headless_streamer;
+
+  enum class Mode { tui, discover, record, headless } mode = Mode::tui;
 
   static struct {
-    std::string docfile, command = "", output_file = "", record_db_path = "";
-    bool show_help = false, headless_mode = false, sync_to_server = false, record_mode = false, tui_mode = false;
+    std::string xlsx_file, csv_file, command = "", output_file = "", record_db_path = "";
+    bool show_help = false;
   } cli_opts;
 
   // Parse cli options
@@ -65,41 +72,48 @@ int32_t main(int32_t argc, char *argv[]) {
 
     auto cli = (
 
-        clipp::option("-hl", "--headless")
-            .doc("Headless mode, write configuration to stdout if output file is not specified")
-            .set(cli_opts.headless_mode)
-            .call([]() { fmt::println("Headless mode"); }),
+        clipp::option("-dscvr, --discovery-mode")
+            .doc("Discover mode: output PGN/SPN structure (only first received falue) to stdout or file")
+            .call([&]() { mode = Mode::discover; }),
 
-        clipp::option("-of", "--output-file") &
-            clipp::value("Output file to write configuration", cli_opts.output_file)
-                .call([&]() { fmt::println("Output file is: {}", cli_opts.output_file); })
-                .doc("Specify output file to write configuration"),
+        clipp::option("-hl", "--headless")
+            .doc("Headless mode: stream all decoded PGN/SPN values to stdout")
+            .call([&]() { mode = Mode::headless; }),
 
         clipp::option("-rec", "--record")
-            .doc("Record mode: collect decoded J1939 SPN values to SQLite DB")
-            .set(cli_opts.record_mode)
-            .call([]() { fmt::println("Record mode"); }),
+            .doc("Record mode: write all decoded PGN/SPN values + timestamps to SQLite DB")
+            .call([&]() { mode = Mode::record; }),
 
-        clipp::option("-db", "--database") &
-            clipp::value("SQLite output database path", cli_opts.record_db_path)
-                .call([&]() { fmt::println("Record DB: {}", cli_opts.record_db_path); })
-                .doc("Path for the output SQLite database (used with -rec)"),
+        clipp::option("-of", "--output-file") &
+            clipp::value("Output file", cli_opts.output_file).doc("Output file path (used with -discover)"),
 
-        clipp::option("-tui").doc("Enable TUI mode (use with -rec to show UI while recording)").set(cli_opts.tui_mode),
+        clipp::option("-db", "--database") & clipp::value("SQLite output database path", cli_opts.record_db_path)
+                                                 .doc("SQLite database path (used with -rec)"),
 
         clipp::option("-e", "--execute-command") &
             clipp::value("command", cli_opts.command).call([&]() {}).doc("execute cli command to read its output"),
 
-        clipp::required("-j1939", "--j1939-document") &
-            clipp::value("J1939 Document file", cli_opts.docfile)
-                .call([&]() {
-                  xlsx_parser_task = std::async(std::launch::async, [&]() {
-                    j1939_db_owner = parseXlsx(cli_opts.docfile);
-                    j1939_db.store(j1939_db_owner.get());
-                    signals.map.get<void(sqlite::database &)>("j1939_database_ready")->operator()(*j1939_db_owner);
-                  });
-                })
-                .doc("provide .xlsx document file for J1939 processing"));
+        (clipp::option("-j1939-xlsx") &
+         clipp::value("J1939 XLSX file", cli_opts.xlsx_file)
+             .call([&]() {
+               j1939_parser_task = std::async(std::launch::async, [&]() {
+                 j1939_db_owner = parseXlsx(cli_opts.xlsx_file);
+                 j1939_db.store(j1939_db_owner.get());
+                 signals.map.get<void(sqlite::database &)>("j1939_database_ready")->operator()(*j1939_db_owner);
+               });
+             })
+             .doc("J1939 Digital Annex .xlsx file")) |
+
+            (clipp::option("-j1939-csv") &
+             clipp::value("J1939 CSV file", cli_opts.csv_file)
+                 .call([&]() {
+                   j1939_parser_task = std::async(std::launch::async, [&]() {
+                     j1939_db_owner = parseCsv(cli_opts.csv_file);
+                     j1939_db.store(j1939_db_owner.get());
+                     signals.map.get<void(sqlite::database &)>("j1939_database_ready")->operator()(*j1939_db_owner);
+                   });
+                 })
+                 .doc("J1939 Digital Annex .csv file")));
 
     auto man = clipp::make_man_page(cli, argv[0]);
     auto cli_with_help = (cli | clipp::option("-h", "--help").set(cli_opts.show_help).doc("show this help").call([&]() {
@@ -111,7 +125,7 @@ int32_t main(int32_t argc, char *argv[]) {
       return -1;
     }
 
-    if (cli_opts.record_mode && cli_opts.record_db_path.empty()) {
+    if (mode == Mode::record && cli_opts.record_db_path.empty()) {
       fmt::println(stderr, "Error: -rec requires -db <path>");
       return -1;
     }
@@ -119,44 +133,105 @@ int32_t main(int32_t argc, char *argv[]) {
 
   // Parse a single candump line and aggregate it
   static const auto parse_candump_line = [](const std::string &line) {
-    if (line.empty())
+    enum class field_e : size_t {
+      INTERFACE = 0,
+      CANID = 1,
+      DLC = 2,
+      PAYLOAD_BEGIN = 3,
+    };
+
+    constexpr auto idx = [](enum field_e f) consteval { return static_cast<size_t>(f); };
+    if (line.empty()) {
       return;
+    }
 
-    std::vector<std::string> words;
-    std::istringstream iss(line);
-    std::string s;
-
-    while (std::getline(iss, s, ' ')) {
-      if (!s.empty()) {
-        words.push_back(s);
+    std::vector<std::string_view> words;
+    for (auto part : std::string_view(line) | std::views::split(' ')) {
+      if (!part.empty()) {
+        words.emplace_back(part.begin(), part.end());
       }
     }
 
-    if (words.size() >= 4u) { // 0 - interface, 1 - canid, 2 - size, 3 - payload (1 byte minimum)
-      if (words[0].starts_with("can") || words[0].starts_with("vcan")) {
-        can_frame_data_s entry;
+    if (words.size() > idx(field_e::PAYLOAD_BEGIN)) {
+      auto &iface = words[idx(field_e::INTERFACE)];
+      can_frame_data_s entry;
+      auto &canid = words[idx(field_e::CANID)];
 
-        // Parse DLC
-        {
-          auto sv = std::string_view(words[2]).substr(1, words[2].size() - 2);
-          auto [ptr, ec] = std::from_chars(sv.data(), sv.data() + sv.size(), entry.size);
-          if (ec != std::errc{}) {
-            entry.size = 0;
+      // Validate CAN ID: 3 hex digits (SFF, 11-bit) or 8 (EFF, 29-bit)
+      {
+        constexpr auto sff_length_bytes = 3u, eff_length_bytes = 8u;
+
+        // Check CAN_ID length
+        if (canid.size() != sff_length_bytes && canid.size() != eff_length_bytes) {
+          return;
+        }
+
+        // Check CAN_ID symbols
+        for (const char &c : canid) {
+          if (!std::isxdigit(static_cast<uint8_t>(c))) {
+            return;
           }
         }
+      }
 
-        // Parse payload bytes directly
-        entry.payload.reserve(words.size() - 3);
-        for (size_t i = 3; i < words.size(); ++i) {
-          uint8_t byte = 0;
-          std::from_chars(words[i].data(), words[i].data() + words[i].size(), byte, 16);
-          entry.payload.push_back(byte);
+      // Parse DLC
+      {
+        auto &dlc = words[idx(field_e::DLC)];
+        if (dlc.size() < 3 /* [${size}] format  */ || dlc.front() != '[' || dlc.back() != ']') {
+          return;
         }
 
-        {
-          std::lock_guard<std::mutex> lock(rw_mtx);
-          aggregated[words[0]][words[1]] = std::make_shared<can_frame_data_s>(std::move(entry));
+        auto sv = dlc.substr(1, dlc.size() - 2);
+        if (auto [ptr, ec] = std::from_chars(sv.data(), sv.data() + sv.size(), entry.size);
+            ec != std::errc{} || ptr != sv.data() + sv.size()) {
+          return;
         }
+
+        // Max payload per CAN frame: 8 (Classic CAN) / 64 (CAN FD). Use 64 as upper bound.
+        constexpr int32_t max_payload_bytes = 64;
+        if (entry.size < 0 || entry.size > max_payload_bytes) {
+          return;
+        }
+      }
+
+      // Detect ERRORFRAME marker (SocketCAN diagnostic pseudo-frame): count and drop
+      if (words.back() == "ERRORFRAME") {
+        g_error_frame_count.fetch_add(1, std::memory_order_relaxed);
+        return;
+      }
+
+      // Detect RTR: candump prints "remote request" in place of payload bytes. Drop silently.
+      if (words[idx(field_e::PAYLOAD_BEGIN)] == "remote") {
+        return;
+      }
+
+      // Parse payload bytes directly
+      entry.payload.reserve(words.size() - idx(field_e::PAYLOAD_BEGIN));
+
+      for (size_t i = idx(field_e::PAYLOAD_BEGIN); i < words.size(); ++i) {
+        if (words[i].size() != 2) { // each byte must be exactly 2 hex digits
+          return;
+        }
+
+        uint8_t byte = 0;
+        auto *first = words[i].data();
+        auto *last = first + words[i].size();
+        if (auto [ptr, ec] = std::from_chars(first, last, byte, 16 /* HEX format */);
+            ec != std::errc{} || ptr != last) {
+          return;
+        }
+
+        entry.payload.push_back(byte);
+      }
+
+      // DLC must match the actual number of payload bytes
+      if (entry.payload.size() != entry.size) {
+        return;
+      }
+
+      {
+        std::lock_guard<std::mutex> lock(rw_mtx);
+        aggregated[std::string(iface)][std::string(canid)] = std::make_shared<can_frame_data_s>(std::move(entry));
       }
     }
   };
@@ -342,14 +417,13 @@ int32_t main(int32_t argc, char *argv[]) {
     ::signal(SIGINT, signal_handler);
   }
 
-  if (cli_opts.record_mode) {
-    bool rec_console = !cli_opts.tui_mode && !cli_opts.headless_mode;
-    recorder = std::make_unique<Recorder>(cli_opts.record_db_path, rec_console);
+  if (mode == Mode::record) {
+    recorder = std::make_unique<Recorder>(cli_opts.record_db_path, true);
     signals.map.get<void(const std::vector<can_frame_update_s> &)>("new_entries_batch")
         ->connect([](const std::vector<can_frame_update_s> &batch) { recorder->onBatch(batch); });
   }
 
-  if (cli_opts.headless_mode) {
+  if (mode == Mode::discover) {
     headless_handler = std::make_unique<HeadlessHandler>(cli_opts.output_file);
 
     signals.map.get<void(sqlite::database &)>("j1939_database_ready")->connect([](sqlite::database &db) {
@@ -360,8 +434,13 @@ int32_t main(int32_t argc, char *argv[]) {
         ->connect([](const std::vector<can_frame_update_s> &batch) { headless_handler->onBatch(batch); });
   }
 
-  // Determine if TUI should run: default on, off if headless, off if rec-only (no -tui)
-  bool run_tui = !cli_opts.headless_mode && (!cli_opts.record_mode || cli_opts.tui_mode);
+  if (mode == Mode::headless) {
+    headless_streamer = std::make_unique<HeadlessStreamer>();
+    signals.map.get<void(const std::vector<can_frame_update_s> &)>("new_entries_batch")
+        ->connect([](const std::vector<can_frame_update_s> &batch) { headless_streamer->onBatch(batch); });
+  }
+
+  bool run_tui = (mode == Mode::tui);
 
   if (run_tui) {
     extern ftxui::Component makeMainForm(ftxui::ScreenInteractive * screen, signals_map_t & smap);
@@ -415,7 +494,7 @@ int32_t main(int32_t argc, char *argv[]) {
   {
     const char *names[] = {"xlsx_parser", "aggregator", "refresh", "headless"};
     int idx = 0;
-    for (auto *task : {&xlsx_parser_task, &aggregator_task, &refresh_task, &headless_task}) {
+    for (auto *task : {&j1939_parser_task, &aggregator_task, &refresh_task, &headless_task}) {
       if (task && task->valid()) {
         task->wait_for(std::chrono::seconds(3));
       }

src/mainform.cpp

@@ -30,6 +30,7 @@ ftxui::Component makeMainForm(ftxui::ScreenInteractive *screen, signals_map_t &s
       static float canbus_params_focus_relative = 0;
       static std::string canid_active;
       static std::string filter_text;
+      static std::string export_filter_text;
       static size_t tags_count = 0u;
       static std::unordered_map<std::string, std::shared_ptr<CanIDUnit>> canid_lookup;
       static std::atomic<sqlite::database *> database_atomic{nullptr};
@@ -47,6 +48,7 @@ ftxui::Component makeMainForm(ftxui::ScreenInteractive *screen, signals_map_t &s
       static std::map<std::string, std::map<int32_t, ftxui::Component>> spnSettingsFormMap;
       static auto spn_export_dialog = ftxui::Container::Vertical({});
       static auto canbus_params_export_dialog = ftxui::Container::Vertical({});
+      CanIDUnit::s_export_filter_text_ = &export_filter_text;
       static auto canbus_player_dialog = ftxui::Container::Vertical({});
 
       static const auto convertParametersMapToJson = []() {
@@ -186,6 +188,15 @@ ftxui::Component makeMainForm(ftxui::ScreenInteractive *screen, signals_map_t &s
                         fmt::format(" Uptime: {} ", fmt::format("{:02}:{:02}:{:02}", hours, minutes, seconds)))});
                   }),
 
+                  ftxui::Renderer([]() {
+                    const auto errors = g_error_frame_count.load(std::memory_order_relaxed);
+                    return ftxui::hbox({
+                        ftxui::text(" Errors: "),
+                        ftxui::text(fmt::format("{} ", errors)) |
+                            ftxui::color(errors ? ftxui::Color::Red : ftxui::Color::GrayDark),
+                    });
+                  }),
+
                   ftxui::Renderer([]() { return ftxui::separator(); }),
                   ftxui::Container::Horizontal({
                       ftxui::Input({
@@ -314,6 +325,37 @@ ftxui::Component makeMainForm(ftxui::ScreenInteractive *screen, signals_map_t &s
                           }) | ftxui::hcenter,
                           ftxui::Renderer([]() { return ftxui::separator(); }),
 
+                          ftxui::Input({
+                              .content = &export_filter_text,
+                              .placeholder = "regex filter ...",
+                              .transform = [](ftxui::InputState state) -> ftxui::Element {
+                                bool valid = true;
+                                if (!export_filter_text.empty()) {
+                                  try {
+                                    boost::regex(export_filter_text, boost::regex_constants::icase);
+                                  } catch (...) {
+                                    valid = false;
+                                  }
+                                }
+
+                                state.element |= (!valid ? ftxui::color(ftxui::Color::Red) : ftxui::nothing) |
+                                                 (state.focused ? ftxui::color(ftxui::Color::Cyan) : ftxui::nothing) |
+                                                 (state.hovered ? ftxui::bold : ftxui::nothing);
+
+                                return ftxui::hbox({
+                                           ftxui::text(" Search: [ "),
+                                           state.element |
+                                               (state.hovered || state.focused ? ftxui::bgcolor(ftxui::Color::Grey11)
+                                                                               : ftxui::nothing) |
+                                               ftxui::xflex,
+                                           ftxui::text(" ]"),
+                                       });
+                              },
+                              .multiline = false,
+                          }),
+
+                          ftxui::Renderer([]() { return ftxui::separator(); }),
+
                           (canbus_params_export_dialog | ftxui::Renderer([](ftxui::Element inner) {
                              return inner | ftxui::focusPositionRelative(0, canbus_params_focus_relative) |
                                     ftxui::vscroll_indicator | ftxui::frame | ftxui::flex;
@@ -362,48 +404,48 @@ ftxui::Component makeMainForm(ftxui::ScreenInteractive *screen, signals_map_t &s
                       }) | ftxui::size(ftxui::WIDTH, ftxui::EQUAL, 96) |
                           ftxui::size(ftxui::HEIGHT, ftxui::EQUAL, 48) | ftxui::border |
                           ftxui::CatchEvent([](ftxui::Event event) {
-                            static constexpr float scroll_step = 0.03f;
-                            static const auto increment_focus = []() {
-                              canbus_params_focus_relative =
-                                  std::clamp(canbus_params_focus_relative + scroll_step, 0.0f, 1.0f);
-                            };
-                            static const auto decrement_focus = []() {
-                              canbus_params_focus_relative =
-                                  std::clamp(canbus_params_focus_relative - scroll_step, 0.0f, 1.0f);
+                            const auto scroll_step = []() -> float {
+                              size_t visible = 0;
+                              for (const auto &[_, entry] : CanIDUnit::s_canbus_parameters_export_map_) {
+                                if (export_filter_text.empty()) {
+                                  ++visible;
+                                } else {
+                                  // Filtering is handled by Maybe in canid_unit.cpp,
+                                  // just count all for scroll step estimation
+                                  ++visible;
+                                }
+                                if (std::get<0u>(entry))
+                                  visible += 5;
+                              }
+                              return visible > 0 ? 1.0f / static_cast<float>(visible) : 0.03f;
                             };
 
                             if (event.is_mouse()) {
                               switch (static_cast<enum ftxui::Mouse::Button>(event.mouse().button)) {
-                              case ftxui::Mouse::Button::WheelDown: {
-                                increment_focus();
-                                goto done;
-                              } break;
-
-                              case ftxui::Mouse::Button::WheelUp: {
-                                decrement_focus();
-                                goto done;
-                              } break;
-
+                              case ftxui::Mouse::Button::WheelDown:
+                                canbus_params_focus_relative =
+                                    std::clamp(canbus_params_focus_relative + scroll_step(), 0.0f, 1.0f);
+                                return true;
+                              case ftxui::Mouse::Button::WheelUp:
+                                canbus_params_focus_relative =
+                                    std::clamp(canbus_params_focus_relative - scroll_step(), 0.0f, 1.0f);
+                                return true;
                               default:
                                 break;
                               }
                             } else if (!event.is_character()) {
                               if (event == ftxui::Event::ArrowDown) {
-
-                                increment_focus();
-                                goto done;
+                                canbus_params_focus_relative =
+                                    std::clamp(canbus_params_focus_relative + scroll_step(), 0.0f, 1.0f);
+                                return true;
                               } else if (event == ftxui::Event::ArrowUp) {
-
-                                decrement_focus();
-                                goto done;
+                                canbus_params_focus_relative =
+                                    std::clamp(canbus_params_focus_relative - scroll_step(), 0.0f, 1.0f);
+                                return true;
                               }
                             }
 
-                          forward:
                             return false;
-
-                          done:
-                            return true;
                           }),
                       &canbus_params_export_dialog_shown) |
                   ftxui::Modal(makeFileDialog(screen, smap, file_dialog_shown), &file_dialog_shown) |

src/parsers.cpp

@@ -4,30 +4,40 @@ namespace parsers {
 std::optional<struct range_s> parseSpnDataRange(const std::string &str) {
   struct range_s result;
   auto begin = str.begin(), end = str.end();
-  return phrase_parse(begin, end, range::range_parser_s<decltype(begin)>{}, ascii::space, result) ? std::optional<range_s>(result) : std::nullopt;
+  return phrase_parse(begin, end, range::range_parser_s<decltype(begin)>{}, ascii::space, result)
+             ? std::optional<range_s>(result)
+             : std::nullopt;
 }
 
 std::optional<struct size_s> parseSpnSize(const std::string &str) {
   struct size_s result;
   auto begin = str.begin(), end = str.end();
-  return phrase_parse(begin, end, size::size_parser_s<decltype(begin)>{}, ascii::space, result) ? std::optional<size_s>(result) : std::nullopt;
+  return phrase_parse(begin, end, size::size_parser_s<decltype(begin)>{}, ascii::space, result)
+             ? std::optional<size_s>(result)
+             : std::nullopt;
 }
 
 std::optional<struct offset_s> parseSpnOffset(const std::string &str) {
   struct offset_s result;
   auto begin = str.begin(), end = str.end();
-  return phrase_parse(begin, end, offset::offset_parser_s<decltype(begin)>{}, ascii::space, result) ? std::optional<struct offset_s>(result) : std::nullopt;
+  return phrase_parse(begin, end, offset::offset_parser_s<decltype(begin)>{}, ascii::space, result)
+             ? std::optional<struct offset_s>(result)
+             : std::nullopt;
 }
 
 std::optional<struct resolution_s> parseSpnResolution(const std::string &str) {
   struct resolution_s result;
   auto begin = str.begin(), end = str.end();
-  return phrase_parse(begin, end, resolution::resolution_parser_s<decltype(begin)>{}, ascii::space, result) ? std::optional<struct resolution_s>(result) : std::nullopt;
+  return phrase_parse(begin, end, resolution::resolution_parser_s<decltype(begin)>{}, ascii::space, result)
+             ? std::optional<struct resolution_s>(result)
+             : std::nullopt;
 }
 
 std::optional<struct spn_fragments_s> parseSpnPosition(size_t spn_size_bits, const std::string &str) {
   struct spn_fragments_s result;
   auto begin = str.begin(), end = str.end();
-  return phrase_parse(begin, end, position::position_parser_s<decltype(begin)>(spn_size_bits), ascii::space, result) ? std::optional<struct spn_fragments_s>(result) : std::nullopt;
+  return phrase_parse(begin, end, position::position_parser_s<decltype(begin)>(spn_size_bits), ascii::space, result)
+             ? std::optional<struct spn_fragments_s>(result)
+             : std::nullopt;
 }
 }; // namespace parsers

src/parsers.hpp

@@ -1,11 +1,11 @@
 #pragma once
 
-#include <fmt/base.h>
 #include <algorithm>
 #include <boost/bind.hpp>
 #include <boost/spirit/home/qi/numeric/uint.hpp>
 #include <boost/spirit/include/phoenix.hpp>
 #include <boost/spirit/include/qi.hpp>
+#include <fmt/base.h>
 #include <optional>
 #include <string>
 
@@ -43,9 +43,26 @@ struct spn_fragments_s {
 
 // Parse resolution string to this struct
 struct resolution_s {
-  double resolution;
+  enum class type_e { numeric, enum_states, binary, ascii };
+  double resolution = 1.0;
+  type_e type = type_e::numeric;
 };
 
+// Map resolution_s::type_e to a lowercase string tag used in SQLite / JSON
+inline const char *resolutionTypeName(resolution_s::type_e t) {
+  switch (t) {
+  case resolution_s::type_e::numeric:
+    return "numeric";
+  case resolution_s::type_e::enum_states:
+    return "enum";
+  case resolution_s::type_e::binary:
+    return "binary";
+  case resolution_s::type_e::ascii:
+    return "ascii";
+  }
+  return "numeric";
+}
+
 namespace _detail {
 template <typename It, typename Res> struct parser_s : qi::grammar<It, Res(), ascii::space_type> {
   parser_s() : parser_s::base_type(rule) {}
@@ -107,8 +124,9 @@ template <typename It> struct size_parser_s : _detail::parser_s<It, size_s> {
       }
     };
 
-    this->rule = ((qi::uint_ >> "byte") | (qi::uint_ >> "bytes"))[_val = boost::phoenix::function<bytes_to_bits_s>{}(qi::_1)] |
-                 ((qi::uint_ >> "bit") | (qi::uint_ >> "bits"))[_val = boost::phoenix::function<as_bits_s>{}(qi::_1)];
+    this->rule =
+        ((qi::uint_ >> "byte") | (qi::uint_ >> "bytes"))[_val = boost::phoenix::function<bytes_to_bits_s>{}(qi::_1)] |
+        ((qi::uint_ >> "bit") | (qi::uint_ >> "bits"))[_val = boost::phoenix::function<as_bits_s>{}(qi::_1)];
   }
 };
 }; // namespace size
@@ -215,7 +233,8 @@ template <typename It> struct position_parser_s : _detail::parser_s<It, spn_frag
 
     // start byte, last integer byte and last byte with bit offset
     struct rule_v5_handler_s {
-      spn_fragments_s operator()(size_t size_bits, uint32_t start_byte, uint32_t last_integer_byte, uint32_t last_byte, uint32_t bit_offset) const {
+      spn_fragments_s operator()(size_t size_bits, uint32_t start_byte, uint32_t last_integer_byte, uint32_t last_byte,
+                                 uint32_t bit_offset) const {
         return {
             .spn_fragments =
                 {
@@ -238,7 +257,8 @@ template <typename It> struct position_parser_s : _detail::parser_s<It, spn_frag
 
     // start byte with bit offset, first integer byte and last integer byte
     struct rule_v6_handler_s {
-      spn_fragments_s operator()(uint32_t start_byte, uint32_t bit_offset, uint32_t first_integer_byte, uint32_t last_byte) const {
+      spn_fragments_s operator()(uint32_t start_byte, uint32_t bit_offset, uint32_t first_integer_byte,
+                                 uint32_t last_byte) const {
         return {
             .spn_fragments =
                 {
@@ -261,20 +281,29 @@ template <typename It> struct position_parser_s : _detail::parser_s<It, spn_frag
     };
 
     position_rule_v0 = (qi::uint_)[qi::_val = boost::phoenix::function<rule_v0_handler_s>{}(qi::_1)];
-    position_rule_v1 = (qi::uint_ >> '.' >> qi::uint_)[qi::_val = boost::phoenix::function<rule_v1_handler_s>{}(size_bits, qi::_1, qi::_2)];
-    position_rule_v2 = (qi::uint_ >> '-' >> qi::uint_)[qi::_val = boost::phoenix::function<rule_v2_handler_s>{}(qi::_1, qi::_2)];
-    position_rule_v3 = (qi::uint_ >> ',' >> qi::uint_ >> '.' >> qi::uint_)[qi::_val = boost::phoenix::function<rule_v3_handler_s>{}(size_bits, qi::_1, qi::_2, qi::_3)];
-    position_rule_v4 = (qi::uint_ >> '.' >> qi::uint_ >> ',' >> qi::uint_)[qi::_val = boost::phoenix::function<rule_v4_handler_s>{}(qi::_1, qi::_2, qi::_3)];
+    position_rule_v1 = (qi::uint_ >> '.' >>
+                        qi::uint_)[qi::_val = boost::phoenix::function<rule_v1_handler_s>{}(size_bits, qi::_1, qi::_2)];
+    position_rule_v2 =
+        (qi::uint_ >> '-' >> qi::uint_)[qi::_val = boost::phoenix::function<rule_v2_handler_s>{}(qi::_1, qi::_2)];
+    position_rule_v3 =
+        (qi::uint_ >> ',' >> qi::uint_ >> '.' >>
+         qi::uint_)[qi::_val = boost::phoenix::function<rule_v3_handler_s>{}(size_bits, qi::_1, qi::_2, qi::_3)];
+    position_rule_v4 = (qi::uint_ >> '.' >> qi::uint_ >> ',' >>
+                        qi::uint_)[qi::_val = boost::phoenix::function<rule_v4_handler_s>{}(qi::_1, qi::_2, qi::_3)];
     position_rule_v5 = (qi::uint_ >> '-' >> qi::uint_ >> ',' >> qi::uint_ >> '.' >>
-                        qi::uint_)[qi::_val = boost::phoenix::function<rule_v5_handler_s>{}(size_bits, qi::_1, qi::_2, qi::_3, qi::_4)];
+                        qi::uint_)[qi::_val = boost::phoenix::function<rule_v5_handler_s>{}(size_bits, qi::_1, qi::_2,
+                                                                                            qi::_3, qi::_4)];
     position_rule_v6 =
-        (qi::uint_ >> '.' >> qi::uint_ >> ',' >> qi::uint_ >> '-' >> qi::uint_)[qi::_val = boost::phoenix::function<rule_v6_handler_s>{}(qi::_1, qi::_2, qi::_3, qi::_4)];
+        (qi::uint_ >> '.' >> qi::uint_ >> ',' >> qi::uint_ >> '-' >>
+         qi::uint_)[qi::_val = boost::phoenix::function<rule_v6_handler_s>{}(qi::_1, qi::_2, qi::_3, qi::_4)];
 
     // If one of rules works
-    this->rule = position_rule_v6 | position_rule_v5 | position_rule_v4 | position_rule_v3 | position_rule_v2 | position_rule_v1 | position_rule_v0;
+    this->rule = position_rule_v6 | position_rule_v5 | position_rule_v4 | position_rule_v3 | position_rule_v2 |
+                 position_rule_v1 | position_rule_v0;
   }
 
-  qi::rule<It, spn_fragments_s(), ascii::space_type> position_rule_v0, position_rule_v1, position_rule_v2, position_rule_v3, position_rule_v4, position_rule_v5, position_rule_v6;
+  qi::rule<It, spn_fragments_s(), ascii::space_type> position_rule_v0, position_rule_v1, position_rule_v2,
+      position_rule_v3, position_rule_v4, position_rule_v5, position_rule_v6;
 };
 }; // namespace position
 
@@ -294,30 +323,51 @@ template <typename It> struct resolution_parser_s : _detail::parser_s<It, resolu
   resolution_parser_s() : _detail::parser_s<It, resolution_s>() {
     struct resolution_rule_v0_handler_s {
       resolution_s operator()(float number) const {
-        return {
-            .resolution = number,
-        };
+        return {.resolution = number, .type = resolution_s::type_e::numeric};
       }
     };
 
     struct resolution_rule_v1_handler_s {
       resolution_s operator()(double first, double second) const {
-        return {
-            .resolution = first / second,
-        };
+        return {.resolution = first / second, .type = resolution_s::type_e::numeric};
       }
     };
 
+    // `N states/M bit` — enum SPN: raw value is the state index, no scaling.
+    struct resolution_rule_v2_handler_s {
+      resolution_s operator()() const { return {.resolution = 1.0, .type = resolution_s::type_e::enum_states}; }
+    };
+
+    // `Binary` — single-bit boolean flag.
+    struct resolution_rule_binary_handler_s {
+      resolution_s operator()() const { return {.resolution = 1.0, .type = resolution_s::type_e::binary}; }
+    };
+
+    // `ASCII` — variable/fixed byte sequence to be rendered as text.
+    struct resolution_rule_ascii_handler_s {
+      resolution_s operator()() const { return {.resolution = 1.0, .type = resolution_s::type_e::ascii}; }
+    };
+
     this->strnum = lexeme[-(qi::char_('+') | qi::char_('-')) >> +(qi::digit | qi::char_(',') | qi::char_('.'))];
     this->num = this->strnum[_val = boost::phoenix::function<_detail::string_to_double_s>{}(qi::_1)];
 
-    resolution_rule_v0 = (this->num >> *qi::char_)[qi::_val = boost::phoenix::function<resolution_rule_v0_handler_s>{}(qi::_1)];
-    resolution_rule_v1 = (qi::uint_ >> '/' >> qi::uint_ >> *qi::char_)[qi::_val = boost::phoenix::function<resolution_rule_v1_handler_s>{}(qi::_1, qi::_2)];
+    resolution_rule_v0 =
+        (this->num >> *qi::char_)[qi::_val = boost::phoenix::function<resolution_rule_v0_handler_s>{}(qi::_1)];
+    resolution_rule_v1 =
+        (qi::uint_ >> '/' >> qi::uint_ >>
+         *qi::char_)[qi::_val = boost::phoenix::function<resolution_rule_v1_handler_s>{}(qi::_1, qi::_2)];
+    resolution_rule_v2 =
+        (this->num >> "states" >> *qi::char_)[qi::_val = boost::phoenix::function<resolution_rule_v2_handler_s>{}()];
+    resolution_rule_binary =
+        qi::lit("Binary")[qi::_val = boost::phoenix::function<resolution_rule_binary_handler_s>{}()];
+    resolution_rule_ascii = qi::lit("ASCII")[qi::_val = boost::phoenix::function<resolution_rule_ascii_handler_s>{}()];
 
-    this->rule = resolution_rule_v1 | resolution_rule_v0;
+    this->rule =
+        resolution_rule_binary | resolution_rule_ascii | resolution_rule_v2 | resolution_rule_v1 | resolution_rule_v0;
   }
 
-  qi::rule<It, resolution_s(), ascii::space_type> resolution_rule_v0, resolution_rule_v1;
+  qi::rule<It, resolution_s(), ascii::space_type> resolution_rule_v0, resolution_rule_v1, resolution_rule_v2,
+      resolution_rule_binary, resolution_rule_ascii;
 };
 }; // namespace resolution
 
@@ -330,7 +380,9 @@ std::optional<struct resolution_s> parseSpnResolution(const std::string &str);
 
 BOOST_FUSION_ADAPT_STRUCT(parsers::range_s, (double, min)(double, max)(std::string, other));
 BOOST_FUSION_ADAPT_STRUCT(parsers::size_s, (size_t, size_bytes)(size_t, size_bits));
-BOOST_FUSION_ADAPT_STRUCT(parsers::spn_fragments_s::spn_part_s, (size_t, byte_offset)(size_t, bit_offset)(size_t, size));
-BOOST_FUSION_ADAPT_STRUCT(parsers::spn_fragments_s, (std::vector<struct parsers::spn_fragments_s::spn_part_s>, spn_fragments));
+BOOST_FUSION_ADAPT_STRUCT(parsers::spn_fragments_s::spn_part_s,
+                          (size_t, byte_offset)(size_t, bit_offset)(size_t, size));
+BOOST_FUSION_ADAPT_STRUCT(parsers::spn_fragments_s,
+                          (std::vector<struct parsers::spn_fragments_s::spn_part_s>, spn_fragments));
 BOOST_FUSION_ADAPT_STRUCT(parsers::offset_s, (double, offset));
 BOOST_FUSION_ADAPT_STRUCT(parsers::resolution_s, (double, resolution));

src/xlsx.cpp

@@ -1,117 +1,28 @@
-#include <filesystem>
-#include <fstream>
-#include <limits>
 #include <map>
 #include <memory>
-#include <nlohmann/json.hpp>
-#include <sstream>
 #include <stdexcept>
-#include <unistd.h>
+#include <string>
 
 #define FMT_HEADER_ONLY
 #include <fmt/format.h>
-#include <fmt/ranges.h>
 
-// for XLSX files
 #include <xlnt/xlnt.hpp>
 
-// For sqlite
-#include "parsers.hpp"
+#include "j1939_db.hpp"
 #include "sqlite_modern_cpp.h"
 
-static void init_db(sqlite::database &db) {
-  try {
-    db << R"(
-      CREATE TABLE IF NOT EXISTS pgns (
-        id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
-        pgn INTEGER UNIQUE,
-        pg_label TEXT,
-        pg_acronym TEXT,
-        pg_descr TEXT,
-        edp INTEGER,
-        dp INTEGER,
-        pf INTEGER,
-        ps INTEGER,
-        pg_datalen INTEGER,
-        pg_priority INTEGER
-      );
-    )";
-
-    db << R"(
-      CREATE TABLE IF NOT EXISTS spns (
-        id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
-        pgn INTEGER,
-        spn INTEGER UNIQUE,
-        spn_name TEXT,
-        spn_pos TEXT,
-        spn_length INTEGER,
-        resolution REAL,
-        offset REAL,
-        data_range TEXT,
-        min_value REAL,
-        max_value REAL,
-        units TEXT,
-        slot_id TEXT,
-        slot_name TEXT,
-        spn_type TEXT
-      );
-    )";
-
-    db << R"(
-      CREATE TABLE IF NOT EXISTS spn_fragments (
-        id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
-        spn INTEGER,
-        pgn INTEGER,
-        byte_offset INTEGER,
-        bit_offset INTEGER,
-        size INTEGER
-      );
-    )";
-
-    db << "PRAGMA journal_mode = OFF";
-    db << "PRAGMA synchronous = OFF";
-    db << "PRAGMA foreign_keys = ON;";
-  } catch (const sqlite::sqlite_exception &e) {
-    throw;
-  }
-}
-
 std::unique_ptr<sqlite::database> parseXlsx(const std::string &file) {
   auto db_ptr = std::make_unique<sqlite::database>(":memory:");
   auto &db = *db_ptr;
 
-  init_db(db);
+  initJ1939Database(db);
 
-  static const std::map<std::string_view, std::tuple<size_t, std::string_view>>
-      pgn_mapping_table =
-          {
-              {"PGN", {5u, "pgn"}},
-              {"Parameter Group Label", {6u, "pg_label"}},
-              {"PG Acronym", {7u, "pg_acronym"}},
-              {"PG Description", {8u, "pg_descr"}},
-              {"EDP", {9u, "edp"}},
-              {"DP", {10u, "dp"}},
-              {"PF", {11u, "pf"}},
-              {"PS", {12u, "ps"}},
-              {"PG Data Length", {15u, "pg_datalen"}},
-              {"Default Priority", {16u, "pg_priority"}},
-          },
+  const auto &pgn_mapping = J1939MappingTables::pgn();
+  const auto &spn_mapping = J1939MappingTables::spn();
 
-      spn_mapping_table = {
-          {"SPN", {19u, "spn"}},
-          {"SPN Name", {20u, "spn_name"}},
-          {"SPN Position in PG", {18u, "spn_pos"}},
-          {"SPN Length", {22u, "spn_length"}},
-          {"Offset", {24u, "offset"}},
-          {"Data Range", {25u, "data_range"}},
-          {"Resolution", {23u, "resolution"}},
-          {"Units", {27u, "units"}},
-          {"SLOT Identifier", {28u, "slot_id"}},
-          {"SLOT Name", {29u, "slot_name"}},
-          {"SPN Type", {30u, "spn_type"}},
-      };
+  // Pre-build reverse lookup: column_index -> db_column_name
+  std::map<size_t, std::string> pgn_col_to_db, spn_col_to_db;
 
-  std::map<size_t, std::string> pgn_headers, spn_headers;
   xlnt::workbook wb;
 
   if (xlnt::path(file).exists()) {
@@ -123,249 +34,48 @@ std::unique_ptr<sqlite::database> parseXlsx(const std::string &file) {
 
     wb.active_sheet(0);
   } else {
-
     throw std::runtime_error(fmt::format("File {} does not exists", file));
   }
 
   xlnt::worksheet ws = wb.active_sheet();
   for (const auto &col : ws.columns()) {
     if (auto [cell, value] = std::pair{col[0], col[0].to_string()}; !value.empty()) {
-      if (pgn_mapping_table.contains(value)) {
-
-        auto it = pgn_headers.insert_or_assign(cell.column_index(), value);
-      } else if (spn_mapping_table.contains(value)) {
-
-        auto it = spn_headers.insert_or_assign(cell.column_index(), value);
+      auto col_idx = cell.column_index();
+      if (auto it = pgn_mapping.find(value); it != pgn_mapping.end() && col_idx == std::get<0u>(it->second)) {
+        pgn_col_to_db[col_idx] = std::string(std::get<1u>(it->second));
+      } else if (auto it2 = spn_mapping.find(value); it2 != spn_mapping.end() && col_idx == std::get<0u>(it2->second)) {
+        spn_col_to_db[col_idx] = std::string(std::get<1u>(it2->second));
       }
     }
   }
 
+  const auto pgn_insert_sql = buildPgnInsertSql();
+  const auto spn_insert_sql = buildSpnInsertSql();
+
+  db << "BEGIN TRANSACTION";
+
   for (const auto &row : ws.rows()) {
     std::map<std::string, std::string> pgn_row_map, spn_row_map;
     for (const auto &cell : row) {
       if (cell.row() != 1u) {
+        auto col_idx = cell.column_index();
+        auto value = cell.to_string();
+        if (value.empty()) continue;
 
-        if (pgn_headers.contains(cell.column_index())) {
-          if (!cell.to_string().empty()) {
-            pgn_row_map.insert_or_assign(
-                [&]() {
-                  std::string ret;
-
-                  for (const auto &[k, v] : pgn_mapping_table) {
-                    if (k == pgn_headers[cell.column_index()] && cell.column_index() == std::get<0u>(v)) {
-
-                      ret = std::get<1u>(v);
-                    }
-                  }
-
-                  return ret;
-                }(),
-
-                cell.to_string());
-          }
-        } else if (spn_headers.contains(cell.column_index())) {
-          if (!cell.to_string().empty()) {
-            spn_row_map.insert_or_assign(
-                [&]() {
-                  std::string ret;
-
-                  for (const auto &[k, v] : spn_mapping_table) {
-                    if (k == spn_headers[cell.column_index()] && cell.column_index() == std::get<0u>(v)) {
-
-                      ret = std::get<1u>(v);
-                    }
-                  }
-
-                  return ret;
-                }(),
-
-                cell.to_string());
-          }
+        if (auto it = pgn_col_to_db.find(col_idx); it != pgn_col_to_db.end()) {
+          pgn_row_map[it->second] = std::move(value);
+        } else if (auto it2 = spn_col_to_db.find(col_idx); it2 != spn_col_to_db.end()) {
+          spn_row_map[it2->second] = std::move(value);
         }
       }
     }
 
-    // Add rows to database
     if (!pgn_row_map.empty() && !spn_row_map.empty()) {
-      for (const auto &[k, v] : pgn_row_map) {
-        try {
-          auto ps = db << fmt::format(
-                        "INSERT OR REPLACE INTO pgns ({}) VALUES ({})",
-                        [&]() {
-                          std::string ret;
-
-                          for (const auto &[k, v] : pgn_mapping_table) {
-                            auto end = pgn_mapping_table.end();
-                            bool is_last = k == (--end)->first;
-                            ret += fmt::format("{}{}", std::get<1u>(v), !is_last ? ", " : "");
-                          }
-
-                          return ret;
-                        }(),
-
-                        [&]() {
-                          std::string ret;
-                          for (const auto &[k, _] : pgn_mapping_table) {
-                            auto end = pgn_mapping_table.end();
-                            bool is_last = k == (--end)->first;
-                            ret += !is_last ? "?, " : "?";
-                          }
-
-                          return ret;
-                        }());
-
-          for (const auto &[k, v] : pgn_mapping_table) {
-            ps << pgn_row_map[std::get<1u>(v).data()];
-          }
-
-          ps.execute();
-        } catch (const sqlite::sqlite_exception &e) {
-          if (e.get_extended_code() == SQLITE_CONSTRAINT_UNIQUE) {
-            continue;
-          }
-
-          throw;
-        }
-      }
-
-      struct {
-        bool parts_inserted_flag = false;
-        double min = 0.0, max = 0.0;
-        size_t size_bits = 0u;
-      } spn_settings_calculated;
-
-      // Pre-compute size_bits before the main loop (resolution calculation depends on it)
-      {
-        auto it = spn_mapping_table.find("SPN Length");
-        if (it != spn_mapping_table.end()) {
-          auto size = parsers::parseSpnSize(spn_row_map[std::get<1u>(it->second).data()]);
-          spn_settings_calculated.size_bits = size.has_value() ? size.value().size_bits : 0u;
-        }
-      }
-
-      for (const auto &[k, v] : spn_row_map) {
-        try {
-          auto ps = db << fmt::format(
-                        "INSERT OR REPLACE INTO spns ({}) VALUES ({})",
-                        [&]() {
-                          std::string ret;
-
-                          ret += "pgn, ";
-                          for (const auto &[k, v] : spn_mapping_table) {
-                            auto end = spn_mapping_table.end();
-                            bool is_last = k == (--end)->first;
-
-                            // Split 'data range' to two columns 'min' and 'max'
-                            if (std::get<1u>(v) == "data_range") {
-
-                              ret += is_last ? ", min_value, max_value" : "min_value, max_value, ";
-                            } else {
-
-                              ret += fmt::format("{}{}", std::get<1u>(v), !is_last ? ", " : "");
-                            }
-                          }
-
-                          return ret;
-                        }(),
-
-                        [&]() {
-                          std::string ret;
-
-                          ret += "?, ";
-                          for (const auto &[k, v] : spn_mapping_table) {
-                            auto end = spn_mapping_table.end();
-                            bool is_last = k == (--end)->first;
-
-                            // Split 'data range' to two columns 'min' and 'max'
-                            if (std::get<1u>(v) == "data_range") {
-
-                              ret += is_last ? ", ?, ?" : "?, ?, ";
-                            } else {
-
-                              ret += !is_last ? "?, " : "?";
-                            }
-                          }
-
-                          return ret;
-                        }());
-
-          ps << pgn_row_map["pgn"];
-          for (const auto &[k, v] : spn_mapping_table) {
-            if (std::get<1u>(v) == "data_range") { // Parse 'data range' string and split it to two columns 'min' and 'max'
-
-              auto range = parsers::parseSpnDataRange(spn_row_map[std::get<1u>(v).data()]);
-              if (range.has_value()) {
-                (spn_settings_calculated.min = range.value().min, spn_settings_calculated.max = range.value().max);
-
-                for (const auto &val : {range.value().min, range.value().max}) {
-                  ps << val;
-                }
-              } else {
-                ps << nullptr << nullptr;
-              }
-            } else if (std::get<1u>(v) == "offset") {
-
-              auto offset = parsers::parseSpnOffset(spn_row_map[std::get<1u>(v).data()]);
-              if (offset.has_value()) {
-                ps << offset.value().offset;
-              } else {
-                ps << nullptr;
-              }
-            } else if (std::get<1u>(v) == "spn_length") {
-
-              auto size = parsers::parseSpnSize(spn_row_map[std::get<1u>(v).data()]);
-              spn_settings_calculated.size_bits = size.has_value() ? size.value().size_bits : 0u;
-              ps << (size.has_value() ? size.value().size_bits : 0u);
-            } else if (std::get<1u>(v) == "resolution") {
-              double calculated = (spn_settings_calculated.max - spn_settings_calculated.min) / (std::pow(2.0f, spn_settings_calculated.size_bits) - 1u);
-
-              // If is discrete -- use calculated resolution
-              if (std::fabs(calculated - 1.0) < 1e-9) {
-
-                ps << calculated;
-              } else {
-
-                auto resolution = parsers::parseSpnResolution(spn_row_map[std::get<1u>(v).data()]);
-                ps << (resolution.has_value() ? resolution.value().resolution : 1.0f);
-              }
-            } else if (std::get<1u>(v) == "spn_pos") {
-              ps << spn_row_map[std::get<1u>(v).data()];
-            } else {
-
-              ps << spn_row_map[std::get<1u>(v).data()];
-            }
-          }
-
-          ps.execute();
-
-          // Insert SPN fragments after successful spns INSERT
-          if (!spn_settings_calculated.parts_inserted_flag) {
-            auto size = parsers::parseSpnSize(spn_row_map[std::get<1u>(spn_mapping_table.at("SPN Length")).data()]);
-
-            if (size.has_value()) {
-              auto spn_fragments = parsers::parseSpnPosition(size.value().size_bits, spn_row_map[std::get<1u>(spn_mapping_table.at("SPN Position in PG")).data()]);
-
-              if (spn_fragments.has_value()) {
-                auto spn = std::stoll(spn_row_map["spn"]);
-                for (const auto &part : spn_fragments.value().spn_fragments) {
-                  db << R"(INSERT OR REPLACE INTO spn_fragments (spn, pgn, byte_offset, bit_offset, size) VALUES (?, ?, ?, ?, ?);)" << spn << std::stoll(pgn_row_map["pgn"])
-                     << part.byte_offset << part.bit_offset << part.size;
-                }
-
-                spn_settings_calculated.parts_inserted_flag = true;
-              }
-            }
-          }
-        } catch (const sqlite::sqlite_exception &e) {
-          if (e.get_extended_code() == SQLITE_CONSTRAINT_UNIQUE) {
-            continue;
-          }
-
-          throw;
-        }
-      }
+      insertJ1939Row(db, pgn_insert_sql, spn_insert_sql, pgn_row_map, spn_row_map);
     }
   }
 
+  db << "COMMIT";
+
   return db_ptr;
 }