OlegShishlyannikov
/
canscope
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add regex search filters, CSV J1939 parser, 

lely-core CANopen dependency, SPN bit highlighting
This commit is contained in:
oleg 2026-04-13 10:05:56 +03:00
parent 22172309e1
commit 16466ed754
19 changed files with 54471 additions and 454 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
CMakeLists.txt
View File

@ -15,9 +15,23 @@ option(BUILD_SHARED_LIBS "Build using shared libraries" OFF)
include(cmake/dependencies.cmake)
project(canscope)
find_package(Boost REQUIRED COMPONENTS regex)
file(GLOB_RECURSE SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/*.hpp ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp)
include(cmake/lely.cmake)
if(NOT BUILD_SHARED_LIBS)
set(Boost_USE_STATIC_LIBS ON)
endif()
find_package(Boost REQUIRED COMPONENTS regex)
# Strip ICU dependencies from Boost::regex for static builds — ICU static libs
# are not available on Manjaro and we don't use ICU-dependent regex features.
if(NOT BUILD_SHARED_LIBS AND TARGET Boost::regex)
set_target_properties(Boost::regex PROPERTIES
INTERFACE_LINK_LIBRARIES ""
)
endif()
file(GLOB_RECURSE SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/*.hpp ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp)
add_executable(${CMAKE_PROJECT_NAME} ${SOURCES})
target_include_directories(${CMAKE_PROJECT_NAME} PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}
@ -31,5 +45,5 @@ target_include_directories(${CMAKE_PROJECT_NAME} PRIVATE
${clipp_SOURCE_DIR}/include
)
target_link_libraries(${CMAKE_PROJECT_NAME} PRIVATE ftxui::component ftxui::screen ftxui::dom tiny-process-library xlnt sqlite3_lib z Boost::regex ${Boost_LIBRARIES})
target_link_libraries(${CMAKE_PROJECT_NAME} PRIVATE ftxui::component ftxui::screen ftxui::dom tiny-process-library xlnt sqlite3_lib z Boost::regex ${Boost_LIBRARIES} lely::coapp lely::co lely::can lely::io2 lely::ev lely::util)

35
README.md
View File

@ -1,6 +1,6 @@
# {canscope}
CAN bus sniffer and SAE J1939 protocol analyzer. Reads CAN frames in `candump` format, decodes them using a J1939 Digital Annex (xlsx), and presents results in an interactive terminal UI or as JSON output.
CAN bus sniffer and SAE J1939 protocol analyzer. Reads CAN frames in `candump` format, decodes them using a J1939 Digital Annex (xlsx or csv), and presents results in an interactive terminal UI or as JSON output.
![demo](canscope-demo.gif)
@ -9,7 +9,7 @@ CAN bus sniffer and SAE J1939 protocol analyzer. Reads CAN frames in `candump` f
- **TUI mode** - full-screen interactive terminal interface (FTXUI). Multiple display modes per CAN ID: deployed, brief, verbose, manual, little-endian
- **Headless mode** - JSON output to stdout or file, for scripting and automation
- **Recording** - decoded J1939 SPN values saved to SQLite database with gzip compression and batch flushing
- **J1939 decoding** - PGN/SPN lookup, bit-level value extraction from payload
- **J1939 decoding** - PGN/SPN lookup, bit-level value extraction from payload. Supports xlsx and csv input formats
- **CAN playback** - replay recorded CAN frames
- **Custom SPN configuration** - per-parameter settings, parameter export
- **Real-time** - 30 fps UI refresh
@ -20,7 +20,7 @@ CAN bus sniffer and SAE J1939 protocol analyzer. Reads CAN frames in `candump` f
- clang++ with C++23 support
- CMake >= 3.13
- Ninja
- System libraries: boost (signals2, spirit, phoenix), sqlite3, zlib
- System libraries: boost (signals2, spirit, phoenix, regex), sqlite3, zlib, icu
Dependencies fetched automatically via CMake FetchContent:
@ -28,6 +28,7 @@ Dependencies fetched automatically via CMake FetchContent:
- [tiny-process-library](https://gitlab.com/eidheim/tiny-process-library) - subprocess management
- [sqlite_modern_cpp](https://github.com/SqliteModernCpp/sqlite_modern_cpp) - modern C++ SQLite wrapper
- [xlnt](https://github.com/xlnt-community/xlnt) - xlsx reading
- [lely-core](https://gitlab.com/lely_industries/lely-core) - CANopen protocol stack
- [fmt](https://github.com/fmtlib/fmt) - text formatting
- [nlohmann/json](https://github.com/nlohmann/json) - JSON library
- [clipp](https://github.com/muellan/clipp) - CLI argument parsing
@ -53,7 +54,9 @@ make clean # Remove all build artifacts
```bash
make build
./build/native/canscope -e "candump can0" -j1939 thirdparty/j1939da_2018.xlsx
./build/native/canscope -e "candump can0" -j1939-xlsx thirdparty/j1939da_2018.xlsx
# or with CSV (faster parsing)
./build/native/canscope -e "candump can0" -j1939-csv thirdparty/j1939da_2018.csv
```
### Docker (cross-platform)
@ -62,13 +65,13 @@ Works on Linux, macOS (?), and Windows (?). Requires only Docker and Make.
```bash
# TUI mode - local CAN interface
make docker-run ARGS='-e "candump can0" -j1939 thirdparty/j1939da_2018.xlsx'
make docker-run ARGS='-e "candump can0" -j1939-xlsx thirdparty/j1939da_2018.xlsx'
# TUI mode - remote CAN interface via SSH (no data if will ask password - use public key access or sshpass utility)
make docker-run ARGS='-e "ssh user@remote candump can0" -j1939 thirdparty/j1939da_2018.xlsx'
make docker-run ARGS='-e "ssh user@remote candump can0" -j1939-xlsx thirdparty/j1939da_2018.xlsx'
# Headless mode - create report about collected PGNs and SPNs
make docker-run ARGS='-hl -e "candump can0" -j1939 thirdparty/j1939da_2018.xlsx -of output.json'
make docker-run ARGS='-hl -e "candump can0" -j1939-xlsx thirdparty/j1939da_2018.xlsx -of output.json'
```
### Cross-compile for arm64
@ -84,22 +87,22 @@ Requires Docker. SSH keys from `~/.ssh` and `/etc/hosts` are forwarded into the
```bash
# TUI mode (default)
canscope -e "candump can0" -j1939 thirdparty/j1939da_2018.xlsx
canscope -e "candump can0" -j1939-xlsx thirdparty/j1939da_2018.xlsx
# Headless - JSON to stdout
canscope -hl -e "candump can0" -j1939 thirdparty/j1939da_2018.xlsx
canscope -hl -e "candump can0" -j1939-xlsx thirdparty/j1939da_2018.xlsx
# Headless - JSON to file
canscope -hl -e "candump can0" -j1939 thirdparty/j1939da_2018.xlsx -of output.json
canscope -hl -e "candump can0" -j1939-xlsx thirdparty/j1939da_2018.xlsx -of output.json
# Read from stdin (pipe)
candump can0 | canscope -j1939 thirdparty/j1939da_2018.xlsx
candump can0 | canscope -j1939-xlsx thirdparty/j1939da_2018.xlsx
# Record to SQLite database
canscope -rec -db recording.db -e "candump can0" -j1939 thirdparty/j1939da_2018.xlsx
canscope -rec -db recording.db -e "candump can0" -j1939-xlsx thirdparty/j1939da_2018.xlsx
# Record + TUI
canscope -rec -db recording.db -tui -e "candump can0" -j1939 thirdparty/j1939da_2018.xlsx
canscope -rec -db recording.db -tui -e "candump can0" -j1939-xlsx thirdparty/j1939da_2018.xlsx
```
> **Note:** J1939 decoding has only been tested with the Digital Annex 2018 edition. Other editions may work but are not guaranteed.
@ -108,7 +111,8 @@ canscope -rec -db recording.db -tui -e "candump can0" -j1939 thirdparty/j1939da_
| Flag | Long form | Description |
|------|-----------|-------------|
| `-j1939` | `--j1939-document` | **(required)** J1939 Digital Annex xlsx file |
| `-j1939-xlsx` | | J1939 Digital Annex xlsx file |
| `-j1939-csv` | | J1939 Digital Annex csv file (faster parsing) |
| `-e` | `--execute-command` | Command to read CAN frames from (e.g. `"candump can0"`) |
| `-hl` | `--headless` | Headless mode (no TUI) |
| `-of` | `--output-file` | Output file path (headless mode) |
@ -119,5 +123,6 @@ canscope -rec -db recording.db -tui -e "candump can0" -j1939 thirdparty/j1939da_
## Roadmap
- **CANopen protocol support** - CANopen decoding alongside J1939
- **NMEA 2000 protocol support** - NMEA 2000 decoding using [canboat](https://github.com/canboat/canboat) PGN database (JSON). Same 29-bit CAN ID as J1939, requires Fast Packet protocol implementation
- **CANopen protocol support** - CANopen decoding alongside J1939 (11-bit CAN ID, SDO/PDO/NMT)
- **Other small features and enhancements** - UI improvements, performance optimizations, additional export formats

13
cmake/dependencies.cmake
View File

@ -112,3 +112,16 @@ FetchContent_GetProperties(clipp)
if(NOT clipp_POPULATED)
FetchContent_Populate(clipp)
endif()
FetchContent_Declare(
lely_core
GIT_REPOSITORY https://gitlab.com/lely_industries/lely-core.git
GIT_TAG v2.3.5
GIT_SHALLOW TRUE
GIT_PROGRESS TRUE
)
FetchContent_GetProperties(lely_core)
if(NOT lely_core_POPULATED)
FetchContent_Populate(lely_core)
endif()

69
cmake/lely.cmake Normal file
View File

@ -0,0 +1,69 @@
set(LELY_INSTALL_DIR ${CMAKE_BINARY_DIR}/lely-install)
file(MAKE_DIRECTORY ${LELY_INSTALL_DIR}/include)
if(BUILD_SHARED_LIBS)
set(LELY_SHARED_FLAG "--enable-shared;--disable-static")
set(LELY_LIB_TYPE SHARED)
set(LELY_LIB_SUFFIX .so)
else()
set(LELY_SHARED_FLAG "--disable-shared;--enable-static")
set(LELY_LIB_TYPE STATIC)
set(LELY_LIB_SUFFIX .a)
endif()
set(LELY_CFLAGS "-Wno-error -Wno-implicit-function-declaration -Wno-unterminated-string-initialization -DATOMIC_VAR_INIT=")
set(LELY_CXXFLAGS "-Wno-error")
if(CMAKE_CROSSCOMPILING)
set(LELY_HOST_FLAG "--host=${CMAKE_C_COMPILER_TARGET}")
set(LELY_CROSS_FLAGS "--target=${CMAKE_C_COMPILER_TARGET}")
if(CMAKE_SYSROOT)
string(APPEND LELY_CROSS_FLAGS " --sysroot=${CMAKE_SYSROOT}")
endif()
set(LELY_CC "${CMAKE_C_COMPILER} ${LELY_CROSS_FLAGS}")
set(LELY_CXX "${CMAKE_CXX_COMPILER} ${LELY_CROSS_FLAGS}")
else()
set(LELY_HOST_FLAG "")
set(LELY_CC "")
set(LELY_CXX "")
endif()
include(ExternalProject)
set(LELY_BYPRODUCTS)
foreach(_lib can co coapp util ev io2)
list(APPEND LELY_BYPRODUCTS ${LELY_INSTALL_DIR}/lib/liblely-${_lib}${LELY_LIB_SUFFIX})
endforeach()
set(LELY_ENV_VARS "CFLAGS=${LELY_CFLAGS}" "CXXFLAGS=${LELY_CXXFLAGS}")
if(LELY_CC)
list(APPEND LELY_ENV_VARS "CC=${LELY_CC}" "CXX=${LELY_CXX}" "LDFLAGS=-fuse-ld=lld")
endif()
ExternalProject_Add(lely_core_ext
SOURCE_DIR ${lely_core_SOURCE_DIR}
BUILD_IN_SOURCE TRUE
CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${LELY_ENV_VARS}
autoreconf -i ${lely_core_SOURCE_DIR}
COMMAND ${CMAKE_COMMAND} -E env ${LELY_ENV_VARS}
${lely_core_SOURCE_DIR}/configure
--prefix=${LELY_INSTALL_DIR}
${LELY_HOST_FLAG}
${LELY_SHARED_FLAG}
--disable-dependency-tracking
--disable-python
--disable-tests
--disable-doc
BUILD_COMMAND ${CMAKE_COMMAND} -E env ${LELY_ENV_VARS}
make -j
INSTALL_COMMAND make install
BUILD_BYPRODUCTS ${LELY_BYPRODUCTS}
)
foreach(_lib can co coapp util ev io2)
add_library(lely::${_lib} ${LELY_LIB_TYPE} IMPORTED GLOBAL)
set_target_properties(lely::${_lib} PROPERTIES
IMPORTED_LOCATION ${LELY_INSTALL_DIR}/lib/liblely-${_lib}${LELY_LIB_SUFFIX}
INTERFACE_INCLUDE_DIRECTORIES ${LELY_INSTALL_DIR}/include
)
add_dependencies(lely::${_lib} lely_core_ext)
endforeach()

4
docker/Dockerfile
View File

@ -5,8 +5,12 @@ RUN pacman -Syu --noconfirm && \
clang \
cmake \
make \
autoconf \
automake \
libtool \
git \
boost \
icu \
zlib && \
pacman -Scc --noconfirm

4
docker/Dockerfile.cross
View File

@ -8,6 +8,10 @@ RUN pacman -Syu --noconfirm && \
ninja \
git \
openssh \
autoconf \
automake \
libtool \
make \
&& pacman -Scc --noconfirm
WORKDIR /app

5
docker/Dockerfile.dev
View File

@ -9,7 +9,12 @@ RUN pacman -Syu --noconfirm && \
git \
openssh \
sshpass \
autoconf \
automake \
libtool \
make \
boost \
icu \
sqlite3 \
zlib && \
pacman -Scc --noconfirm

1
docker/Dockerfile.sysroot
View File

@ -3,6 +3,7 @@ FROM --platform=linux/arm64 manjarolinux/base:latest
RUN pacman -Syu --noconfirm && \
pacman -S --noconfirm \
boost \
icu \
zlib \
gcc && \
pacman -Scc --noconfirm

220
src/can_player_dialog.cpp
View File

@ -29,6 +29,8 @@
#include <linux/can.h>
#include <linux/can/raw.h>
#include <boost/regex.hpp>
#define FMT_HEADER_ONLY
#include <fmt/format.h>
@ -45,6 +47,7 @@ ftxui::Component makeCanPlayerDialog(ftxui::ScreenInteractive *scr, signals_map_
Impl(ftxui::ScreenInteractive *scr, signals_map_t &smap, bool &is_ready) {
static sqlite::database *database = nullptr;
static float canbus_player_focus_relative = 0;
static std::string player_filter_text;
auto pgnContainer = ftxui::Container::Vertical({});
@ -192,12 +195,14 @@ ftxui::Component makeCanPlayerDialog(ftxui::ScreenInteractive *scr, signals_map_
->connect([](const std::vector<can_frame_update_s> &batch) {
for (const auto &entry : batch) {
uint32_t pgn_num = 0;
if (entry.canid.size() >= 6) {
auto pgn_str = entry.canid.substr(entry.canid.size() >= 8 ? entry.canid.size() - 6 : 2, 4);
std::stringstream ss;
ss << std::hex << pgn_str;
ss >> pgn_num;
}
received_pgns.insert(pgn_num);
if (pgs.contains(pgn_num) && pgs[pgn_num].forward && pgs[pgn_num].is_running) {
auto &pg = pgs[pgn_num];
@ -405,6 +410,7 @@ ftxui::Component makeCanPlayerDialog(ftxui::ScreenInteractive *scr, signals_map_
{"offset", spn_params.offset},
};
}(),
false, -100, ExpanderImpl::Root()) |
ftxui::Renderer([](ftxui::Element inner) {
return ftxui::hbox({
@ -433,21 +439,52 @@ ftxui::Component makeCanPlayerDialog(ftxui::ScreenInteractive *scr, signals_map_
fmt::format("{0:#b}", spn_params.raw))),
}),
ftxui::hbox({
ftxui::text("PG payload: ") | ftxui::bold |
ftxui::color(ftxui::Color::Cyan),
ftxui::text(fmt::format(
"[{}]",
[&]() {
std::string ret;
for (const auto &byte :
spn_params.pg_ref->payload) {
ret += fmt::format("{0:#010b} ", byte);
}
[&]() -> ftxui::Element {
// Build bit mask for this SPN's fragments
const auto &payload = spn_params.pg_ref->payload;
std::vector<bool> highlight(payload.size() * 8, false);
for (const auto &frag : spn_params.fragments) {
int32_t start_bit =
frag.byte_offset * 8 + frag.bit_offset;
return ret;
}())),
}),
for (int32_t b = 0; b < frag.size; ++b) {
auto idx = static_cast<size_t>(start_bit + b);
if (idx < highlight.size()) {
highlight[idx] = true;
}
}
}
ftxui::Elements parts;
parts.push_back(ftxui::text("PG payload: ") |
ftxui::bold |
ftxui::color(ftxui::Color::Cyan));
parts.push_back(ftxui::text("["));
for (size_t i = 0; i < payload.size(); ++i) {
parts.push_back(ftxui::text("0b"));
for (int32_t bit = 7; bit >= 0; --bit) {
bool is_set = (payload[i] >> bit) & 1;
bool is_spn = highlight[i * 8 + bit];
auto ch = ftxui::text(is_set ? "1" : "0");
if (is_spn) {
ch = ch | ftxui::color(ftxui::Color::Red) |
ftxui::bold;
}
parts.push_back(ch);
}
parts.push_back(ftxui::text(" "));
}
parts.push_back(ftxui::text("]"));
return ftxui::hbox(std::move(parts));
}(),
ftxui::separatorEmpty(),
});
@ -461,7 +498,7 @@ ftxui::Component makeCanPlayerDialog(ftxui::ScreenInteractive *scr, signals_map_
});
};
pgnContainer->Add(ftxui::Container::Vertical({
auto pgn_entry = ftxui::Container::Vertical({
ftxui::Container::Horizontal({
ftxui::Checkbox({
.checked = &pg_ref.selected,
@ -621,6 +658,18 @@ ftxui::Component makeCanPlayerDialog(ftxui::ScreenInteractive *scr, signals_map_
spnContainer,
}) | ftxui::border,
&pg_ref.selected),
});
pgnContainer->Add(ftxui::Maybe(pgn_entry, [&pg_ref = pgs[pgn]]() -> bool {
if (player_filter_text.empty())
return true;
try {
boost::regex re(player_filter_text, boost::regex_constants::icase);
std::string subject = fmt::format("0x{:x} {}", pg_ref.pgn, pg_ref.label);
return boost::regex_search(subject, re);
} catch (...) {
return true;
}
}));
};
@ -634,57 +683,106 @@ ftxui::Component makeCanPlayerDialog(ftxui::ScreenInteractive *scr, signals_map_
}
auto main = ftxui::Container::Vertical({
ftxui::Input({
.content = &player_filter_text,
.placeholder = "regex filter ...",
.transform = [](ftxui::InputState state) -> ftxui::Element {
bool valid = true;
if (!player_filter_text.empty()) {
try {
boost::regex(player_filter_text, boost::regex_constants::icase);
} catch (...) {
valid = false;
}
}
state.element |= (!valid ? ftxui::color(ftxui::Color::Red) : ftxui::nothing) |
(state.focused ? ftxui::color(ftxui::Color::Cyan) : ftxui::nothing) |
(state.hovered ? ftxui::bold : ftxui::nothing);
return ftxui::hbox({
ftxui::text(" Search: [ "),
state.element |
(state.hovered || state.focused ? ftxui::bgcolor(ftxui::Color::Grey11) : ftxui::nothing) |
ftxui::xflex,
ftxui::text(" ]"),
});
},
.multiline = false,
}),
ftxui::Renderer([]() { return ftxui::separator(); }),
(pgnContainer | ftxui::Renderer([](ftxui::Element inner) {
return inner | ftxui::focusPositionRelative(0, canbus_player_focus_relative) | ftxui::vscroll_indicator |
ftxui::frame | ftxui::flex;
})) |
ftxui::CatchEvent([pgnContainer](ftxui::Event event) {
static const auto increment_focus = [pgnContainer = pgnContainer]() {
if (auto n = pgnContainer->ChildCount(); n > 0)
canbus_player_focus_relative =
std::clamp(canbus_player_focus_relative + 1.0f / static_cast<float>(n), 0.0f, 1.0f);
};
static const auto decrement_focus = [pgnContainer = pgnContainer]() {
if (auto n = pgnContainer->ChildCount(); n > 0)
canbus_player_focus_relative =
std::clamp(canbus_player_focus_relative - 1.0f / static_cast<float>(n), 0.0f, 1.0f);
};
if (!database) {
return true;
}
if (event.is_mouse()) {
switch (static_cast<enum ftxui::Mouse::Button>(event.mouse().button)) {
case ftxui::Mouse::Button::WheelDown: {
increment_focus();
return true;
} break;
case ftxui::Mouse::Button::WheelUp: {
decrement_focus();
return true;
} break;
default:
break;
}
} else if (!event.is_character()) {
if (event == ftxui::Event::ArrowDown) {
increment_focus();
return true;
} else if (event == ftxui::Event::ArrowUp) {
decrement_focus();
return true;
}
}
return false;
}),
})),
});
Add({main});
Add({main | ftxui::CatchEvent([pgnContainer](ftxui::Event event) {
if (!database)
return true;
const auto scroll_step = []() -> float {
size_t visible_lines = 0;
boost::regex re;
bool has_filter = !player_filter_text.empty();
if (has_filter) {
try {
re = boost::regex(player_filter_text, boost::regex_constants::icase);
} catch (...) {
has_filter = false;
}
}
for (const auto &[_, pg] : pgs) {
if (has_filter) {
std::string subject = fmt::format("0x{:x} {}", pg.pgn, pg.label);
if (!boost::regex_search(subject, re)) {
continue;
}
}
++visible_lines;
if (pg.selected) {
visible_lines += 15;
}
}
return visible_lines > 0 ? 1.0f / static_cast<float>(visible_lines) : 0.03f;
};
if (event.is_mouse()) {
switch (static_cast<enum ftxui::Mouse::Button>(event.mouse().button)) {
case ftxui::Mouse::Button::WheelDown: {
canbus_player_focus_relative = std::clamp(canbus_player_focus_relative + scroll_step(), 0.0f, 1.0f);
return true;
}
case ftxui::Mouse::Button::WheelUp: {
canbus_player_focus_relative = std::clamp(canbus_player_focus_relative - scroll_step(), 0.0f, 1.0f);
return true;
}
default:
break;
}
} else if (!event.is_character()) {
if (event == ftxui::Event::ArrowDown) {
canbus_player_focus_relative = std::clamp(canbus_player_focus_relative + scroll_step(), 0.0f, 1.0f);
return true;
} else if (event == ftxui::Event::ArrowUp) {
canbus_player_focus_relative = std::clamp(canbus_player_focus_relative - scroll_step(), 0.0f, 1.0f);
return true;
}
}
return false;
})});
}
};

36
src/canid_unit.cpp
View File

@ -1,5 +1,6 @@
// #include <algorithm>
// #include <cmath>
#include <boost/regex.hpp>
#include <cstdint>
#include <ftxui/component/component.hpp>
#include <ftxui/component/component_options.hpp>
@ -330,16 +331,19 @@ void CanIDUnit::update(const can_frame_data_s &data, const can_frame_diff_s &dif
total_bits += bc;
}
}
if (settings.big_endian && total_bits > 8) {
int64_t swapped = 0;
size_t total_bytes = (total_bits + 7) / 8;
for (size_t i = 0; i < total_bytes; ++i)
swapped |= ((result >> (i * 8)) & 0xFF) << ((total_bytes - 1 - i) * 8);
result = swapped;
}
double spn_val = static_cast<double>(result) * resolution + offset_val;
double spn_val = static_cast<double>(result) * resolution + offset_val;
nlohmann::json::array_t frags_json;
for (size_t fi = 0; fi < settings.fragments.size(); ++fi) {
const auto &f = settings.fragments[fi];
int32_t bo = 0, bi = 0, bc = 0;
@ -424,8 +428,10 @@ void CanIDUnit::update(const can_frame_data_s &data, const can_frame_diff_s &dif
for (const auto &[spn_k, spn_v] : spns_arr_v.items()) {
if (spn_k == "SPN name") {
std::string spn_name = spn_v.get<std::string>();
if (spn_name.find("(custom)") != std::string::npos)
if (spn_name.find("(custom)") != std::string::npos) {
continue;
}
auto &spn_map = std::get<2u>(s_canbus_parameters_export_map_[m_canid_]);
spn_map.insert_or_assign(spn_name, std::make_tuple(false, false, spns_arr_v));
@ -529,7 +535,19 @@ void CanIDUnit::update(const can_frame_data_s &data, const can_frame_diff_s &dif
!selectors_container->ChildCount();
}));
m_cansettings_dialog_->Add(container);
m_cansettings_dialog_->Add(ftxui::Maybe(container, [this]() -> bool {
if (!s_export_filter_text_ || s_export_filter_text_->empty()) {
return true;
}
try {
boost::regex re(*s_export_filter_text_, boost::regex_constants::icase);
std::string subject = m_canid_ + " " + getLabel();
return boost::regex_search(subject, re);
} catch (...) {
return true;
}
}));
}
// Add custom SPNs to export dialog if not already present (tracked by tag_id)
@ -579,6 +597,7 @@ void CanIDUnit::update(const can_frame_data_s &data, const can_frame_diff_s &dif
.transform =
[this, tag_id](const ftxui::EntryState state) {
std::string display_name = "custom";
if (m_spnSettingsMap_ && m_spnSettingsMap_->contains(m_canid_) &&
(*m_spnSettingsMap_)[m_canid_].contains(tag_id)) {
auto &s = (*m_spnSettingsMap_)[m_canid_][tag_id];
@ -619,26 +638,33 @@ void CanIDUnit::update(const can_frame_data_s &data, const can_frame_diff_s &dif
}
// Rebuild FromLive for custom SPN export entries when fragment count changes
if (m_spnSettingsMap_ && m_spnSettingsMap_->contains(m_canid_) && !m_data_verbose_->is_null() && m_data_verbose_->contains("SPNs")) {
if (m_spnSettingsMap_ && m_spnSettingsMap_->contains(m_canid_) && !m_data_verbose_->is_null() &&
m_data_verbose_->contains("SPNs")) {
for (auto &[tag_id, pair] : m_export_custom_containers_) {
auto &[wrapper, prev_frag_count] = pair;
size_t cur_frag_count = 0;
if ((*m_spnSettingsMap_)[m_canid_].contains(tag_id)) {
cur_frag_count = (*m_spnSettingsMap_)[m_canid_][tag_id].fragments.size();
}
if (cur_frag_count != prev_frag_count) {
wrapper->DetachAllChildren();
std::string search_name;
if ((*m_spnSettingsMap_)[m_canid_].contains(tag_id)) {
search_name = (*m_spnSettingsMap_)[m_canid_][tag_id].spn_name + " (custom)";
}
const auto &spns = (*m_data_verbose_)["SPNs"];
for (size_t i = 0; i < spns.size(); ++i) {
if (spns[i].contains("SPN name") && spns[i]["SPN name"].get<std::string>() == search_name) {
wrapper->Add(FromLive(m_data_verbose_, nlohmann::json::json_pointer("/SPNs/" + std::to_string(i)), false, -100, ExpanderImpl::Root()));
wrapper->Add(FromLive(m_data_verbose_, nlohmann::json::json_pointer("/SPNs/" + std::to_string(i)), false,
-100, ExpanderImpl::Root()));
break;
}
}
prev_frag_count = cur_frag_count;
}
}

18
src/canid_unit.hpp
View File

@ -48,6 +48,16 @@ public:
inline ftxui::Component getSpnSettingsForm() { return m_spnSettingsForm_; }
inline const auto &getParametersExportMap() const { return s_canbus_parameters_export_map_; }
static inline std::string *s_export_filter_text_ = nullptr;
static inline std::map<
/* canid */ std::string,
std::tuple</* deployed flag */ bool, /* has data flag */ bool,
/* Selected spns to export */
std::map</* spn name */ std::string,
std::tuple</* deployed */ bool, /* selected */ bool, /* data */ nlohmann::json>>>>
s_canbus_parameters_export_map_ = {};
void update(const can_frame_data_s &data, const can_frame_diff_s &diff, std::shared_ptr<nlohmann::json> verbose,
std::shared_ptr<nlohmann::json> brief);
@ -71,12 +81,4 @@ private:
size_t m_last_brief_spn_count_ = 0;
ftxui::Component m_export_selectors_;
std::map<int32_t, std::pair<ftxui::Component, size_t>> m_export_custom_containers_;
static inline std::map<
/* canid */ std::string,
std::tuple</* deployed flag */ bool, /* has data flag */ bool,
/* Selected spns to export */
std::map</* spn name */ std::string,
std::tuple</* deployed */ bool, /* selected */ bool, /* data */ nlohmann::json>>>>
s_canbus_parameters_export_map_ = {};
};

117
src/csv.cpp Normal file
View File

@ -0,0 +1,117 @@
#include <fstream>
#include <map>
#include <memory>
#include <sstream>
#include <stdexcept>
#include <string>
#include <vector>
#define FMT_HEADER_ONLY
#include <fmt/format.h>
#include "j1939_db.hpp"
#include "sqlite_modern_cpp.h"
static std::vector<std::string> parseCsvLine(const std::string &line) {
std::vector<std::string> fields;
std::string field;
bool in_quotes = false;
for (size_t i = 0; i < line.size(); ++i) {
char c = line[i];
if (in_quotes) {
if (c == '"') {
if (i + 1 < line.size() && line[i + 1] == '"') {
field += '"';
++i;
} else {
in_quotes = false;
}
} else {
field += c;
}
} else {
if (c == '"') {
in_quotes = true;
} else if (c == ',') {
fields.push_back(std::move(field));
field.clear();
} else {
field += c;
}
}
}
fields.push_back(std::move(field));
return fields;
}
std::unique_ptr<sqlite::database> parseCsv(const std::string &file) {
std::ifstream ifs(file);
if (!ifs.is_open()) {
throw std::runtime_error(fmt::format("Cannot open file {}", file));
}
auto db_ptr = std::make_unique<sqlite::database>(":memory:");
auto &db = *db_ptr;
initJ1939Database(db);
const auto &pgn_mapping = J1939MappingTables::pgn();
const auto &spn_mapping = J1939MappingTables::spn();
// Read header line and build column_index -> db_column_name maps
std::map<size_t, std::string> pgn_col_to_db, spn_col_to_db;
std::string header_line;
if (!std::getline(ifs, header_line)) {
throw std::runtime_error("CSV file is empty");
}
auto headers = parseCsvLine(header_line);
for (size_t i = 0; i < headers.size(); ++i) {
const auto &h = headers[i];
if (auto it = pgn_mapping.find(h); it != pgn_mapping.end()) {
pgn_col_to_db[i] = std::string(std::get<1u>(it->second));
} else if (auto it2 = spn_mapping.find(h); it2 != spn_mapping.end()) {
spn_col_to_db[i] = std::string(std::get<1u>(it2->second));
}
}
const auto pgn_insert_sql = buildPgnInsertSql();
const auto spn_insert_sql = buildSpnInsertSql();
db << "BEGIN TRANSACTION";
std::string line;
while (std::getline(ifs, line)) {
if (line.empty())
continue;
auto fields = parseCsvLine(line);
std::map<std::string, std::string> pgn_row_map, spn_row_map;
for (size_t i = 0; i < fields.size(); ++i) {
if (fields[i].empty())
continue;
if (auto it = pgn_col_to_db.find(i); it != pgn_col_to_db.end()) {
pgn_row_map[it->second] = std::move(fields[i]);
} else if (auto it2 = spn_col_to_db.find(i); it2 != spn_col_to_db.end()) {
spn_row_map[it2->second] = std::move(fields[i]);
}
}
if (!pgn_row_map.empty() && !spn_row_map.empty()) {
insertJ1939Row(db, pgn_insert_sql, spn_insert_sql, pgn_row_map, spn_row_map);
}
}
db << "COMMIT";
return db_ptr;
}

9
src/filedialog.cpp
View File

@ -77,6 +77,7 @@ ftxui::Component makeFileDialog(ftxui::ScreenInteractive *scr, signals_map_t &sm
ftxui::Container::Vertical({
ftxui::Renderer([&]() -> ftxui::Element { return ftxui::text("Export file"); }) |
ftxui::color(ftxui::Color::Red) | ftxui::hcenter,
ftxui::Renderer([]() { return ftxui::separator(); }),
ftxui::Container::Horizontal({
ftxui::Renderer([]() -> ftxui::Element {
@ -90,9 +91,9 @@ ftxui::Component makeFileDialog(ftxui::ScreenInteractive *scr, signals_map_t &sm
}),
}),
ftxui::Renderer([]() { return ftxui::separatorEmpty(); }),
ftxui::Renderer([]() { return ftxui::separator(); }),
entryList | ftxui::vscroll_indicator | ftxui::frame | ftxui::flex,
ftxui::Renderer([]() { return ftxui::separator(); }),
ftxui::Container::Horizontal({
ftxui::Button({
@ -104,7 +105,7 @@ ftxui::Component makeFileDialog(ftxui::ScreenInteractive *scr, signals_map_t &sm
},
.transform = [this](const ftxui::EntryState &state) -> ftxui::Element {
return ftxui::text(" >[Export]< ") | (state.focused ? ftxui::bold : ftxui::nothing) |
return ftxui::text(" >[export]< ") | (state.focused ? ftxui::bold : ftxui::nothing) |
ftxui::color(ftxui::Color::Cyan) |
(state.focused ? ftxui::bgcolor(ftxui::Color::Grey11) : ftxui::nothing);
},
@ -113,7 +114,7 @@ ftxui::Component makeFileDialog(ftxui::ScreenInteractive *scr, signals_map_t &sm
ftxui::Button({
.on_click = [&]() { shown = false; },
.transform = [this](const ftxui::EntryState &state) -> ftxui::Element {
return ftxui::text(" >[Cancel]< ") | (state.focused ? ftxui::bold : ftxui::nothing) |
return ftxui::text(" >[cancel]< ") | (state.focused ? ftxui::bold : ftxui::nothing) |
ftxui::color(ftxui::Color::Cyan) |
(state.focused ? ftxui::bgcolor(ftxui::Color::Grey11) : ftxui::nothing);
},

179
src/j1939_db.cpp Normal file
View File

@ -0,0 +1,179 @@
#include "j1939_db.hpp"
#include "parsers.hpp"
#include <cmath>
#define FMT_HEADER_ONLY
#include <fmt/format.h>
void initJ1939Database(sqlite::database &db) {
db << R"(
CREATE TABLE IF NOT EXISTS pgns (
id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
pgn INTEGER UNIQUE,
pg_label TEXT,
pg_acronym TEXT,
pg_descr TEXT,
edp INTEGER,
dp INTEGER,
pf INTEGER,
ps INTEGER,
pg_datalen INTEGER,
pg_priority INTEGER
);
)";
db << R"(
CREATE TABLE IF NOT EXISTS spns (
id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
pgn INTEGER,
spn INTEGER UNIQUE,
spn_name TEXT,
spn_pos TEXT,
spn_length INTEGER,
resolution REAL,
offset REAL,
data_range TEXT,
min_value REAL,
max_value REAL,
units TEXT,
slot_id TEXT,
slot_name TEXT,
spn_type TEXT
);
)";
db << R"(
CREATE TABLE IF NOT EXISTS spn_fragments (
id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
spn INTEGER,
pgn INTEGER,
byte_offset INTEGER,
bit_offset INTEGER,
size INTEGER
);
)";
db << "PRAGMA journal_mode = OFF";
db << "PRAGMA synchronous = OFF";
db << "PRAGMA foreign_keys = ON;";
}
std::string buildPgnInsertSql() {
const auto &pgn_mapping = J1939MappingTables::pgn();
std::string cols, placeholders;
for (const auto &[k, v] : pgn_mapping) {
if (!cols.empty()) { cols += ", "; placeholders += ", "; }
cols += std::get<1u>(v);
placeholders += "?";
}
return fmt::format("INSERT OR REPLACE INTO pgns ({}) VALUES ({})", cols, placeholders);
}
std::string buildSpnInsertSql() {
const auto &spn_mapping = J1939MappingTables::spn();
std::string cols = "pgn", placeholders = "?";
for (const auto &[k, v] : spn_mapping) {
cols += ", ";
placeholders += ", ";
if (std::get<1u>(v) == "data_range") {
cols += "min_value, max_value";
placeholders += "?, ?";
} else {
cols += std::get<1u>(v);
placeholders += "?";
}
}
return fmt::format("INSERT OR REPLACE INTO spns ({}) VALUES ({})", cols, placeholders);
}
void insertJ1939Row(sqlite::database &db, const std::string &pgn_insert_sql, const std::string &spn_insert_sql,
std::map<std::string, std::string> &pgn_row_map, std::map<std::string, std::string> &spn_row_map) {
const auto &pgn_mapping = J1939MappingTables::pgn();
const auto &spn_mapping = J1939MappingTables::spn();
// Insert PGN
try {
auto ps = db << pgn_insert_sql;
for (const auto &[k, v] : pgn_mapping) {
ps << pgn_row_map[std::get<1u>(v).data()];
}
ps.execute();
} catch (const sqlite::sqlite_exception &e) {
if (e.get_extended_code() != SQLITE_CONSTRAINT_UNIQUE) throw;
}
// Insert SPN
struct {
bool parts_inserted_flag = false;
double min = 0.0, max = 0.0;
size_t size_bits = 0u;
} calc;
// Pre-compute size_bits
{
auto it = spn_mapping.find("SPN Length");
if (it != spn_mapping.end()) {
auto size = parsers::parseSpnSize(spn_row_map[std::get<1u>(it->second).data()]);
calc.size_bits = size.has_value() ? size.value().size_bits : 0u;
}
}
for (const auto &[k, v] : spn_row_map) {
try {
auto ps = db << spn_insert_sql;
ps << pgn_row_map["pgn"];
for (const auto &[k, v] : spn_mapping) {
if (std::get<1u>(v) == "data_range") {
auto range = parsers::parseSpnDataRange(spn_row_map[std::get<1u>(v).data()]);
if (range.has_value()) {
calc.min = range.value().min;
calc.max = range.value().max;
ps << range.value().min << range.value().max;
} else {
ps << nullptr << nullptr;
}
} else if (std::get<1u>(v) == "offset") {
auto offset = parsers::parseSpnOffset(spn_row_map[std::get<1u>(v).data()]);
ps << (offset.has_value() ? offset.value().offset : 0.0);
} else if (std::get<1u>(v) == "spn_length") {
auto size = parsers::parseSpnSize(spn_row_map[std::get<1u>(v).data()]);
calc.size_bits = size.has_value() ? size.value().size_bits : 0u;
ps << calc.size_bits;
} else if (std::get<1u>(v) == "resolution") {
double calculated = (calc.max - calc.min) / (std::pow(2.0, calc.size_bits) - 1.0);
if (std::fabs(calculated - 1.0) < 1e-9) {
ps << calculated;
} else {
auto resolution = parsers::parseSpnResolution(spn_row_map[std::get<1u>(v).data()]);
ps << (resolution.has_value() ? resolution.value().resolution : 1.0);
}
} else {
ps << spn_row_map[std::get<1u>(v).data()];
}
}
ps.execute();
// Insert SPN fragments
if (!calc.parts_inserted_flag) {
auto size = parsers::parseSpnSize(spn_row_map[std::get<1u>(spn_mapping.at("SPN Length")).data()]);
if (size.has_value()) {
auto spn_fragments = parsers::parseSpnPosition(
size.value().size_bits, spn_row_map[std::get<1u>(spn_mapping.at("SPN Position in PG")).data()]);
if (spn_fragments.has_value()) {
auto spn = std::stoll(spn_row_map["spn"]);
for (const auto &part : spn_fragments.value().spn_fragments) {
db << R"(INSERT OR REPLACE INTO spn_fragments (spn, pgn, byte_offset, bit_offset, size) VALUES (?, ?, ?, ?, ?);)"
<< spn << std::stoll(pgn_row_map["pgn"]) << part.byte_offset << part.bit_offset << part.size;
}
calc.parts_inserted_flag = true;
}
}
}
} catch (const sqlite::sqlite_exception &e) {
if (e.get_extended_code() != SQLITE_CONSTRAINT_UNIQUE) throw;
}
}
}

56
src/j1939_db.hpp Normal file
View File

@ -0,0 +1,56 @@
#pragma once
#include <map>
#include <memory>
#include <string>
#include <string_view>
#include <tuple>
#include "sqlite_modern_cpp.h"
struct J1939MappingTables {
static const std::map<std::string_view, std::tuple<size_t, std::string_view>> &pgn() {
static const std::map<std::string_view, std::tuple<size_t, std::string_view>> t = {
{"PGN", {5u, "pgn"}},
{"Parameter Group Label", {6u, "pg_label"}},
{"PG Acronym", {7u, "pg_acronym"}},
{"PG Description", {8u, "pg_descr"}},
{"EDP", {9u, "edp"}},
{"DP", {10u, "dp"}},
{"PF", {11u, "pf"}},
{"PS", {12u, "ps"}},
{"PG Data Length", {15u, "pg_datalen"}},
{"Default Priority", {16u, "pg_priority"}},
};
return t;
}
static const std::map<std::string_view, std::tuple<size_t, std::string_view>> &spn() {
static const std::map<std::string_view, std::tuple<size_t, std::string_view>> t = {
{"SPN", {19u, "spn"}},
{"SPN Name", {20u, "spn_name"}},
{"SPN Position in PG", {18u, "spn_pos"}},
{"SPN Length", {22u, "spn_length"}},
{"Offset", {24u, "offset"}},
{"Data Range", {25u, "data_range"}},
{"Resolution", {23u, "resolution"}},
{"Units", {27u, "units"}},
{"SLOT Identifier", {28u, "slot_id"}},
{"SLOT Name", {29u, "slot_name"}},
{"SPN Type", {30u, "spn_type"}},
};
return t;
}
};
void initJ1939Database(sqlite::database &db);
// Build INSERT SQL for pgns table (call once, reuse)
std::string buildPgnInsertSql();
// Build INSERT SQL for spns table (call once, reuse)
std::string buildSpnInsertSql();
// Insert a row into the database. pgn_row_map/spn_row_map are column_name -> value.
void insertJ1939Row(sqlite::database &db, const std::string &pgn_insert_sql, const std::string &spn_insert_sql,
std::map<std::string, std::string> &pgn_row_map, std::map<std::string, std::string> &spn_row_map);

37
src/main.cpp
View File

@ -46,14 +46,15 @@ int32_t main(int32_t argc, char *argv[]) {
static std::atomic<sqlite::database *> j1939_db{nullptr};
static std::stop_source aggregator_task_stop, refresh_task_stop, headless_task_stop;
static TinyProcessLib::Process *p = nullptr;
std::future<void> xlsx_parser_task, headless_task;
std::future<void> j1939_parser_task, headless_task;
extern std::unique_ptr<sqlite::database> parseXlsx(const std::string &file);
extern std::unique_ptr<sqlite::database> parseCsv(const std::string &file);
static std::unique_ptr<sqlite::database> j1939_db_owner;
static std::unique_ptr<Recorder> recorder;
static std::unique_ptr<HeadlessHandler> headless_handler;
static struct {
std::string docfile, command = "", output_file = "", record_db_path = "";
std::string xlsx_file, csv_file, command = "", output_file = "", record_db_path = "";
bool show_help = false, headless_mode = false, sync_to_server = false, record_mode = false, tui_mode = false;
} cli_opts;
@ -90,16 +91,26 @@ int32_t main(int32_t argc, char *argv[]) {
clipp::option("-e", "--execute-command") &
clipp::value("command", cli_opts.command).call([&]() {}).doc("execute cli command to read its output"),
clipp::required("-j1939", "--j1939-document") &
clipp::value("J1939 Document file", cli_opts.docfile)
.call([&]() {
xlsx_parser_task = std::async(std::launch::async, [&]() {
j1939_db_owner = parseXlsx(cli_opts.docfile);
j1939_db.store(j1939_db_owner.get());
signals.map.get<void(sqlite::database &)>("j1939_database_ready")->operator()(*j1939_db_owner);
});
})
.doc("provide .xlsx document file for J1939 processing"));
(clipp::option("-j1939-xlsx") &
clipp::value("J1939 XLSX file", cli_opts.xlsx_file)
.call([&]() {
j1939_parser_task = std::async(std::launch::async, [&]() {
j1939_db_owner = parseXlsx(cli_opts.xlsx_file);
j1939_db.store(j1939_db_owner.get());
signals.map.get<void(sqlite::database &)>("j1939_database_ready")->operator()(*j1939_db_owner);
});
})
.doc("J1939 Digital Annex .xlsx file")) |
(clipp::option("-j1939-csv") &
clipp::value("J1939 CSV file", cli_opts.csv_file)
.call([&]() {
j1939_parser_task = std::async(std::launch::async, [&]() {
j1939_db_owner = parseCsv(cli_opts.csv_file);
j1939_db.store(j1939_db_owner.get());
signals.map.get<void(sqlite::database &)>("j1939_database_ready")->operator()(*j1939_db_owner);
});
})
.doc("J1939 Digital Annex .csv file")));
auto man = clipp::make_man_page(cli, argv[0]);
auto cli_with_help = (cli | clipp::option("-h", "--help").set(cli_opts.show_help).doc("show this help").call([&]() {
@ -415,7 +426,7 @@ int32_t main(int32_t argc, char *argv[]) {
{
const char *names[] = {"xlsx_parser", "aggregator", "refresh", "headless"};
int idx = 0;
for (auto *task : {&xlsx_parser_task, &aggregator_task, &refresh_task, &headless_task}) {
for (auto *task : {&j1939_parser_task, &aggregator_task, &refresh_task, &headless_task}) {
if (task && task->valid()) {
task->wait_for(std::chrono::seconds(3));
}

89
src/mainform.cpp
View File

@ -30,6 +30,7 @@ ftxui::Component makeMainForm(ftxui::ScreenInteractive *screen, signals_map_t &s
static float canbus_params_focus_relative = 0;
static std::string canid_active;
static std::string filter_text;
static std::string export_filter_text;
static size_t tags_count = 0u;
static std::unordered_map<std::string, std::shared_ptr<CanIDUnit>> canid_lookup;
static std::atomic<sqlite::database *> database_atomic{nullptr};
@ -47,6 +48,7 @@ ftxui::Component makeMainForm(ftxui::ScreenInteractive *screen, signals_map_t &s
static std::map<std::string, std::map<int32_t, ftxui::Component>> spnSettingsFormMap;
static auto spn_export_dialog = ftxui::Container::Vertical({});
static auto canbus_params_export_dialog = ftxui::Container::Vertical({});
CanIDUnit::s_export_filter_text_ = &export_filter_text;
static auto canbus_player_dialog = ftxui::Container::Vertical({});
static const auto convertParametersMapToJson = []() {
@ -314,6 +316,37 @@ ftxui::Component makeMainForm(ftxui::ScreenInteractive *screen, signals_map_t &s
}) | ftxui::hcenter,
ftxui::Renderer([]() { return ftxui::separator(); }),
ftxui::Input({
.content = &export_filter_text,
.placeholder = "regex filter ...",
.transform = [](ftxui::InputState state) -> ftxui::Element {
bool valid = true;
if (!export_filter_text.empty()) {
try {
boost::regex(export_filter_text, boost::regex_constants::icase);
} catch (...) {
valid = false;
}
}
state.element |= (!valid ? ftxui::color(ftxui::Color::Red) : ftxui::nothing) |
(state.focused ? ftxui::color(ftxui::Color::Cyan) : ftxui::nothing) |
(state.hovered ? ftxui::bold : ftxui::nothing);
return ftxui::hbox({
ftxui::text(" Search: [ "),
state.element |
(state.hovered || state.focused ? ftxui::bgcolor(ftxui::Color::Grey11)
: ftxui::nothing) |
ftxui::xflex,
ftxui::text(" ]"),
});
},
.multiline = false,
}),
ftxui::Renderer([]() { return ftxui::separator(); }),
(canbus_params_export_dialog | ftxui::Renderer([](ftxui::Element inner) {
return inner | ftxui::focusPositionRelative(0, canbus_params_focus_relative) |
ftxui::vscroll_indicator | ftxui::frame | ftxui::flex;
@ -362,48 +395,48 @@ ftxui::Component makeMainForm(ftxui::ScreenInteractive *screen, signals_map_t &s
}) | ftxui::size(ftxui::WIDTH, ftxui::EQUAL, 96) |
ftxui::size(ftxui::HEIGHT, ftxui::EQUAL, 48) | ftxui::border |
ftxui::CatchEvent([](ftxui::Event event) {
static constexpr float scroll_step = 0.03f;
static const auto increment_focus = []() {
canbus_params_focus_relative =
std::clamp(canbus_params_focus_relative + scroll_step, 0.0f, 1.0f);
};
static const auto decrement_focus = []() {
canbus_params_focus_relative =
std::clamp(canbus_params_focus_relative - scroll_step, 0.0f, 1.0f);
const auto scroll_step = []() -> float {
size_t visible = 0;
for (const auto &[_, entry] : CanIDUnit::s_canbus_parameters_export_map_) {
if (export_filter_text.empty()) {
++visible;
} else {
// Filtering is handled by Maybe in canid_unit.cpp,
// just count all for scroll step estimation
++visible;
}
if (std::get<0u>(entry))
visible += 5;
}
return visible > 0 ? 1.0f / static_cast<float>(visible) : 0.03f;
};
if (event.is_mouse()) {
switch (static_cast<enum ftxui::Mouse::Button>(event.mouse().button)) {
case ftxui::Mouse::Button::WheelDown: {
increment_focus();
goto done;
} break;
case ftxui::Mouse::Button::WheelUp: {
decrement_focus();
goto done;
} break;
case ftxui::Mouse::Button::WheelDown:
canbus_params_focus_relative =
std::clamp(canbus_params_focus_relative + scroll_step(), 0.0f, 1.0f);
return true;
case ftxui::Mouse::Button::WheelUp:
canbus_params_focus_relative =
std::clamp(canbus_params_focus_relative - scroll_step(), 0.0f, 1.0f);
return true;
default:
break;
}
} else if (!event.is_character()) {
if (event == ftxui::Event::ArrowDown) {
increment_focus();
goto done;
canbus_params_focus_relative =
std::clamp(canbus_params_focus_relative + scroll_step(), 0.0f, 1.0f);
return true;
} else if (event == ftxui::Event::ArrowUp) {
decrement_focus();
goto done;
canbus_params_focus_relative =
std::clamp(canbus_params_focus_relative - scroll_step(), 0.0f, 1.0f);
return true;
}
}
forward:
return false;
done:
return true;
}),
&canbus_params_export_dialog_shown) |
ftxui::Modal(makeFileDialog(screen, smap, file_dialog_shown), &file_dialog_shown) |

344
src/xlsx.cpp
View File

@ -1,117 +1,28 @@
#include <filesystem>
#include <fstream>
#include <limits>
#include <map>
#include <memory>
#include <nlohmann/json.hpp>
#include <sstream>
#include <stdexcept>
#include <unistd.h>
#include <string>
#define FMT_HEADER_ONLY
#include <fmt/format.h>
#include <fmt/ranges.h>
// for XLSX files
#include <xlnt/xlnt.hpp>
// For sqlite
#include "parsers.hpp"
#include "j1939_db.hpp"
#include "sqlite_modern_cpp.h"
static void init_db(sqlite::database &db) {
try {
db << R"(
CREATE TABLE IF NOT EXISTS pgns (
id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
pgn INTEGER UNIQUE,
pg_label TEXT,
pg_acronym TEXT,
pg_descr TEXT,
edp INTEGER,
dp INTEGER,
pf INTEGER,
ps INTEGER,
pg_datalen INTEGER,
pg_priority INTEGER
);
)";
db << R"(
CREATE TABLE IF NOT EXISTS spns (
id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
pgn INTEGER,
spn INTEGER UNIQUE,
spn_name TEXT,
spn_pos TEXT,
spn_length INTEGER,
resolution REAL,
offset REAL,
data_range TEXT,
min_value REAL,
max_value REAL,
units TEXT,
slot_id TEXT,
slot_name TEXT,
spn_type TEXT
);
)";
db << R"(
CREATE TABLE IF NOT EXISTS spn_fragments (
id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
spn INTEGER,
pgn INTEGER,
byte_offset INTEGER,
bit_offset INTEGER,
size INTEGER
);
)";
db << "PRAGMA journal_mode = OFF";
db << "PRAGMA synchronous = OFF";
db << "PRAGMA foreign_keys = ON;";
} catch (const sqlite::sqlite_exception &e) {
throw;
}
}
std::unique_ptr<sqlite::database> parseXlsx(const std::string &file) {
auto db_ptr = std::make_unique<sqlite::database>(":memory:");
auto &db = *db_ptr;
init_db(db);
initJ1939Database(db);
static const std::map<std::string_view, std::tuple<size_t, std::string_view>>
pgn_mapping_table =
{
{"PGN", {5u, "pgn"}},
{"Parameter Group Label", {6u, "pg_label"}},
{"PG Acronym", {7u, "pg_acronym"}},
{"PG Description", {8u, "pg_descr"}},
{"EDP", {9u, "edp"}},
{"DP", {10u, "dp"}},
{"PF", {11u, "pf"}},
{"PS", {12u, "ps"}},
{"PG Data Length", {15u, "pg_datalen"}},
{"Default Priority", {16u, "pg_priority"}},
},
const auto &pgn_mapping = J1939MappingTables::pgn();
const auto &spn_mapping = J1939MappingTables::spn();
spn_mapping_table = {
{"SPN", {19u, "spn"}},
{"SPN Name", {20u, "spn_name"}},
{"SPN Position in PG", {18u, "spn_pos"}},
{"SPN Length", {22u, "spn_length"}},
{"Offset", {24u, "offset"}},
{"Data Range", {25u, "data_range"}},
{"Resolution", {23u, "resolution"}},
{"Units", {27u, "units"}},
{"SLOT Identifier", {28u, "slot_id"}},
{"SLOT Name", {29u, "slot_name"}},
{"SPN Type", {30u, "spn_type"}},
};
// Pre-build reverse lookup: column_index -> db_column_name
std::map<size_t, std::string> pgn_col_to_db, spn_col_to_db;
std::map<size_t, std::string> pgn_headers, spn_headers;
xlnt::workbook wb;
if (xlnt::path(file).exists()) {
@ -123,249 +34,48 @@ std::unique_ptr<sqlite::database> parseXlsx(const std::string &file) {
wb.active_sheet(0);
} else {
throw std::runtime_error(fmt::format("File {} does not exists", file));
}
xlnt::worksheet ws = wb.active_sheet();
for (const auto &col : ws.columns()) {
if (auto [cell, value] = std::pair{col[0], col[0].to_string()}; !value.empty()) {
if (pgn_mapping_table.contains(value)) {
auto it = pgn_headers.insert_or_assign(cell.column_index(), value);
} else if (spn_mapping_table.contains(value)) {
auto it = spn_headers.insert_or_assign(cell.column_index(), value);
auto col_idx = cell.column_index();
if (auto it = pgn_mapping.find(value); it != pgn_mapping.end() && col_idx == std::get<0u>(it->second)) {
pgn_col_to_db[col_idx] = std::string(std::get<1u>(it->second));
} else if (auto it2 = spn_mapping.find(value); it2 != spn_mapping.end() && col_idx == std::get<0u>(it2->second)) {
spn_col_to_db[col_idx] = std::string(std::get<1u>(it2->second));
}
}
}
const auto pgn_insert_sql = buildPgnInsertSql();
const auto spn_insert_sql = buildSpnInsertSql();
db << "BEGIN TRANSACTION";
for (const auto &row : ws.rows()) {
std::map<std::string, std::string> pgn_row_map, spn_row_map;
for (const auto &cell : row) {
if (cell.row() != 1u) {
auto col_idx = cell.column_index();
auto value = cell.to_string();
if (value.empty()) continue;
if (pgn_headers.contains(cell.column_index())) {
if (!cell.to_string().empty()) {
pgn_row_map.insert_or_assign(
[&]() {
std::string ret;
for (const auto &[k, v] : pgn_mapping_table) {
if (k == pgn_headers[cell.column_index()] && cell.column_index() == std::get<0u>(v)) {
ret = std::get<1u>(v);
}
}
return ret;
}(),
cell.to_string());
}
} else if (spn_headers.contains(cell.column_index())) {
if (!cell.to_string().empty()) {
spn_row_map.insert_or_assign(
[&]() {
std::string ret;
for (const auto &[k, v] : spn_mapping_table) {
if (k == spn_headers[cell.column_index()] && cell.column_index() == std::get<0u>(v)) {
ret = std::get<1u>(v);
}
}
return ret;
}(),
cell.to_string());
}
if (auto it = pgn_col_to_db.find(col_idx); it != pgn_col_to_db.end()) {
pgn_row_map[it->second] = std::move(value);
} else if (auto it2 = spn_col_to_db.find(col_idx); it2 != spn_col_to_db.end()) {
spn_row_map[it2->second] = std::move(value);
}
}
}
// Add rows to database
if (!pgn_row_map.empty() && !spn_row_map.empty()) {
for (const auto &[k, v] : pgn_row_map) {
try {
auto ps = db << fmt::format(
"INSERT OR REPLACE INTO pgns ({}) VALUES ({})",
[&]() {
std::string ret;
for (const auto &[k, v] : pgn_mapping_table) {
auto end = pgn_mapping_table.end();
bool is_last = k == (--end)->first;
ret += fmt::format("{}{}", std::get<1u>(v), !is_last ? ", " : "");
}
return ret;
}(),
[&]() {
std::string ret;
for (const auto &[k, _] : pgn_mapping_table) {
auto end = pgn_mapping_table.end();
bool is_last = k == (--end)->first;
ret += !is_last ? "?, " : "?";
}
return ret;
}());
for (const auto &[k, v] : pgn_mapping_table) {
ps << pgn_row_map[std::get<1u>(v).data()];
}
ps.execute();
} catch (const sqlite::sqlite_exception &e) {
if (e.get_extended_code() == SQLITE_CONSTRAINT_UNIQUE) {
continue;
}
throw;
}
}
struct {
bool parts_inserted_flag = false;
double min = 0.0, max = 0.0;
size_t size_bits = 0u;
} spn_settings_calculated;
// Pre-compute size_bits before the main loop (resolution calculation depends on it)
{
auto it = spn_mapping_table.find("SPN Length");
if (it != spn_mapping_table.end()) {
auto size = parsers::parseSpnSize(spn_row_map[std::get<1u>(it->second).data()]);
spn_settings_calculated.size_bits = size.has_value() ? size.value().size_bits : 0u;
}
}
for (const auto &[k, v] : spn_row_map) {
try {
auto ps = db << fmt::format(
"INSERT OR REPLACE INTO spns ({}) VALUES ({})",
[&]() {
std::string ret;
ret += "pgn, ";
for (const auto &[k, v] : spn_mapping_table) {
auto end = spn_mapping_table.end();
bool is_last = k == (--end)->first;
// Split 'data range' to two columns 'min' and 'max'
if (std::get<1u>(v) == "data_range") {
ret += is_last ? ", min_value, max_value" : "min_value, max_value, ";
} else {
ret += fmt::format("{}{}", std::get<1u>(v), !is_last ? ", " : "");
}
}
return ret;
}(),
[&]() {
std::string ret;
ret += "?, ";
for (const auto &[k, v] : spn_mapping_table) {
auto end = spn_mapping_table.end();
bool is_last = k == (--end)->first;
// Split 'data range' to two columns 'min' and 'max'
if (std::get<1u>(v) == "data_range") {
ret += is_last ? ", ?, ?" : "?, ?, ";
} else {
ret += !is_last ? "?, " : "?";
}
}
return ret;
}());
ps << pgn_row_map["pgn"];
for (const auto &[k, v] : spn_mapping_table) {
if (std::get<1u>(v) == "data_range") { // Parse 'data range' string and split it to two columns 'min' and 'max'
auto range = parsers::parseSpnDataRange(spn_row_map[std::get<1u>(v).data()]);
if (range.has_value()) {
(spn_settings_calculated.min = range.value().min, spn_settings_calculated.max = range.value().max);
for (const auto &val : {range.value().min, range.value().max}) {
ps << val;
}
} else {
ps << nullptr << nullptr;
}
} else if (std::get<1u>(v) == "offset") {
auto offset = parsers::parseSpnOffset(spn_row_map[std::get<1u>(v).data()]);
if (offset.has_value()) {
ps << offset.value().offset;
} else {
ps << nullptr;
}
} else if (std::get<1u>(v) == "spn_length") {
auto size = parsers::parseSpnSize(spn_row_map[std::get<1u>(v).data()]);
spn_settings_calculated.size_bits = size.has_value() ? size.value().size_bits : 0u;
ps << (size.has_value() ? size.value().size_bits : 0u);
} else if (std::get<1u>(v) == "resolution") {
double calculated = (spn_settings_calculated.max - spn_settings_calculated.min) / (std::pow(2.0f, spn_settings_calculated.size_bits) - 1u);
// If is discrete -- use calculated resolution
if (std::fabs(calculated - 1.0) < 1e-9) {
ps << calculated;
} else {
auto resolution = parsers::parseSpnResolution(spn_row_map[std::get<1u>(v).data()]);
ps << (resolution.has_value() ? resolution.value().resolution : 1.0f);
}
} else if (std::get<1u>(v) == "spn_pos") {
ps << spn_row_map[std::get<1u>(v).data()];
} else {
ps << spn_row_map[std::get<1u>(v).data()];
}
}
ps.execute();
// Insert SPN fragments after successful spns INSERT
if (!spn_settings_calculated.parts_inserted_flag) {
auto size = parsers::parseSpnSize(spn_row_map[std::get<1u>(spn_mapping_table.at("SPN Length")).data()]);
if (size.has_value()) {
auto spn_fragments = parsers::parseSpnPosition(size.value().size_bits, spn_row_map[std::get<1u>(spn_mapping_table.at("SPN Position in PG")).data()]);
if (spn_fragments.has_value()) {
auto spn = std::stoll(spn_row_map["spn"]);
for (const auto &part : spn_fragments.value().spn_fragments) {
db << R"(INSERT OR REPLACE INTO spn_fragments (spn, pgn, byte_offset, bit_offset, size) VALUES (?, ?, ?, ?, ?);)" << spn << std::stoll(pgn_row_map["pgn"])
<< part.byte_offset << part.bit_offset << part.size;
}
spn_settings_calculated.parts_inserted_flag = true;
}
}
}
} catch (const sqlite::sqlite_exception &e) {
if (e.get_extended_code() == SQLITE_CONSTRAINT_UNIQUE) {
continue;
}
throw;
}
}
insertJ1939Row(db, pgn_insert_sql, spn_insert_sql, pgn_row_map, spn_row_map);
}
}
db << "COMMIT";
return db_ptr;
}

53669
thirdparty/j1939da_2018.csv vendored Normal file
View File

File diff suppressed because it is too large Load Diff