diff --git a/CMakeLists.txt b/CMakeLists.txt index 41e7bea..3de7c2f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,13 +7,13 @@ set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CUDA_STANDARD 17) find_package(CUDAToolkit REQUIRED) -find_package(OpenCV REQUIRED COMPONENTS core imgproc imgcodecs highgui) +find_package(OpenCV REQUIRED COMPONENTS core imgproc imgcodecs highgui videoio) find_package(Threads REQUIRED) find_path(TENSORRT_INCLUDE_DIR NvInfer.h PATH_SUFFIXES aarch64-linux-gnu x86_64-linux-gnu REQUIRED) find_library(TENSORRT_LIBRARY nvinfer REQUIRED) add_library(camera_risk_core src/trt_engine.cpp src/pose_engine.cu src/gpu_preprocess.cu src/uyvy_converter.cu - src/detector.cpp src/door_risk.cpp src/camera.cpp src/camera_pipeline.cpp src/serialization.cpp src/output_writer.cpp src/result_renderer.cpp src/config.cpp) + src/detector.cpp src/door_risk.cpp src/camera.cpp src/camera_pipeline.cpp src/inference_session.cpp src/serialization.cpp src/output_writer.cpp src/result_renderer.cpp src/config.cpp) target_include_directories(camera_risk_core PUBLIC include third_party ${TENSORRT_INCLUDE_DIR} ${OpenCV_INCLUDE_DIRS}) target_link_libraries(camera_risk_core PUBLIC ${TENSORRT_LIBRARY} ${OpenCV_LIBS} Threads::Threads CUDA::cudart CUDA::nppc CUDA::nppig CUDA::nppidei CUDA::nppicc) target_compile_options(camera_risk_core PRIVATE $<$:-Wall;-Wextra;-Wpedantic>) diff --git a/README.md b/README.md index e8d6e85..259ab15 100644 --- a/README.md +++ b/README.md @@ -96,3 +96,40 @@ NPP 用于 UYVY 转换、YOLO 缩放填充、RTMW 仿射裁剪；CUDA 用于通 相机超时回归测试：`./tests/run_camera_timeout.sh`；加 `--wall-clock` 验证默认 6 次超时约 12 秒。测试直接编译实际相机代码，使用模拟采集和转换器，无需摄像头、GPU 或 OpenCV；不覆盖视频回放路径。 板端真实摄像头 / TensorRT 恢复与停止测试记录见 [Orin 测试报告](tests/ORIN_TEST_REPORT.md)。 + +## 推理过程录制与视频回灌 + +当前 `inference-only` 分支支持 H.264/MP4 会话；默认关闭，不再为会话逐帧保存图片。 + +```bash +# 录制实际完成推理的帧，Ctrl+C / SIGTERM 正常停止并封装视频 +./run_camera.sh --record-session output/sessions/drive-001 + +# 视频解码后重新运行 YOLO、RTMW、融合与风险推理；文件结束自动退出 +./run_camera.sh --replay-session output/sessions/drive-001 --save-results + +# 同时保存回灌后的新视频及推理过程，便于对比 +./run_camera.sh --replay-session output/sessions/drive-001 \ + --record-session output/sessions/drive-001-replay +``` + +独立包使用同样参数，例如 `./orin/run.sh --record-session output/sessions/drive-001`。 +会话路径相对于当前工作目录；录制目录必须尚不存在，避免覆盖原始记录。 + +会话内容： + +- `frames.mp4`：H.264 有损压缩视频，按完成推理的顺序编码，尺寸与原图一致。 +- `frames/*.json`：对应视频帧序号、原推理帧号、完整采集元数据、检测框、原始/隔离关键点、融合记录、最终结果与耗时。 +- `frames.txt`：视频帧与 JSON 的顺序索引。 +- `session.json`、`door_mask.json`：录制时的推理参数、分辨率、名义帧率、门区副本、模型路径与大小。 +- `complete.json`：正常停止后写入的帧数与视频大小，用于检查会话是否完成。 + +回灌不打开摄像头。它恢复会话中的分辨率、检测阈值、人数上限、风险阈值和门区，使用 `--config` 指定的当前 TensorRT 引擎，按顺序尽快推理，不按墙钟时间限速。视频使用配置的固定名义帧率；真实采集间隔和断流间隔以 JSON 中的纳秒时间戳为准。原始采集时间在再次录制时保存为 `source_capture`，回灌耗时使用本次运行的单调时钟。 + +编码使用 OpenCV 的 FFmpeg H.264 编码器；需 OpenCV `videoio` 与可用的 H.264 编码支持，宽高必须为正偶数。不支持时明确报错。当前使用软件编解码，会增加 CPU、编码延迟和磁盘开销；保存对象是完成推理的帧，并非相机全部原始帧。H.264 有损压缩和不同引擎可能改变检测、关键点与风险结果，不承诺逐位一致。模型只记录来源和大小，不复制引擎，也不保证大小相同即模型相同；精确对比应使用同一组引擎。 + +Ctrl+C / SIGTERM 会封装已写入的视频，`CameraPipeline::join()` 返回前完成收尾。强制杀进程、断电或写入失败可能留下 `frames.pending.mp4`；缺少完成标记的会话拒绝回灌。编码器封装后检查帧数，回灌遇到缺帧、缺失 JSON、损坏索引或解码错误会报错，不静默跳帧。录制没有自动限时或磁盘轮转，请按可用空间控制录制时长。 + +编解码及会话回归：`./tests/run_session.sh`，无需摄像头或 TensorRT。真实摄像头与 TensorRT 验证脚本：`python3 tests/orin_session_test.py --binary build/camera_risk --config config/camera.json --output /tmp/new-session-test`；输出目录必须不存在，脚本不会停止已有部署进程。 + +本次实机结果与未覆盖范围见 [H.264 会话测试报告](tests/ORIN_SESSION_REPORT.md)。 diff --git a/include/camera.hpp b/include/camera.hpp index c07c729..35a8d86 100644 --- a/include/camera.hpp +++ b/include/camera.hpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace cms { struct CameraConfig { @@ -13,6 +14,8 @@ struct CameraConfig { struct CapturedFrame { CaptureMetadata metadata; std::shared_ptr image; + // Present only for offline replay; keeps original timestamps and context. + std::shared_ptr replay_record; }; // The capture thread exclusively owns read(); requestStop() is thread safe. class Camera { diff --git a/include/camera_pipeline.hpp b/include/camera_pipeline.hpp index 9d5dac8..cf32bb4 100644 --- a/include/camera_pipeline.hpp +++ b/include/camera_pipeline.hpp @@ -7,8 +7,10 @@ namespace cms { struct PipelineConfig { + static constexpr char const* session_variant="inference-only"; CameraConfig camera; OutputOptions output; + std::filesystem::path record_session,replay_session; std::filesystem::path engines,door_mask; double detector_confidence{0.30}; int max_people{3}; diff --git a/include/inference_session.hpp b/include/inference_session.hpp new file mode 100644 index 0000000..7131b5d --- /dev/null +++ b/include/inference_session.hpp @@ -0,0 +1,38 @@ +#pragma once +#include "camera.hpp" +#include "frame_result.hpp" +#include +#include +#include +#include + +namespace cms { +struct PipelineConfig; +// H.264/MP4 plus a matching frame index and per-frame inference records. +// finish() flushes the encoder and marks the session replayable. +class SessionWriter { +public: + explicit SessionWriter(PipelineConfig const& config); + void write(FrameResult const& result, nlohmann::json const& stages, + nlohmann::json const* source = nullptr); + void finish(); +private: + std::filesystem::path directory_; + std::ofstream index_; + cv::VideoWriter video_; + int width_{},height_{}; + std::uint64_t count_{}; + bool finished_{}; +}; +class SessionReader { +public: + explicit SessionReader(PipelineConfig& config); + std::optional read(); +private: + std::filesystem::path directory_; + std::ifstream index_; + cv::VideoCapture video_; + int width_{},height_{}; + std::uint64_t count_{},lastFrame_{},expectedCount_{}; +}; +} diff --git a/orin/README.md b/orin/README.md index 3b96a13..d95837b 100644 --- a/orin/README.md +++ b/orin/README.md @@ -41,3 +41,13 @@ bin/drivers 中 GMSL 模块对应内核 5.10.120-tegra。初始化脚本检查 同一帧三类文件同名（扩展名不同），文件名包含运行标识和帧号，多次运行不覆盖。JSON 的 `images` 提供相对于 `output/` 的图片路径，未选择保存的图片为 null。源码入口写入工程 `output/`；独立包入口写入 `orin/output/`。默认不创建目录、不保存文件。 保存对象是完成推理的帧；惰性分支暂停时不新增文件。开启保存会增加编码和磁盘 I/O 开销。写入失败时报告错误并停止，不静默丢失文件。 + +## H.264 推理过程录制与回灌 + +```bash +./run.sh --record-session output/sessions/drive-001 +# Ctrl+C 正常停止，完成 MP4 封装后再回灌 +./run.sh --replay-session output/sessions/drive-001 --save-results +``` + +录制目录必须不存在。会话保存一份 H.264 视频、逐帧过程 JSON、推理配置与门区副本；回灌解码视频并重新执行 TensorRT 推理，不打开摄像头，结束后自动退出。使用当前配置指向的引擎；有损压缩会引起推理结果差异。需要 OpenCV FFmpeg 的 H.264 编码支持。强制退出或断电导致未封装完成的会话不可回灌；正常 Ctrl+C / SIGTERM 会完成封装。详细格式与限制见工程 README。 diff --git a/orin/SHA256SUMS b/orin/SHA256SUMS index 6bf7136..1fc8fc7 100644 --- a/orin/SHA256SUMS +++ b/orin/SHA256SUMS @@ -1,11 +1,11 @@ -49a272525082edf5ef4619efc57bc2509c06653af262fc8a8d257d138f9206fb ./README.md 6d5e874fd27fcfd75d303d7e35d9672028af71041f52dcc82a1e83676faefa65 ./bin/drivers/max9295.ko 8a78d04ef8be44e9b62a5e4ee315d3d6d679e34aedff02a234bbf4d40e222933 ./bin/drivers/max9296.ko 3da628587477cc35d5e3f8aee86e718e2ddf16808e893e00ba6e164b2307e843 ./bin/drivers/sgx-yuv-gmsl2.ko -3a984e1e4c0fa12285967085e3359a5b7c698d744e9c2deb2480a312eb637c1c ./build/camera_risk +0f178057529eb8be44bb9a639504dfe57a1e353a84d2351fd907304f6d951c11 ./build/camera_risk 8a572f661e056e3916758a442e73212842bfe89361dd1b07b87de8f6ae523cf8 ./config/camera.json 9ba768d559fd3b64e833b50c62c70f2adfd6c169620aea5fba32cfb1b4c5b0e9 ./config/door_mask.json ebcd09e8c2e35008adbb0c1d222e580b17e0df1c8093df3481ba3374cd9a8174 ./engines/rtmw_l_384.engine f6a65b926ac1144484f4aa93cd26d7997d32d62310ab93fe867f2bd4412222e9 ./engines/yolo11s.engine +eb90be468ff835bc2a1a79b83dbb3c9cfe78449e79d005596f887e8ac9981b86 ./README.md 7f42b0e7f9396e0d82313a892fc62c153261dab1206a206a466bce6376f58a36 ./run.sh 54e1d63e3aa72ad9b056a224d55968d4947ca6703a8d1c405570de5707a7e7ef ./scripts/initialize_camera.sh diff --git a/orin/build/camera_risk b/orin/build/camera_risk index 9929599..869acaa 100755 Binary files a/orin/build/camera_risk and b/orin/build/camera_risk differ diff --git a/src/camera_pipeline.cpp b/src/camera_pipeline.cpp index 063f3e2..37c97b7 100644 --- a/src/camera_pipeline.cpp +++ b/src/camera_pipeline.cpp @@ -4,6 +4,7 @@ #include "detector.hpp" #include "door_risk.hpp" #include "records.hpp" +#include "inference_session.hpp" #include "result_conversion.hpp" #include #include @@ -51,6 +52,8 @@ struct CameraPipeline::Impl { BoundedQueue detections; LatestResult results; std::unique_ptr outputWriter; + std::unique_ptr sessionWriter; + std::unique_ptr sessionReader; fangjia::TrtLogger logger; std::unique_ptr detector; std::unique_ptr pose; @@ -66,6 +69,7 @@ struct CameraPipeline::Impl { double totalLatency{}; explicit Impl(PipelineConfig c):config(std::move(c)),captures(config.queue_capacity),detections(config.queue_capacity) { + if(!config.replay_session.empty())sessionReader=std::make_unique(config); if(config.queue_capacity>8)throw std::invalid_argument("Queue capacity must be between 1 and 8"); if(config.max_people<1||config.max_people>16||!std::isfinite(config.detector_confidence)||config.detector_confidence<=0||config.detector_confidence>=1) throw std::invalid_argument("Invalid detector configuration"); @@ -95,6 +99,7 @@ struct CameraPipeline::Impl { {std::lock_guard lock(stateMutex);if(started||stopped)throw std::logic_error("CameraPipeline can only be started once");started=true;} try { outputWriter=std::make_unique(config.output); + sessionWriter=std::make_unique(config); detector=std::make_unique(config.engines/"yolo11s.engine",logger,config.detector_confidence,config.max_people); pose=std::make_unique(config.engines/"rtmw_l_384.engine",logger,config.camera.width,config.camera.height,config.max_people); DoorOptions options;options.constant=config.door_mask;options.warning=config.warning_relative;options.release=config.release_relative; @@ -107,7 +112,7 @@ struct CameraPipeline::Impl { Stream warmup; pose->warmup(warmup.value); detector->infer(cv::Mat(config.camera.height,config.camera.width,CV_8UC3,cv::Scalar()),warmup.value); - {std::lock_guard lock(stateMutex);camera=std::make_unique(config.camera);if(stopped)camera->requestStop();} + {std::lock_guard lock(stateMutex);if(!sessionReader)camera=std::make_unique(config.camera);if(stopped&&camera)camera->requestStop();} active=!stopped.load(); captureThread=std::thread([this]{try{captureLoop();}catch(...){fail();}}); detectorThread=std::thread([this]{try{detectorLoop();}catch(...){fail();}}); @@ -116,7 +121,7 @@ struct CameraPipeline::Impl { } void captureLoop() { while(!stopped) { - auto frame=camera->read();if(!frame)break; + auto frame=sessionReader?sessionReader->read():camera->read();if(!frame)break; gaps+=frame->metadata.skipped_driver_frames; errorFrames+=frame->metadata.discarded_error_frames; if(!captures.push(std::move(*frame)))break; @@ -152,7 +157,7 @@ struct CameraPipeline::Impl { } auto poseElapsed=elapsedMs(poseStart);auto fusionStart=monotonicNs(); auto fused=fusion.update(raw,isolated,index); - auto record=makeRecord(index,raw,isolated,fused,false,false); + auto record=makeRecord(index,raw,isolated,fused,!config.record_session.empty(),false); auto riskData=risk->evaluate(index,record); auto result=std::make_shared(makeFrameResult(record,riskData,detected->frame.metadata)); attachPoses(*result,fused); @@ -162,8 +167,10 @@ struct CameraPipeline::Impl { FrameResultPtr immutable=result; results.publish(std::move(immutable)); {std::lock_guard lock(stateMutex);totalLatency+=result->timing.capture_to_result_ms;++published;} + if(!config.record_session.empty())sessionWriter->write(*result,record,detected->frame.replay_record.get()); outputWriter->write(*result); } + sessionWriter->finish(); active=false; } }; diff --git a/src/inference_session.cpp b/src/inference_session.cpp new file mode 100644 index 0000000..76a537d --- /dev/null +++ b/src/inference_session.cpp @@ -0,0 +1,150 @@ +#include "inference_session.hpp" +#include "camera_pipeline.hpp" +#include "serialization.hpp" + +#include +#include +#include +#include +#include + +namespace cms { +namespace { +using Json = nlohmann::json; +Json readJson(std::filesystem::path const& path) { + std::ifstream input(path); + if(!input)throw std::runtime_error("Cannot read session file: "+path.string()); + return Json::parse(input); +} +void atomicWrite(std::filesystem::path const& path,char const* data,std::size_t size) { + auto tmp=path;tmp+=".tmp"; + std::ofstream out;out.exceptions(std::ios::failbit|std::ios::badbit); + out.open(tmp,std::ios::binary|std::ios::trunc); + out.write(data,static_cast(size));out.close(); + std::filesystem::rename(tmp,path); +} +void writeJson(std::filesystem::path const& path,Json const& value) { + auto text=value.dump(2)+"\n";atomicWrite(path,text.data(),text.size()); +} +std::string frameName(std::uint64_t index) { + std::ostringstream name;name<16384||height_>16384||width_%2||height_%2||c.camera.fps<=0) + throw std::invalid_argument("H.264 recording requires positive even dimensions and positive FPS"); + // Exclusive creation: never append to or overwrite another session. + if(!directory_.parent_path().empty())std::filesystem::create_directories(directory_.parent_path()); + if(!std::filesystem::create_directory(directory_))throw std::runtime_error("Recording directory already exists: "+directory_.string()); + std::filesystem::create_directory(directory_/"frames"); + Json config={{"width",c.camera.width},{"height",c.camera.height},{"fps",c.camera.fps}, + {"detector_confidence",c.detector_confidence},{"max_people",c.max_people}, + {"warning_relative",c.warning_relative},{"release_relative",c.release_relative}, + {"calibration_verified",c.door_mask_calibration_verified}}; + Json models=Json::object(); + for(auto name:{"yolo11s.engine","rtmw_l_384.engine"}) + models[name]={{"path",(c.engines/name).string()},{"bytes",std::filesystem::file_size(c.engines/name)}}; + writeJson(directory_/"door_mask.json",readJson(c.door_mask)); + if(!video_.open((directory_/"frames.pending.mp4").string(),cv::CAP_FFMPEG, + cv::VideoWriter::fourcc('a','v','c','1'),c.camera.fps,cv::Size(width_,height_))) + throw std::runtime_error("Cannot open H.264 encoder; OpenCV FFmpeg with an H.264 encoder is required"); + writeJson(directory_/"session.json",{{"schema_version",2},{"format","cms-inference-session"}, + {"video",{{"file","frames.mp4"},{"codec","h264"},{"lossy",true}}},{"variant",PipelineConfig::session_variant},{"config",config},{"models",models},{"source_session",c.replay_session.string()}}); + index_.exceptions(std::ios::failbit|std::ios::badbit);index_.open(directory_/"frames.txt"); + std::cout<<"Recording inference session: "<(im.pixels.data()),im.stride); + video_.write(view); + Json record={{"video_frame_index",count_},{"frame_index",r.frame_index},{"capture",captureJson(r.capture)}, + {"stages",stages},{"result",serializeResult(r)}}; + // Preserve original acquisition timestamps across replay -> record cycles. + if(source)record["source_capture"]=source->contains("source_capture")?source->at("source_capture"):source->at("capture"); + writeJson(directory_/"frames"/(stem+".json"),record); + index_<(); + if(completion.at("video_bytes")!=std::filesystem::file_size(directory_/"frames.mp4")) + throw std::runtime_error("Session video size does not match finalized recording"); + auto const& j=manifest.at("config"); + width_=j.at("width");height_=j.at("height"); + if(width_<=0||height_<=0||width_>16384||height_>16384||width_%2||height_%2)throw std::runtime_error("Invalid session dimensions"); + c.camera.width=width_;c.camera.height=height_;c.camera.fps=j.at("fps"); + c.detector_confidence=j.at("detector_confidence");c.max_people=j.at("max_people"); + c.warning_relative=j.at("warning_relative");c.release_relative=j.at("release_relative"); + c.door_mask_calibration_verified=j.at("calibration_verified"); + c.door_mask=directory_/"door_mask.json"; + index_.open(directory_/"frames.txt"); + if(!index_)throw std::runtime_error("Cannot open session frame index"); + std::cout<<"Replaying inference session: "< SessionReader::read() { + std::string stem; + if(!std::getline(index_,stem)) { + if(!index_.eof()||count_!=expectedCount_)throw std::runtime_error("Truncated session index"); + return {}; + } + // No path strings from the archive are used as filenames. Check sequence, + // including its newline, so corruption cannot silently skip a frame. + if(index_.eof()||count_>=expectedCount_||stem!=frameName(count_))throw std::runtime_error("Invalid or truncated session frame index"); + auto record=std::make_shared(readJson(directory_/"frames"/(stem+".json"))); + if(record->at("video_frame_index")!=count_)throw std::runtime_error("Video/record frame index mismatch"); + auto frameIndex=record->at("frame_index").get(); + if(frameIndex<0||frameIndex>std::numeric_limits::max()||(count_&&std::uint64_t(frameIndex)<=lastFrame_)) + throw std::runtime_error("Session frame indices must increase and fit int"); + // Opening FFmpeg can create decoder threads. Do this in the capture + // worker, after StopSignals has installed the inherited signal mask. + if(!video_.isOpened()&&(!video_.open((directory_/"frames.mp4").string(),cv::CAP_FFMPEG)|| + video_.get(cv::CAP_PROP_FRAME_COUNT)!=double(expectedCount_))) + throw std::runtime_error("Cannot open complete session video"); + cv::Mat im; + if(!video_.read(im)||im.empty()||im.cols!=width_||im.rows!=height_||im.type()!=CV_8UC3)throw std::runtime_error("Missing or malformed session video frame: "+stem); + auto image=std::make_shared();image->width=width_;image->height=height_;image->stride=width_*3; + image->pixels.resize(std::size_t(image->stride)*height_); + for(int y=0;ypixels.data()+std::size_t(y)*image->stride,im.ptr(y),image->stride); + auto const& c=record->at("capture");CaptureMetadata metadata; + metadata.frame_index=frameIndex;metadata.driver_sequence=c.at("driver_sequence"); + metadata.driver_timestamp_ns=c.at("driver_timestamp_ns");metadata.driver_timestamp_monotonic=c.at("driver_timestamp_monotonic"); + metadata.skipped_driver_frames=c.at("skipped_driver_frames");metadata.discarded_error_frames=c.at("discarded_error_frames"); + // Old monotonic clocks cannot measure current replay latency. + metadata.received_monotonic_ns=metadata.capture_monotonic_ns=monotonicNs(); + lastFrame_=frameIndex;++count_; + return CapturedFrame{metadata,std::move(image),std::move(record)}; +} +} diff --git a/src/main.cpp b/src/main.cpp index b388207..7a97637 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -11,17 +11,20 @@ int main(int argc,char** argv)try { auto configPath=root/"config/camera.json"; cms::OutputOptions output;output.directory=root/"output"; bool visualize=false; + std::filesystem::path recordSession,replaySession; std::map> arguments{ + {"--record-session",[&](int&i){if(++i>=argc)throw std::invalid_argument("--record-session needs a directory");recordSession=argv[i];}}, + {"--replay-session",[&](int&i){if(++i>=argc)throw std::invalid_argument("--replay-session needs a directory");replaySession=argv[i];}}, {"--save-original",[&](int&){output.original=true;}}, {"--save-annotated",[&](int&){output.annotated=true;}}, {"--save-results",[&](int&){output.results=true;}}, {"--visualize",[&](int&){visualize=true;}}, {"--config",[&](int&i){if(++i>=argc)throw std::invalid_argument("--config needs a file");configPath=argv[i];}}, - {"--help",[](int&){std::cout<<"camera_risk [--save-original] [--save-annotated] [--save-results] [--config FILE] [--visualize]\nContinuous camera inference; latest FrameResult stays in memory. Stop with Ctrl+C or SIGTERM.\n";std::exit(0);}} + {"--help",[](int&){std::cout<<"camera_risk [--record-session NEW_DIR] [--replay-session DIR] [--save-original] [--save-annotated] [--save-results] [--config FILE] [--visualize]\nContinuous camera inference; replay runs to EOF without a camera.\nH.264 sessions require a new recording directory. Stop with Ctrl+C or SIGTERM to finalize video.\n";std::exit(0);}} }; for(int i=1;isecond(i);} - auto config=cms::loadConfig(configPath);config.output=output; + auto config=cms::loadConfig(configPath);config.output=output;config.record_session=recordSession;config.replay_session=replaySession; cms::CameraPipeline pipeline(config); cms::StopSignals signals(pipeline); pipeline.start(); diff --git a/tests/ORIN_SESSION_REPORT.md b/tests/ORIN_SESSION_REPORT.md new file mode 100644 index 0000000..9da5d51 --- /dev/null +++ b/tests/ORIN_SESSION_REPORT.md @@ -0,0 +1,38 @@ +# H.264 推理过程录制与回灌验证（2026-09-24） + +范围：按本次要求，仅修改 `inference-only`。测试直接在当前 `orin-desktop` 的 Jetson Orin 上执行；真实 `/dev/video5`，1920×1536 UYVY，L4T R35.4.1，CUDA 11.4，TensorRT，OpenCV 4.5.4 / FFmpeg H.264。未使用 SSH。 + +## 本机模拟与编解码回归 + +- `tests/run_session.sh`：实际 H.264 编码/解码，使用人工图像及模拟结果，不加载 TensorRT。覆盖有步长的 BGR 输入、视频帧顺序、帧号间隔、配置恢复、原始时间戳保留、编码器刷新、正常 EOF、空会话、拒绝覆盖、缺失 JSON、非法/截断索引、不完整会话和奇数尺寸。启用 UBSan。 +- `tests/run_camera_timeout.sh`：模拟 V4L2 采集；通过超时重试、有效帧后重新计数、第六次超时失败与主动停止回归。此项不是本次真实硬件故障恢复的证明。 + +## 真实摄像头与 TensorRT + +脚本：`tests/orin_session_test.py`。完整数据见 `orin_session_results.json`。 + +| 项目 | 结果 | +|---|---| +| 摄像头录制 | 13 个完成推理的帧，H.264 / MP4，1920×1536 | +| 视频体积 | 303,989 字节；同帧数未压缩 BGR 为 115,015,680 字节，约缩小 378 倍，仅代表本次静态场景 | +| 完整回灌 | 13/13 帧，顺序与原帧号一致，原始采集元数据保留，EOF 正常退出 | +| 结果检查 | 13/13 跟踪 ID 与风险决策一致；本次无检测到的人员 | +| 录制期间 SIGTERM | 1.417 秒完整退出，视频封装成功 | +| 回灌期间 SIGINT | 0.865 秒完整退出，保存 2 帧后停止，视频封装成功 | +| 实际运行程序核对 | `/proc//exe` 与源码构建二进制 SHA256 相同 | + +实机首次测试发现：若在 `CameraPipeline` 构造期间打开视频解码器，FFmpeg 工作线程可能在 `StopSignals` 安装前创建，SIGINT 可直接终止进程。已将解码器打开操作移至继承信号掩码的采集线程，修复后重跑上述实机测试全部通过。 + +测试前没有 `camera_risk` 进程，未更改既有启动参数；测试脚本仅管理自己启动的进程，测试后不留下常驻进程。原始视频、逐帧记录与日志在 `/tmp/vap-video-session-hardware-20260924-fixed/`，不提交摄像头图像。 + +## 板端故障注入与范围限制 + +本次未做板端 V4L2 故障注入或物理拔线操作；共享相机恢复实现保持原样，其既有硬件报告见 `ORIN_TEST_REPORT.md`，不能视作本次新增功能的实测覆盖。当前实机画面没有检测到人员，RTMW 完成初始化/预热，但未覆盖有人员的逐帧姿态推理、遮挡隔离推理或风险变化；报告中的关键点差异为 0 不代表有人员场景的精度结论。 + +未覆盖 GUI 预览、GPU 故障、磁盘写满、断电/强杀后的媒体恢复、长时间录制与 NVENC/NVDEC 硬件编解码。当前为软件 H.264 编解码；强杀留下的未完成会话按设计拒绝回灌。 + +## 经测试的程序 SHA256 + +`0f178057529eb8be44bb9a639504dfe57a1e353a84d2351fd907304f6d951c11` + +独立包 `orin/build/camera_risk` 已更新，`orin/SHA256SUMS` 全部校验通过。使用独立包实际运行并完整回灌 13 帧，再次核对 `/proc//exe` 与上述哈希一致；部署检查日志位于 `/tmp/vap-video-session-deployment-20260924/`。测试结束恢复原来的无常驻进程状态。 diff --git a/tests/orin_session_results.json b/tests/orin_session_results.json new file mode 100644 index 0000000..abfe7dd --- /dev/null +++ b/tests/orin_session_results.json @@ -0,0 +1,23 @@ +{ + "platform": "orin-desktop", + "binary": "/home/orin/vision-anti-pinch-0923/build/camera_risk", + "binary_sha256": "0f178057529eb8be44bb9a639504dfe57a1e353a84d2351fd907304f6d951c11", + "running_binary_sha256": "0f178057529eb8be44bb9a639504dfe57a1e353a84d2351fd907304f6d951c11", + "recorded_frames": 13, + "replayed_frames": 13, + "frames_with_people": 0, + "max_raw_pose_error_px": 0.0, + "matching_track_frames": 13, + "matching_decision_frames": 13, + "video_bytes": 303989, + "uncompressed_bgr_bytes": 115015680, + "sigterm_seconds": 1.4174426249992393, + "replay_sigint_seconds": 0.8654193540005508, + "interrupted_frames": 2, + "standalone_deployment": { + "binary": "/home/orin/vision-anti-pinch-0923/orin/build/camera_risk", + "binary_sha256": "0f178057529eb8be44bb9a639504dfe57a1e353a84d2351fd907304f6d951c11", + "running_binary_sha256": "0f178057529eb8be44bb9a639504dfe57a1e353a84d2351fd907304f6d951c11", + "replayed_frames": 13 + } +} diff --git a/tests/orin_session_test.py b/tests/orin_session_test.py new file mode 100644 index 0000000..bdf2f0e --- /dev/null +++ b/tests/orin_session_test.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +"""Real camera + TensorRT record/replay check. Runs only explicitly on Orin. +Records to a new test directory; never stops an existing deployment process. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import signal +import subprocess +import time + + +def rows(session): + index = session / 'frames.txt' + return index.read_text().splitlines() if index.exists() else [] + + +def wait_frames(process, session, count, timeout=90): + deadline = time.monotonic() + timeout + while len(rows(session)) < count: + if process.poll() is not None: + raise RuntimeError(f'Inference exited early: {process.returncode}') + if time.monotonic() > deadline: + raise RuntimeError('Timed out awaiting real inference frames') + time.sleep(.05) + + +def stop(process, signum=signal.SIGTERM): + start = time.monotonic() + process.send_signal(signum) + code = process.wait(timeout=15) + if code: + raise RuntimeError(f'Nonzero stop exit: {code}') + return time.monotonic() - start + + +def run_record(command, session, log, count, signum=signal.SIGTERM): + with log.open('w') as output: + process = subprocess.Popen(command + ['--record-session', str(session)], stdout=output, stderr=subprocess.STDOUT) + try: + wait_frames(process, session, count) + return stop(process, signum) + finally: + if process.poll() is None: + process.terminate() + process.wait(timeout=15) + + +def load(session): + return [json.loads((session / 'frames' / f'{stem}.json').read_text()) for stem in rows(session)] + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--binary', type=Path, required=True) + parser.add_argument('--config', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--replay-input', type=Path, help='Verify deployment with an existing session; no camera access') + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=False) + binary = args.binary.resolve() + config = args.config.resolve() + command = [str(binary), '--config', str(config)] + # Refuse camera contention rather than alter an existing deployment. + if subprocess.run(['pgrep', '-x', 'camera_risk'], stdout=subprocess.DEVNULL).returncode == 0: + raise RuntimeError('An existing camera_risk process is running; preserve its launch parameters first') + if args.replay_input: + replayed = args.output / 'replayed' + with (args.output/'replay.log').open('w') as log: + process = subprocess.Popen(command + ['--replay-session', str(args.replay_input.resolve()), '--record-session', str(replayed)], stdout=log, stderr=subprocess.STDOUT) + try: + running_hash = hashlib.sha256(Path(f'/proc/{process.pid}/exe').read_bytes()).hexdigest() + assert process.wait(timeout=120) == 0, 'Deployment replay failed' + finally: + if process.poll() is None: + process.terminate() + process.wait(timeout=15) + original, restored = load(args.replay_input), load(replayed) + assert [r['frame_index'] for r in original] == [r['frame_index'] for r in restored] + result = {'binary': str(binary), 'binary_sha256': hashlib.sha256(binary.read_bytes()).hexdigest(), + 'running_binary_sha256': running_hash, 'replayed_frames': len(restored)} + assert result['binary_sha256'] == running_hash + (args.output/'report.json').write_text(json.dumps(result, indent=2)+'\n') + print(json.dumps(result, indent=2)) + return + recorded, replayed = args.output / 'recorded', args.output / 'replayed' + term_seconds = run_record(command, recorded, args.output / 'record.log', 12) + original = load(recorded) + with (args.output / 'replay.log').open('w') as log: + process = subprocess.Popen(command + ['--replay-session', str(recorded), '--record-session', str(replayed)], stdout=log, stderr=subprocess.STDOUT) + try: + # Verify the running executable, not just the file on disk. + running_hash = hashlib.sha256(Path(f'/proc/{process.pid}/exe').read_bytes()).hexdigest() + if process.wait(timeout=120): + raise RuntimeError('Replay failed') + finally: + if process.poll() is None: + process.terminate() + process.wait(timeout=15) + restored = load(replayed) + assert len(original) == len(restored) >= 12, 'EOF dropped frames' + max_pose_error = 0.0 + matching_tracks = matching_decisions = 0 + for before, after, stem in zip(original, restored, rows(recorded)): + assert before['frame_index'] == after['frame_index'] + assert before['capture'] == after['source_capture'] + assert before['video_frame_index'] == after['video_frame_index'] + matching_tracks += before['result']['track_ids'] == after['result']['track_ids'] + matching_decisions += before['result']['risk']['decision'] == after['result']['risk']['decision'] + for stage in ('raw_keypoints', 'isolated_keypoints'): + for a, b in zip(before['stages'][stage], after['stages'][stage]): + for ka, kb in zip(a, b): + max_pose_error = max(max_pose_error, *(abs(x-y) for x, y in zip(ka, kb))) + # H.264 is lossy: report algorithm differences, do not require bitwise equality. + probe = json.loads(subprocess.check_output(['ffprobe', '-v', 'error', '-select_streams', 'v:0', '-show_entries', 'stream=codec_name,nb_frames,width,height', '-of', 'json', str(recorded/'frames.mp4')]))['streams'][0] + assert probe['codec_name'] == 'h264' and int(probe['nb_frames']) == len(original) + # Exercise cancellation while replay has queued input, and reuse afterwards. + interrupt_seconds = run_record(command + ['--replay-session', str(recorded)], args.output/'interrupted', args.output/'interrupt.log', 1, signal.SIGINT) + interrupted_count = json.loads((args.output/'interrupted/complete.json').read_text())['frame_count'] + assert 0 < interrupted_count < len(original), 'Cancellation did not stop before EOF' + result = {'platform': os.uname().nodename, 'binary': str(binary), 'binary_sha256': hashlib.sha256(binary.read_bytes()).hexdigest(), + 'running_binary_sha256': running_hash, 'recorded_frames': len(original), 'replayed_frames': len(restored), + 'frames_with_people': sum(bool(row['result']['track_ids']) for row in original), + 'max_raw_pose_error_px': max_pose_error, 'matching_track_frames': matching_tracks, 'matching_decision_frames': matching_decisions, + 'video_bytes': (recorded/'frames.mp4').stat().st_size, 'uncompressed_bgr_bytes': len(original)*int(probe['width'])*int(probe['height'])*3, 'sigterm_seconds': term_seconds, 'replay_sigint_seconds': interrupt_seconds, 'interrupted_frames': interrupted_count} + assert result['binary_sha256'] == running_hash + (args.output/'report.json').write_text(json.dumps(result, indent=2)+'\n') + print(json.dumps(result, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/tests/run_camera_timeout.sh b/tests/run_camera_timeout.sh index b2e795b..d6e1f1f 100755 --- a/tests/run_camera_timeout.sh +++ b/tests/run_camera_timeout.sh @@ -3,7 +3,7 @@ set -euo pipefail root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" test_dir="$(mktemp -d)" trap 'rm -rf "$test_dir"' EXIT -flags=(-std=c++17 -O1 -g -Wall -Wextra -Wpedantic -pthread -fsanitize=undefined -fno-sanitize-recover=all -I"$root/include") +flags=(-std=c++17 -O1 -g -Wall -Wextra -Wpedantic -pthread -fsanitize=undefined -fno-sanitize-recover=all -I"$root/include" -I"$root/third_party") links=() if [[ -f "$root/include/v4l2_capture.h" ]]; then flags+=(-DTEST_V4L2_CAPTURE -I"$root/tests/stubs") diff --git a/tests/run_session.sh b/tests/run_session.sh new file mode 100755 index 0000000..2682348 --- /dev/null +++ b/tests/run_session.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail +root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +test_dir="$(mktemp -d)" +trap 'rm -rf "$test_dir"' EXIT +read -ra opencv_flags <<< "$(pkg-config --cflags --libs opencv4)" +"${CXX:-g++}" -std=c++17 -O1 -g -Wall -Wextra -Wpedantic -fsanitize=undefined -fno-sanitize-recover=all \ + -I"$root/include" -I"$root/third_party" "$root/tests/session_test.cpp" \ + "$root/src/inference_session.cpp" "$root/src/serialization.cpp" "$root/src/door_risk.cpp" \ + "${opencv_flags[@]}" -o "$test_dir/session_test" +"$test_dir/session_test" "$test_dir" diff --git a/tests/session_test.cpp b/tests/session_test.cpp new file mode 100644 index 0000000..183e183 --- /dev/null +++ b/tests/session_test.cpp @@ -0,0 +1,66 @@ +#include "inference_session.hpp" +#include "camera_pipeline.hpp" +#include +#include +#include +#include + +namespace cms { +std::uint64_t monotonicNs() { + return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); +} +} +using namespace cms; +void check(bool condition,char const* message){if(!condition)throw std::runtime_error(message);} +template void rejects(F f){bool failed=false;try{f();}catch(std::exception const&){failed=true;}check(failed,"Expected rejection");} +int main(int argc,char** argv)try { + if(argc!=2)return 2; + std::filesystem::path root=argv[1]; + PipelineConfig c;c.record_session=root/"recorded";c.engines=root;c.door_mask=root/"mask.json"; + c.camera.width=64;c.camera.height=48;c.detector_confidence=.42; + std::ofstream(c.door_mask)<<"{}"; + for(auto name:{"yolo11s.engine","rtmw_l_384.engine"})std::ofstream(root/name)<<"fixture"; + auto im=std::make_shared();im->width=64;im->height=48;im->stride=200; + im->pixels.resize(9600); + for(int y=0;y<48;++y)for(int x=0;x<192;++x)im->pixels[y*200+x]=32+x/3+y%4; + FrameResult r;r.image=im;r.frame_index=3;r.capture.frame_index=3;r.capture.driver_sequence=42; + r.capture.capture_monotonic_ns=123;r.capture.received_monotonic_ns=124; + { + SessionWriter writer(c);writer.write(r,{{"raw_keypoints",{1,2,3}}}); + r.frame_index=9;r.capture.frame_index=9; + for(auto& pixel:im->pixels)pixel+=80; + writer.write(r,{{"raw_keypoints",{4,5,6}}}); + writer.finish();writer.finish(); + rejects([&]{writer.write(r,{});}); + } + rejects([&]{SessionWriter duplicate(c);}); + PipelineConfig replay;replay.replay_session=c.record_session; + SessionReader reader(replay);check(replay.camera.width==64&&replay.detector_confidence==.42,"Settings not restored"); + auto first=reader.read();check(first&&first->metadata.frame_index==3,"First index mismatch"); + check(first->metadata.driver_sequence==42&&first->metadata.capture_monotonic_ns!=123,"Capture metadata mismatch"); + check(first->replay_record->at("capture").at("capture_monotonic_ns")==123,"Original clock lost"); + for(int y=0;y<48;++y)for(int x=0;x<192;++x)check(std::abs(int(first->image->pixels[y*192+x])-(32+x/3+y%4))<12,"Decoded first frame mismatch"); + auto second=reader.read();check(second&&second->metadata.frame_index==9,"Gap/order lost"); + check(second->image->pixels[0]>first->image->pixels[0]+60,"Video frame order lost"); + check(!reader.read(),"EOF not drained"); + // Unindexed temporary metadata must not become input. + std::ofstream(c.record_session/"frames/000000000002.json.tmp")<<"incomplete"; + SessionReader prefix(replay);check(prefix.read()&&prefix.read()&&!prefix.read(),"Uncommitted data replayed"); + std::filesystem::remove(c.record_session/"frames/000000000001.json"); + SessionReader missing(replay);missing.read();rejects([&]{missing.read();}); + std::ofstream(c.record_session/"frames.txt",std::ios::trunc)<<"../bad\n"; + SessionReader invalid(replay);rejects([&]{invalid.read();}); + std::ofstream(c.record_session/"frames.txt",std::ios::trunc)<<"000000000000"; + SessionReader truncated(replay);rejects([&]{truncated.read();}); + std::ofstream(c.record_session/"frames.txt",std::ios::trunc)<<"000000000000\n"; + SessionReader shortIndex(replay);shortIndex.read();rejects([&]{shortIndex.read();}); + std::filesystem::remove(c.record_session/"complete.json"); + rejects([&]{SessionReader incomplete(replay);}); + c.record_session=root/"empty"; + {SessionWriter empty(c);empty.finish();} + replay.replay_session=c.record_session; + SessionReader empty(replay);check(!empty.read(),"Empty recording has frames"); + c.camera.width=63;c.record_session=root/"odd"; + rejects([&]{SessionWriter odd(c);}); + std::cout<<"H.264 encode/decode, settings, order, EOF, stop finalization and corruption checks passed\n"; +}catch(std::exception const& e){std::cerr<