sieni
/
luckfox-pico
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

1) Created saix_threads.h and saix_threads.c 

2) Moved all the thread workers to this file
3) Moved common defines to a newly created common_defines.h
This commit is contained in:
Abhinav Bharti 2025-04-24 00:28:34 +01:00
parent 0fcd0b333d
commit ccac5d7e27
5 changed files with 707 additions and 626 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
project/app/rkipc/rkipc/src/rv1106_ipc/video/common_defines.h Normal file
View File

@ -0,0 +1,36 @@
/*
* Copyright (c) 2025 NeuralSense AI Private Limited
* Trading as swatah.ai. All rights reserved.
*
* This file is part of the swatah.ai software stack and is licensed under
* the terms defined in the accompanying LICENSE file. Unauthorized copying,
* distribution, or modification of this file, via any medium, is strictly prohibited.
*
* For more information, visit: https://swatah.ai
*/
#define RKISP_MAINPATH 0
#define RKISP_SELFPATH 1
#define RKISP_FBCPATH 2
#define VIDEO_PIPE_0 0
#define VIDEO_PIPE_1 1
#define VIDEO_PIPE_2 2
#define VIDEO_PIPE_3 3
#define JPEG_VENC_CHN 4
#define DRAW_NN_VENC_CHN_ID 0
#define VPSS_ROTATE 6
#define VPSS_BGR 0
#define DRAW_NN_OSD_ID 7
#define RED_COLOR 0x0000FF
#define BLUE_COLOR 0xFF0000
#define RK3588_VO_DEV_HDMI 0
#define RK3588_VO_DEV_MIPI 3
#define RK3588_VOP_LAYER_CLUSTER0 0
#define RTSP_URL_0 "/live/0"
#define RTSP_URL_1 "/live/1"
#define RTSP_URL_2 "/live/2"
#define RTMP_URL_0 "rtmp://127.0.0.1:1935/live/mainstream"
#define RTMP_URL_1 "rtmp://127.0.0.1:1935/live/substream"
#define RTMP_URL_2 "rtmp://127.0.0.1:1935/live/thirdstream"

594
project/app/rkipc/rkipc/src/rv1106_ipc/video/saix_threads.c Normal file
View File

@ -0,0 +1,594 @@
/*
* Copyright (c) 2025 NeuralSense AI Private Limited
* Trading as swatah.ai. All rights reserved.
*
* This file is part of the swatah.ai software stack and is licensed under
* the terms defined in the accompanying LICENSE file. Unauthorized copying,
* distribution, or modification of this file, via any medium, is strictly prohibited.
*
* For more information, visit: https://swatah.ai
*/
#include "common.h"
#include "common_defines.h"
#include "saix_threads.h"
#include <rk_mpi_venc.h>
#include <rk_mpi_vi.h>
#include <rk_mpi_vpss.h>
#include <rk_mpi_ivs.h>
#include <rk_debug.h>
#include <rga/rga.h>
#include <rk_comm_tde.h>
#include "rockiva.h"
#include <rga/im2d.h>
#define HAS_VO 0
#if HAS_VO
#include <rk_mpi_vo.h>
#endif
#ifdef LOG_TAG
#undef LOG_TAG
#endif
#define LOG_TAG "saix_threads"
void *saix_send_cam_input_to_display(void *arg) {
LOG_DEBUG("#Start %s thread, arg:%p\n", __func__, arg);
prctl(PR_SET_NAME, "saix_send_cam_input_to_display", 0, 0, 0);
VIDEO_FRAME_INFO_S stViFrame;
VI_CHN_STATUS_S stChnStatus;
int loopCount = 0;
int ret = 0;
while (g_video_run_) {
// 5.get the frame
ret = RK_MPI_VI_GetChnFrame(pipe_id_, VIDEO_PIPE_1, &stViFrame, 1000);
if (ret == RK_SUCCESS) {
void *data = RK_MPI_MB_Handle2VirAddr(stViFrame.stVFrame.pMbBlk);
LOG_ERROR("RK_MPI_VI_GetChnFrame ok:data %p loop:%d pts:%" PRId64 " ms\n", data,
loopCount, stViFrame.stVFrame.u64PTS / 1000);
// 6.get the channel status
// ret = RK_MPI_VI_QueryChnStatus(pipe_id_, VIDEO_PIPE_1, &stChnStatus);
// LOG_ERROR("RK_MPI_VI_QueryChnStatus ret %x, "
// "w:%d,h:%d,enable:%d,lost:%d,framerate:%d,vbfail:%d\n",
// ret, stChnStatus.stSize.u32Width, stChnStatus.stSize.u32Height,
// stChnStatus.bEnable, stChnStatus.u32LostFrame, stChnStatus.u32FrameRate,
// stChnStatus.u32VbFail);
// send vo
ret = RK_MPI_VO_SendFrame(VoLayer, 0, &stViFrame, 1000);
if (ret)
LOG_ERROR("RK_MPI_VO_SendFrame timeout %x\n", ret);
// 7.release the frame
ret = RK_MPI_VI_ReleaseChnFrame(pipe_id_, VIDEO_PIPE_1, &stViFrame);
if (ret != RK_SUCCESS) {
LOG_ERROR("RK_MPI_VI_ReleaseChnFrame fail %x\n", ret);
}
loopCount++;
} else {
LOG_ERROR("RK_MPI_VI_GetChnFrame timeout %x\n", ret);
}
usleep(10 * 1000);
}
return 0;
}
void *saix_rtsp_stream_from_venc0(void *arg) {
LOG_DEBUG("#Start %s thread, arg:%p\n", __func__, arg);
prctl(PR_SET_NAME, "saixIpcVenc0", 0, 0, 0);
VENC_STREAM_S stFrame;
VI_CHN_STATUS_S stChnStatus;
int loopCount = 0;
int ret = 0;
// FILE *fp = fopen("/data/venc.h265", "wb");
stFrame.pstPack = malloc(sizeof(VENC_PACK_S));
while (g_video_run_) {
// 5.get the frame
ret = RK_MPI_VENC_GetStream(VIDEO_PIPE_0, &stFrame, 2500);
if (ret == RK_SUCCESS) {
void *data = RK_MPI_MB_Handle2VirAddr(stFrame.pstPack->pMbBlk);
// fwrite(data, 1, stFrame.pstPack->u32Len, fp);
// fflush(fp);
// LOG_DEBUG("Count:%d, Len:%d, PTS is %" PRId64", enH264EType is %d\n", loopCount,
// stFrame.pstPack->u32Len, stFrame.pstPack->u64PTS,
// stFrame.pstPack->DataType.enH264EType);
rkipc_rtsp_write_video_frame(0, data, stFrame.pstPack->u32Len, stFrame.pstPack->u64PTS);
if ((stFrame.pstPack->DataType.enH264EType == H264E_NALU_IDRSLICE) ||
(stFrame.pstPack->DataType.enH264EType == H264E_NALU_ISLICE) ||
(stFrame.pstPack->DataType.enH265EType == H265E_NALU_IDRSLICE) ||
(stFrame.pstPack->DataType.enH265EType == H265E_NALU_ISLICE)) {
rk_storage_write_video_frame(0, data, stFrame.pstPack->u32Len,
stFrame.pstPack->u64PTS, 1);
if (enable_rtmp)
rk_rtmp_write_video_frame(0, data, stFrame.pstPack->u32Len,
stFrame.pstPack->u64PTS, 1);
} else {
rk_storage_write_video_frame(0, data, stFrame.pstPack->u32Len,
stFrame.pstPack->u64PTS, 0);
if (enable_rtmp)
rk_rtmp_write_video_frame(0, data, stFrame.pstPack->u32Len,
stFrame.pstPack->u64PTS, 0);
}
// 7.release the frame
ret = RK_MPI_VENC_ReleaseStream(VIDEO_PIPE_0, &stFrame);
if (ret != RK_SUCCESS) {
LOG_ERROR("RK_MPI_VENC_ReleaseStream fail %x\n", ret);
}
loopCount++;
} else {
LOG_ERROR("RK_MPI_VENC_GetStream timeout %x\n", ret);
}
}
if (stFrame.pstPack)
free(stFrame.pstPack);
// if (fp)
// fclose(fp);
return 0;
}
void *saix_vi_snapshot_jpeg(void *arg) {
LOG_DEBUG("#Start %s thread, arg:%p\n", __func__, arg);
prctl(PR_SET_NAME, "saixIpcSendJPEG", 0, 0, 0);
int jpeg_width, jpeg_height, ret;
TDE_HANDLE hHandle;
TDE_SURFACE_S pstSrc, pstDst;
TDE_RECT_S pstSrcRect, pstDstRect;
VIDEO_FRAME_INFO_S stViFrame, DstFrame;
PIC_BUF_ATTR_S Dst_stPicBufAttr;
MB_PIC_CAL_S Dst_stMbPicCalResult;
VENC_CHN_ATTR_S pstChnAttr;
MB_BLK dstBlk = RK_NULL;
Dst_stPicBufAttr.u32Width = rk_param_get_int("video.0:max_width", 2304);
Dst_stPicBufAttr.u32Height = rk_param_get_int("video.0:max_height", 1296);
Dst_stPicBufAttr.enPixelFormat = RK_FMT_YUV420SP;
Dst_stPicBufAttr.enCompMode = COMPRESS_MODE_NONE;
ret = RK_MPI_CAL_TDE_GetPicBufferSize(&Dst_stPicBufAttr, &Dst_stMbPicCalResult);
if (ret != RK_SUCCESS) {
LOG_ERROR("get picture buffer size failed. err 0x%x\n", ret);
return NULL;
}
ret = RK_MPI_SYS_MmzAlloc(&dstBlk, RK_NULL, RK_NULL, Dst_stMbPicCalResult.u32MBSize);
if (ret != RK_SUCCESS) {
LOG_ERROR("RK_MPI_SYS_MmzAlloc err 0x%x\n", ret);
return; NULL;
}
pstSrc.enColorFmt = RK_FMT_YUV420SP;
pstSrc.enComprocessMode = COMPRESS_MODE_NONE;
pstSrcRect.s32Xpos = 0;
pstSrcRect.s32Ypos = 0;
pstDst.enColorFmt = RK_FMT_YUV420SP;
pstDst.enComprocessMode = COMPRESS_MODE_NONE;
pstDstRect.s32Xpos = 0;
pstDstRect.s32Ypos = 0;
memset(&DstFrame, 0, sizeof(VIDEO_FRAME_INFO_S));
DstFrame.stVFrame.enPixelFormat = RK_FMT_YUV420SP;
DstFrame.stVFrame.enCompressMode = COMPRESS_MODE_NONE;
ret = RK_TDE_Open();
if (ret != RK_SUCCESS) {
LOG_ERROR("RK_TDE_Open fail %x\n", ret);
RK_MPI_SYS_Free(dstBlk);
return NULL;
}
while (g_video_run_) {
if (!send_jpeg_cnt) {
usleep(300 * 1000);
continue;
}
pstSrc.u32Width = rk_param_get_int("video.0:width", -1);
pstSrc.u32Height = rk_param_get_int("video.0:height", -1);
pstSrcRect.u32Width = rk_param_get_int("video.0:width", -1);
pstSrcRect.u32Height = rk_param_get_int("video.0:height", -1);
jpeg_width = rk_param_get_int("video.jpeg:width", 1920);
jpeg_height = rk_param_get_int("video.jpeg:height", 1080);
ret = RK_MPI_VI_GetChnFrame(pipe_id_, VIDEO_PIPE_0, &stViFrame, 1000);
if (ret == RK_SUCCESS) {
// tde begin job
hHandle = RK_TDE_BeginJob();
if (RK_ERR_TDE_INVALID_HANDLE == hHandle) {
LOG_ERROR("start job fail\n");
RK_MPI_VI_ReleaseChnFrame(pipe_id_, VIDEO_PIPE_0, &stViFrame);
continue;
}
// tde quick resize
pstSrc.pMbBlk = stViFrame.stVFrame.pMbBlk;
pstDst.pMbBlk = dstBlk;
pstDst.u32Width = jpeg_width;
pstDst.u32Height = jpeg_height;
pstDstRect.u32Width = jpeg_width;
pstDstRect.u32Height = jpeg_height;
ret = RK_TDE_QuickResize(hHandle, &pstSrc, &pstSrcRect, &pstDst, &pstDstRect);
if (ret != RK_SUCCESS) {
LOG_ERROR("RK_TDE_QuickResize failed. err 0x%x\n", ret);
RK_TDE_CancelJob(hHandle);
RK_MPI_VI_ReleaseChnFrame(pipe_id_, VIDEO_PIPE_0, &stViFrame);
continue;
}
// tde end job
ret = RK_TDE_EndJob(hHandle, RK_FALSE, RK_TRUE, 10);
if (ret != RK_SUCCESS) {
LOG_ERROR("RK_TDE_EndJob failed. err 0x%x\n", ret);
RK_TDE_CancelJob(hHandle);
RK_MPI_VI_ReleaseChnFrame(pipe_id_, VIDEO_PIPE_0, &stViFrame);
continue;
}
ret = RK_TDE_WaitForDone(hHandle);
if (ret != RK_SUCCESS)
LOG_ERROR("RK_TDE_WaitForDone fail %x\n", ret);
// set jpeg venc w,h
ret = RK_MPI_VENC_GetChnAttr(JPEG_VENC_CHN, &pstChnAttr);
if (ret != RK_SUCCESS)
LOG_ERROR("RK_MPI_VENC_GetChnAttr fail %x\n", ret);
pstChnAttr.stVencAttr.u32PicWidth = pstDst.u32Width;
pstChnAttr.stVencAttr.u32PicHeight = pstDst.u32Height;
ret = RK_MPI_VENC_SetChnAttr(JPEG_VENC_CHN, &pstChnAttr);
if (ret != RK_SUCCESS)
LOG_ERROR("RK_MPI_VENC_SetChnAttr fail %x\n", ret);
// send frame to jpeg venc
DstFrame.stVFrame.pMbBlk = dstBlk;
DstFrame.stVFrame.u32Width = pstDst.u32Width;
DstFrame.stVFrame.u32Height = pstDst.u32Height;
DstFrame.stVFrame.u32VirWidth = pstDst.u32Width;
DstFrame.stVFrame.u32VirHeight = pstDst.u32Height;
ret = RK_MPI_VENC_SendFrame(JPEG_VENC_CHN, &DstFrame, 1000);
if (ret != RK_SUCCESS)
LOG_ERROR("RK_MPI_VENC_SendFrame fail %x\n", ret);
// release the frame
ret = RK_MPI_VI_ReleaseChnFrame(pipe_id_, VIDEO_PIPE_0, &stViFrame);
if (ret != RK_SUCCESS)
LOG_ERROR("RK_MPI_VI_ReleaseChnFrame fail %x\n", ret);
} else {
LOG_ERROR("RK_MPI_VI_GetChnFrame timeout %x\n", ret);
}
send_jpeg_cnt--;
}
RK_TDE_Close();
RK_MPI_SYS_Free(dstBlk);
return NULL;
}
void *saix_vi_overlay_to_venc(void *arg) {
LOG_DEBUG("#Start %s thread, arg:%p\n", __func__, arg);
prctl(PR_SET_NAME, "saixIpcVi2Venc", 0, 0, 0);
VIDEO_FRAME_INFO_S stViFrame;
VI_CHN_STATUS_S stChnStatus;
int loopCount = 0;
int ret = 0;
int line_pixel = 2;
long long last_ba_result_time;
RockIvaBaResult ba_result;
im_handle_param_t param;
RockIvaBaObjectInfo *object;
rga_buffer_handle_t handle;
rga_buffer_t src;
memset(&ba_result, 0, sizeof(ba_result));
memset(&param, 0, sizeof(im_handle_param_t));
while (g_video_run_) {
// 5.get the frame
ret = RK_MPI_VI_GetChnFrame(pipe_id_, VIDEO_PIPE_1, &stViFrame, 1000);
if (ret == RK_SUCCESS) {
uint64_t phy_data = RK_MPI_MB_Handle2PhysAddr(stViFrame.stVFrame.pMbBlk);
// LOG_DEBUG("phy_data %p, loop:%d pts:%" PRId64 " ms\n", phy_data, loopCount,
// stViFrame.stVFrame.u64PTS / 1000);
ret = rkipc_rknn_object_get(&ba_result);
if ((!ret && ba_result.objNum) ||
((ret == -1) && (rkipc_get_curren_time_ms() - last_ba_result_time < 300))) {
// LOG_DEBUG("ret is %d, ba_result.objNum is %d\n", ret, ba_result.objNum);
handle = importbuffer_physicaladdr(phy_data, &param);
src = wrapbuffer_handle_t(handle, stViFrame.stVFrame.u32Width,
stViFrame.stVFrame.u32Height, stViFrame.stVFrame.u32Width,
stViFrame.stVFrame.u32Height, RK_FORMAT_YCbCr_420_SP);
if (!ret)
last_ba_result_time = rkipc_get_curren_time_ms();
for (int i = 0; i < ba_result.objNum; i++) {
int x, y, w, h;
object = &ba_result.triggerObjects[i];
LOG_DEBUG("topLeft:[%d,%d], bottomRight:[%d,%d],"
"objId is %d, frameId is %d, score is %d, type is %d\n",
object->objInfo.rect.topLeft.x, object->objInfo.rect.topLeft.y,
object->objInfo.rect.bottomRight.x,
object->objInfo.rect.bottomRight.y, object->objInfo.objId,
object->objInfo.frameId, object->objInfo.score, object->objInfo.type);
x = stViFrame.stVFrame.u32Width * object->objInfo.rect.topLeft.x / 10000;
y = stViFrame.stVFrame.u32Height * object->objInfo.rect.topLeft.y / 10000;
w = stViFrame.stVFrame.u32Width *
(object->objInfo.rect.bottomRight.x - object->objInfo.rect.topLeft.x) /
10000;
h = stViFrame.stVFrame.u32Height *
(object->objInfo.rect.bottomRight.y - object->objInfo.rect.topLeft.y) /
10000;
x = x / 2 * 2;
y = y / 2 * 2;
w = w / 2 * 2;
h = h / 2 * 2;
while (x + w + line_pixel >= stViFrame.stVFrame.u32Width) {
w -= 8;
}
while (y + h + line_pixel >= stViFrame.stVFrame.u32Height) {
h -= 8;
}
LOG_DEBUG("i is %d, x,y,w,h is %d,%d,%d,%d\n", i, x, y, w, h);
rga_nv12_border(src, x, y, w, h, line_pixel, 0x000000ff);
// LOG_INFO("draw rect time-consuming is %lld\n",(rkipc_get_curren_time_ms() -
// last_ba_result_time));
// LOG_INFO("triggerRules is %d, ruleID is %d, triggerType is %d\n",
// object->triggerRules,
// object->firstTrigger.ruleID,
// object->firstTrigger.triggerType);
}
releasebuffer_handle(handle);
}
// send venc
ret = RK_MPI_VENC_SendFrame(VIDEO_PIPE_1, &stViFrame, 1000);
if (ret)
LOG_ERROR("RK_MPI_VENC_SendFrame timeout %x\n", ret);
// 7.release the frame
ret = RK_MPI_VI_ReleaseChnFrame(pipe_id_, VIDEO_PIPE_1, &stViFrame);
if (ret != RK_SUCCESS) {
LOG_ERROR("RK_MPI_VI_ReleaseChnFrame fail %x\n", ret);
}
loopCount++;
} else {
LOG_ERROR("RK_MPI_VI_GetChnFrame timeout %x\n", ret);
}
}
return 0;
}
void *saix_venc1_dispatch(void *arg) {
LOG_DEBUG("#Start %s thread, arg:%p\n", __func__, arg);
prctl(PR_SET_NAME, "saixIcVenc1", 0, 0, 0);
VENC_STREAM_S stFrame;
VI_CHN_STATUS_S stChnStatus;
int loopCount = 0;
int ret = 0;
stFrame.pstPack = malloc(sizeof(VENC_PACK_S));
while (g_video_run_) {
// 5.get the frame
ret = RK_MPI_VENC_GetStream(VIDEO_PIPE_1, &stFrame, 2500);
if (ret == RK_SUCCESS) {
void *data = RK_MPI_MB_Handle2VirAddr(stFrame.pstPack->pMbBlk);
// LOG_INFO("Count:%d, Len:%d, PTS is %" PRId64", enH264EType is %d\n", loopCount,
// stFrame.pstPack->u32Len, stFrame.pstPack->u64PTS,
// stFrame.pstPack->DataType.enH264EType);
rkipc_rtsp_write_video_frame(1, data, stFrame.pstPack->u32Len, stFrame.pstPack->u64PTS);
if ((stFrame.pstPack->DataType.enH264EType == H264E_NALU_IDRSLICE) ||
(stFrame.pstPack->DataType.enH264EType == H264E_NALU_ISLICE) ||
(stFrame.pstPack->DataType.enH265EType == H265E_NALU_IDRSLICE) ||
(stFrame.pstPack->DataType.enH265EType == H265E_NALU_ISLICE)) {
rk_storage_write_video_frame(1, data, stFrame.pstPack->u32Len,
stFrame.pstPack->u64PTS, 1);
if (enable_rtmp)
rk_rtmp_write_video_frame(1, data, stFrame.pstPack->u32Len,
stFrame.pstPack->u64PTS, 1);
} else {
rk_storage_write_video_frame(1, data, stFrame.pstPack->u32Len,
stFrame.pstPack->u64PTS, 0);
if (enable_rtmp)
rk_rtmp_write_video_frame(1, data, stFrame.pstPack->u32Len,
stFrame.pstPack->u64PTS, 0);
}
// 7.release the frame
ret = RK_MPI_VENC_ReleaseStream(VIDEO_PIPE_1, &stFrame);
if (ret != RK_SUCCESS)
LOG_ERROR("RK_MPI_VENC_ReleaseStream fail %x\n", ret);
loopCount++;
} else {
LOG_ERROR("RK_MPI_VENC_GetStream timeout %x\n", ret);
}
}
if (stFrame.pstPack)
free(stFrame.pstPack);
return 0;
}
void *saix_process_jpeg(void *arg) {
LOG_DEBUG("#Start %s thread, arg:%p\n", __func__, arg);
prctl(PR_SET_NAME, "saixIpcGetJpeg", 0, 0, 0);
VENC_STREAM_S stFrame;
int loopCount = 0;
int ret = 0;
char file_name[128] = {0};
char record_path[256];
memset(&record_path, 0, sizeof(record_path));
strcat(record_path, rk_param_get_string("storage:mount_path", "/userdata"));
strcat(record_path, "/");
strcat(record_path, rk_param_get_string("storage.0:folder_name", "video0"));
stFrame.pstPack = malloc(sizeof(VENC_PACK_S));
// drop first frame
ret = RK_MPI_VENC_GetStream(JPEG_VENC_CHN, &stFrame, 1000);
if (ret == RK_SUCCESS)
RK_MPI_VENC_ReleaseStream(JPEG_VENC_CHN, &stFrame);
else
LOG_ERROR("RK_MPI_VENC_GetStream timeout %x\n", ret);
while (g_video_run_) {
if (!get_jpeg_cnt) {
usleep(300 * 1000);
continue;
}
// get the frame
ret = RK_MPI_VENC_GetStream(JPEG_VENC_CHN, &stFrame, 1000);
if (ret == RK_SUCCESS) {
void *data = RK_MPI_MB_Handle2VirAddr(stFrame.pstPack->pMbBlk);
LOG_DEBUG("Count:%d, Len:%d, PTS is %" PRId64 ", enH264EType is %d\n", loopCount,
stFrame.pstPack->u32Len, stFrame.pstPack->u64PTS,
stFrame.pstPack->DataType.enH264EType);
// save jpeg file
time_t t = time(NULL);
struct tm tm = *localtime(&t);
snprintf(file_name, 128, "%s/%d%02d%02d%02d%02d%02d.jpeg", record_path,
tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min,
tm.tm_sec);
LOG_DEBUG("file_name is %s, u32Len is %d\n", file_name, stFrame.pstPack->u32Len);
FILE *fp = fopen(file_name, "wb");
if (fp == NULL) {
LOG_ERROR("fp is NULL\n");
} else {
fwrite(data, 1, stFrame.pstPack->u32Len, fp);
}
// release the frame
ret = RK_MPI_VENC_ReleaseStream(JPEG_VENC_CHN, &stFrame);
if (ret != RK_SUCCESS) {
LOG_ERROR("RK_MPI_VENC_ReleaseStream fail %x\n", ret);
}
if (fp) {
fflush(fp);
fclose(fp);
}
loopCount++;
} else {
LOG_ERROR("RK_MPI_VENC_GetStream timeout %x\n", ret);
}
get_jpeg_cnt--;
RK_MPI_VENC_StopRecvFrame(JPEG_VENC_CHN);
}
if (stFrame.pstPack)
free(stFrame.pstPack);
return 0;
}
void *saix_cycle_snapshot(void *arg) {
LOG_INFO("start %s thread, arg:%p\n", __func__, arg);
prctl(PR_SET_NAME, "RkipcCycleSnapshot", 0, 0, 0);
while (g_video_run_ && cycle_snapshot_flag) {
usleep(rk_param_get_int("video.jpeg:snapshot_interval_ms", 1000) * 1000);
rk_take_photo();
}
LOG_INFO("exit %s thread, arg:%p\n", __func__, arg);
return 0;
}
void *saix_push_vi2_to_iva(void *arg) {
LOG_DEBUG("#Start %s thread, arg:%p\n", __func__, arg);
prctl(PR_SET_NAME, "saixIpcGetVi2", 0, 0, 0);
int ret;
int32_t loopCount = 0;
VIDEO_FRAME_INFO_S stViFrame;
int npu_cycle_time_ms = 1000 / rk_param_get_int("video.source:npu_fps", 10);
long long before_time, cost_time;
while (g_video_run_) {
before_time = rkipc_get_curren_time_ms();
ret = RK_MPI_VI_GetChnFrame(pipe_id_, VIDEO_PIPE_2, &stViFrame, 1000);
if (ret == RK_SUCCESS) {
void *data = RK_MPI_MB_Handle2VirAddr(stViFrame.stVFrame.pMbBlk);
uint8_t *phy_addr = (uint8_t *)RK_MPI_MB_Handle2PhysAddr(stViFrame.stVFrame.pMbBlk);
rkipc_rockiva_write_nv12_frame_by_phy_addr(
stViFrame.stVFrame.u32Width, stViFrame.stVFrame.u32Height, loopCount, phy_addr);
ret = RK_MPI_VI_ReleaseChnFrame(pipe_id_, VIDEO_PIPE_2, &stViFrame);
if (ret != RK_SUCCESS)
LOG_ERROR("RK_MPI_VI_ReleaseChnFrame fail %x", ret);
loopCount++;
} else {
LOG_ERROR("RK_MPI_VI_GetChnFrame timeout %x", ret);
}
cost_time = rkipc_get_curren_time_ms() - before_time;
if ((cost_time > 0) && (cost_time < npu_cycle_time_ms))
usleep((npu_cycle_time_ms - cost_time) * 1000);
}
return NULL;
}
void *saix_get_vpss_bgr_frame(void *arg) {
LOG_DEBUG("#Start %s thread, arg:%p\n", __func__, arg);
prctl(PR_SET_NAME, "RkipcGetVpssBgr", 0, 0, 0);
VIDEO_FRAME_INFO_S frame;
VI_CHN_STATUS_S stChnStatus;
int32_t loopCount = 0;
int ret = 0;
while (g_video_run_) {
ret = RK_MPI_VPSS_GetChnFrame(VPSS_BGR, 0, &frame, 1000);
if (ret == RK_SUCCESS) {
void *data = RK_MPI_MB_Handle2VirAddr(frame.stVFrame.pMbBlk);
// LOG_INFO("data:%p, u32Width:%d, u32Height:%d, PTS is %" PRId64 "\n", data,
// frame.stVFrame.u32Width, frame.stVFrame.u32Height, frame.stVFrame.u64PTS);
// rkipc_rockiva_write_rgb888_frame(frame.stVFrame.u32Width, frame.stVFrame.u32Height,
// data);
int32_t fd = RK_MPI_MB_Handle2Fd(frame.stVFrame.pMbBlk);
#if 0
FILE *fp = fopen("/data/test.bgr", "wb");
fwrite(data, 1, frame.stVFrame.u32Width * frame.stVFrame.u32Height * 3, fp);
fflush(fp);
fclose(fp);
exit(1);
#endif
// long long last_nn_time = rkipc_get_curren_time_ms();
rkipc_rockiva_write_rgb888_frame_by_fd(frame.stVFrame.u32Width,
frame.stVFrame.u32Height, loopCount, fd);
// LOG_DEBUG("nn time-consuming is %lld\n",(rkipc_get_curren_time_ms() - last_nn_time));
ret = RK_MPI_VPSS_ReleaseChnFrame(VPSS_BGR, 0, &frame);
if (ret != RK_SUCCESS)
LOG_ERROR("RK_MPI_VPSS_ReleaseChnFrame fail %x\n", ret);
loopCount++;
} else {
LOG_ERROR("RK_MPI_VPSS_GetChnFrame timeout %x\n", ret);
}
}
return 0;
}
void *saix_fetch_ivs_results(void *arg) {
LOG_DEBUG("#Start %s thread, arg:%p\n", __func__, arg);
prctl(PR_SET_NAME, "saixIpcGetIVS", 0, 0, 0);
int ret, i;
IVS_RESULT_INFO_S stResults;
int resultscount = 0;
int count = 0;
int md = rk_param_get_int("ivs:md", 0);
int od = rk_param_get_int("ivs:od", 0);
int width = rk_param_get_int("video.2:width", 960);
int height = rk_param_get_int("video.2:height", 540);
int md_area_threshold = width * height * 0.3;
while (g_video_run_) {
ret = RK_MPI_IVS_GetResults(0, &stResults, 1000);
if (ret >= 0) {
resultscount++;
if (md == 1) {
if (stResults.pstResults->stMdInfo.u32Square > md_area_threshold) {
LOG_INFO("MD: md_area is %d, md_area_threshold is %d\n",
stResults.pstResults->stMdInfo.u32Square, md_area_threshold);
}
}
if (od == 1) {
if (stResults.s32ResultNum > 0) {
if (stResults.pstResults->stOdInfo.u32Flag)
LOG_INFO("OD flag:%d\n", stResults.pstResults->stOdInfo.u32Flag);
}
}
RK_MPI_IVS_ReleaseResults(0, &stResults);
} else {
LOG_ERROR("get chn %d fail %d\n", 0, ret);
usleep(50000llu);
}
}
return NULL;
}

26
project/app/rkipc/rkipc/src/rv1106_ipc/video/saix_threads.h Normal file
View File

@ -0,0 +1,26 @@
#include<rk_common.h>
extern int g_video_run_;
extern int enable_ivs, enable_jpeg, enable_venc_0, enable_venc_1, enable_rtsp, enable_rtmp;
extern int pipe_id_;
extern int cycle_snapshot_flag;
extern int send_jpeg_cnt;
extern int get_jpeg_cnt;
extern VO_DEV VoLayer;
#if HAS_VO
void *saix_send_cam_input_to_display(void *arg);
#endif
void *saix_rtsp_stream_from_venc0(void *arg);
void *saix_vi_snapshot_jpeg(void *arg);
void *saix_vi_overlay_to_venc(void *arg);
void *saix_venc1_dispatch(void *arg);
void *saix_process_jpeg(void *arg);
void *saix_cycle_snapshot(void *arg);
void *saix_push_vi2_to_iva(void *arg);
void *saix_get_vpss_bgr_frame(void *arg);
void *saix_fetch_ivs_results(void *arg);
void *saix_push_vi2_to_iva(void *arg);

660
project/app/rkipc/rkipc/src/rv1106_ipc/video/video.c
View File

@ -1,56 +1,36 @@
// Copyright 2022 Rockchip Electronics Co., Ltd. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
/*
* Copyright (c) 2025 NeuralSense AI Private Limited
* Trading as swatah.ai. All rights reserved.
*
* This file is part of the swatah.ai software stack and is licensed under
* the terms defined in the accompanying LICENSE file. Unauthorized copying,
* distribution, or modification of this file, via any medium, is strictly prohibited.
*
* For more information, visit: https://swatah.ai
*/
#include "common_defines.h"
#include "video.h"
#include "audio.h"
#include "rockiva.h"
#define HAS_VO 0
#if HAS_VO
#include <rk_mpi_vo.h>
#endif
#include "saix_threads.h"
#ifdef LOG_TAG
#undef LOG_TAG
#endif
#define LOG_TAG "video.c"
#define RKISP_MAINPATH 0
#define RKISP_SELFPATH 1
#define RKISP_FBCPATH 2
#define VIDEO_PIPE_0 0
#define VIDEO_PIPE_1 1
#define VIDEO_PIPE_2 2
#define VIDEO_PIPE_3 3
#define JPEG_VENC_CHN 4
#define DRAW_NN_VENC_CHN_ID 0
#define VPSS_ROTATE 6
#define VPSS_BGR 0
#define DRAW_NN_OSD_ID 7
#define RED_COLOR 0x0000FF
#define BLUE_COLOR 0xFF0000
#define RK3588_VO_DEV_HDMI 0
#define RK3588_VO_DEV_MIPI 3
#define RK3588_VOP_LAYER_CLUSTER0 0
#define RTSP_URL_0 "/live/0"
#define RTSP_URL_1 "/live/1"
#define RTSP_URL_2 "/live/2"
#define RTMP_URL_0 "rtmp://127.0.0.1:1935/live/mainstream"
#define RTMP_URL_1 "rtmp://127.0.0.1:1935/live/substream"
#define RTMP_URL_2 "rtmp://127.0.0.1:1935/live/thirdstream"
static int send_jpeg_cnt = 0;
static int get_jpeg_cnt = 0;
static int enable_ivs, enable_jpeg, enable_venc_0, enable_venc_1, enable_rtsp, enable_rtmp;
int send_jpeg_cnt = 0;
int get_jpeg_cnt = 0;
int enable_ivs, enable_jpeg, enable_venc_0, enable_venc_1, enable_rtsp, enable_rtmp;
static int g_enable_vo, g_vo_dev_id, g_vi_chn_id, enable_npu, enable_osd;
static int g_video_run_ = 1;
int g_video_run_ = 1;
static int g_nn_osd_run_ = 0;
static int pipe_id_ = 0;
int pipe_id_ = 0;
static int dev_id_ = 0;
static int cycle_snapshot_flag = 0;
int cycle_snapshot_flag = 0;
static const char *tmp_output_data_type = "H.264";
static const char *tmp_rc_mode;
static const char *tmp_h264_profile;
@ -62,586 +42,24 @@ static pthread_t vi_thread_1, venc_thread_0, venc_thread_1, venc_thread_2, jpeg_
get_vi_send_jpeg_thread_id, get_vi_2_send_thread, get_ivs_result_thread;
static MPP_CHN_S vi_chn, vpss_bgr_chn, vpss_rotate_chn, vo_chn, vpss_out_chn[4], venc_chn, ivs_chn;
static VO_DEV VoLayer = RK3588_VOP_LAYER_CLUSTER0;
VO_DEV VoLayer = RK3588_VOP_LAYER_CLUSTER0;
typedef enum rkCOLOR_INDEX_E {
RGN_COLOR_LUT_INDEX_0 = 0,
RGN_COLOR_LUT_INDEX_1 = 1,
} COLOR_INDEX_E;
#if HAS_VO
static void *get_vi_send_vo(void *arg) {
LOG_DEBUG("#Start %s thread, arg:%p\n", __func__, arg);
prctl(PR_SET_NAME, "get_vi_send_vo", 0, 0, 0);
VIDEO_FRAME_INFO_S stViFrame;
VI_CHN_STATUS_S stChnStatus;
int loopCount = 0;
int ret = 0;
while (g_video_run_) {
// 5.get the frame
ret = RK_MPI_VI_GetChnFrame(pipe_id_, VIDEO_PIPE_1, &stViFrame, 1000);
if (ret == RK_SUCCESS) {
void *data = RK_MPI_MB_Handle2VirAddr(stViFrame.stVFrame.pMbBlk);
LOG_ERROR("RK_MPI_VI_GetChnFrame ok:data %p loop:%d pts:%" PRId64 " ms\n", data,
loopCount, stViFrame.stVFrame.u64PTS / 1000);
// 6.get the channel status
// ret = RK_MPI_VI_QueryChnStatus(pipe_id_, VIDEO_PIPE_1, &stChnStatus);
// LOG_ERROR("RK_MPI_VI_QueryChnStatus ret %x, "
// "w:%d,h:%d,enable:%d,lost:%d,framerate:%d,vbfail:%d\n",
// ret, stChnStatus.stSize.u32Width, stChnStatus.stSize.u32Height,
// stChnStatus.bEnable, stChnStatus.u32LostFrame, stChnStatus.u32FrameRate,
// stChnStatus.u32VbFail);
// send vo
ret = RK_MPI_VO_SendFrame(VoLayer, 0, &stViFrame, 1000);
if (ret)
LOG_ERROR("RK_MPI_VO_SendFrame timeout %x\n", ret);
// 7.release the frame
ret = RK_MPI_VI_ReleaseChnFrame(pipe_id_, VIDEO_PIPE_1, &stViFrame);
if (ret != RK_SUCCESS) {
LOG_ERROR("RK_MPI_VI_ReleaseChnFrame fail %x\n", ret);
}
loopCount++;
} else {
LOG_ERROR("RK_MPI_VI_GetChnFrame timeout %x\n", ret);
}
usleep(10 * 1000);
}
return 0;
}
#endif
static void *rkipc_get_venc_0(void *arg) {
LOG_DEBUG("#Start %s thread, arg:%p\n", __func__, arg);
prctl(PR_SET_NAME, "RkipcVenc0", 0, 0, 0);
VENC_STREAM_S stFrame;
VI_CHN_STATUS_S stChnStatus;
int loopCount = 0;
int ret = 0;
// FILE *fp = fopen("/data/venc.h265", "wb");
stFrame.pstPack = malloc(sizeof(VENC_PACK_S));
while (g_video_run_) {
// 5.get the frame
ret = RK_MPI_VENC_GetStream(VIDEO_PIPE_0, &stFrame, 2500);
if (ret == RK_SUCCESS) {
void *data = RK_MPI_MB_Handle2VirAddr(stFrame.pstPack->pMbBlk);
// fwrite(data, 1, stFrame.pstPack->u32Len, fp);
// fflush(fp);
// LOG_DEBUG("Count:%d, Len:%d, PTS is %" PRId64", enH264EType is %d\n", loopCount,
// stFrame.pstPack->u32Len, stFrame.pstPack->u64PTS,
// stFrame.pstPack->DataType.enH264EType);
rkipc_rtsp_write_video_frame(0, data, stFrame.pstPack->u32Len, stFrame.pstPack->u64PTS);
if ((stFrame.pstPack->DataType.enH264EType == H264E_NALU_IDRSLICE) ||
(stFrame.pstPack->DataType.enH264EType == H264E_NALU_ISLICE) ||
(stFrame.pstPack->DataType.enH265EType == H265E_NALU_IDRSLICE) ||
(stFrame.pstPack->DataType.enH265EType == H265E_NALU_ISLICE)) {
rk_storage_write_video_frame(0, data, stFrame.pstPack->u32Len,
stFrame.pstPack->u64PTS, 1);
if (enable_rtmp)
rk_rtmp_write_video_frame(0, data, stFrame.pstPack->u32Len,
stFrame.pstPack->u64PTS, 1);
} else {
rk_storage_write_video_frame(0, data, stFrame.pstPack->u32Len,
stFrame.pstPack->u64PTS, 0);
if (enable_rtmp)
rk_rtmp_write_video_frame(0, data, stFrame.pstPack->u32Len,
stFrame.pstPack->u64PTS, 0);
}
// 7.release the frame
ret = RK_MPI_VENC_ReleaseStream(VIDEO_PIPE_0, &stFrame);
if (ret != RK_SUCCESS) {
LOG_ERROR("RK_MPI_VENC_ReleaseStream fail %x\n", ret);
}
loopCount++;
} else {
LOG_ERROR("RK_MPI_VENC_GetStream timeout %x\n", ret);
}
}
if (stFrame.pstPack)
free(stFrame.pstPack);
// if (fp)
// fclose(fp);
return 0;
}
static void *rkipc_get_vi_send_jpeg(void *arg) {
LOG_DEBUG("#Start %s thread, arg:%p\n", __func__, arg);
prctl(PR_SET_NAME, "RkipcSendJPEG", 0, 0, 0);
int jpeg_width, jpeg_height, ret;
TDE_HANDLE hHandle;
TDE_SURFACE_S pstSrc, pstDst;
TDE_RECT_S pstSrcRect, pstDstRect;
VIDEO_FRAME_INFO_S stViFrame, DstFrame;
PIC_BUF_ATTR_S Dst_stPicBufAttr;
MB_PIC_CAL_S Dst_stMbPicCalResult;
VENC_CHN_ATTR_S pstChnAttr;
MB_BLK dstBlk = RK_NULL;
Dst_stPicBufAttr.u32Width = rk_param_get_int("video.0:max_width", 2304);
Dst_stPicBufAttr.u32Height = rk_param_get_int("video.0:max_height", 1296);
Dst_stPicBufAttr.enPixelFormat = RK_FMT_YUV420SP;
Dst_stPicBufAttr.enCompMode = COMPRESS_MODE_NONE;
ret = RK_MPI_CAL_TDE_GetPicBufferSize(&Dst_stPicBufAttr, &Dst_stMbPicCalResult);
if (ret != RK_SUCCESS) {
LOG_ERROR("get picture buffer size failed. err 0x%x\n", ret);
return;
}
ret = RK_MPI_SYS_MmzAlloc(&dstBlk, RK_NULL, RK_NULL, Dst_stMbPicCalResult.u32MBSize);
if (ret != RK_SUCCESS) {
LOG_ERROR("RK_MPI_SYS_MmzAlloc err 0x%x\n", ret);
return;
}
pstSrc.enColorFmt = RK_FMT_YUV420SP;
pstSrc.enComprocessMode = COMPRESS_MODE_NONE;
pstSrcRect.s32Xpos = 0;
pstSrcRect.s32Ypos = 0;
pstDst.enColorFmt = RK_FMT_YUV420SP;
pstDst.enComprocessMode = COMPRESS_MODE_NONE;
pstDstRect.s32Xpos = 0;
pstDstRect.s32Ypos = 0;
memset(&DstFrame, 0, sizeof(VIDEO_FRAME_INFO_S));
DstFrame.stVFrame.enPixelFormat = RK_FMT_YUV420SP;
DstFrame.stVFrame.enCompressMode = COMPRESS_MODE_NONE;
ret = RK_TDE_Open();
if (ret != RK_SUCCESS) {
LOG_ERROR("RK_TDE_Open fail %x\n", ret);
RK_MPI_SYS_Free(dstBlk);
return;
}
while (g_video_run_) {
if (!send_jpeg_cnt) {
usleep(300 * 1000);
continue;
}
pstSrc.u32Width = rk_param_get_int("video.0:width", -1);
pstSrc.u32Height = rk_param_get_int("video.0:height", -1);
pstSrcRect.u32Width = rk_param_get_int("video.0:width", -1);
pstSrcRect.u32Height = rk_param_get_int("video.0:height", -1);
jpeg_width = rk_param_get_int("video.jpeg:width", 1920);
jpeg_height = rk_param_get_int("video.jpeg:height", 1080);
ret = RK_MPI_VI_GetChnFrame(pipe_id_, VIDEO_PIPE_0, &stViFrame, 1000);
if (ret == RK_SUCCESS) {
// tde begin job
hHandle = RK_TDE_BeginJob();
if (RK_ERR_TDE_INVALID_HANDLE == hHandle) {
LOG_ERROR("start job fail\n");
RK_MPI_VI_ReleaseChnFrame(pipe_id_, VIDEO_PIPE_0, &stViFrame);
continue;
}
// tde quick resize
pstSrc.pMbBlk = stViFrame.stVFrame.pMbBlk;
pstDst.pMbBlk = dstBlk;
pstDst.u32Width = jpeg_width;
pstDst.u32Height = jpeg_height;
pstDstRect.u32Width = jpeg_width;
pstDstRect.u32Height = jpeg_height;
ret = RK_TDE_QuickResize(hHandle, &pstSrc, &pstSrcRect, &pstDst, &pstDstRect);
if (ret != RK_SUCCESS) {
LOG_ERROR("RK_TDE_QuickResize failed. err 0x%x\n", ret);
RK_TDE_CancelJob(hHandle);
RK_MPI_VI_ReleaseChnFrame(pipe_id_, VIDEO_PIPE_0, &stViFrame);
continue;
}
// tde end job
ret = RK_TDE_EndJob(hHandle, RK_FALSE, RK_TRUE, 10);
if (ret != RK_SUCCESS) {
LOG_ERROR("RK_TDE_EndJob failed. err 0x%x\n", ret);
RK_TDE_CancelJob(hHandle);
RK_MPI_VI_ReleaseChnFrame(pipe_id_, VIDEO_PIPE_0, &stViFrame);
continue;
}
ret = RK_TDE_WaitForDone(hHandle);
if (ret != RK_SUCCESS)
LOG_ERROR("RK_TDE_WaitForDone fail %x\n", ret);
// set jpeg venc w,h
ret = RK_MPI_VENC_GetChnAttr(JPEG_VENC_CHN, &pstChnAttr);
if (ret != RK_SUCCESS)
LOG_ERROR("RK_MPI_VENC_GetChnAttr fail %x\n", ret);
pstChnAttr.stVencAttr.u32PicWidth = pstDst.u32Width;
pstChnAttr.stVencAttr.u32PicHeight = pstDst.u32Height;
ret = RK_MPI_VENC_SetChnAttr(JPEG_VENC_CHN, &pstChnAttr);
if (ret != RK_SUCCESS)
LOG_ERROR("RK_MPI_VENC_SetChnAttr fail %x\n", ret);
// send frame to jpeg venc
DstFrame.stVFrame.pMbBlk = dstBlk;
DstFrame.stVFrame.u32Width = pstDst.u32Width;
DstFrame.stVFrame.u32Height = pstDst.u32Height;
DstFrame.stVFrame.u32VirWidth = pstDst.u32Width;
DstFrame.stVFrame.u32VirHeight = pstDst.u32Height;
ret = RK_MPI_VENC_SendFrame(JPEG_VENC_CHN, &DstFrame, 1000);
if (ret != RK_SUCCESS)
LOG_ERROR("RK_MPI_VENC_SendFrame fail %x\n", ret);
// release the frame
ret = RK_MPI_VI_ReleaseChnFrame(pipe_id_, VIDEO_PIPE_0, &stViFrame);
if (ret != RK_SUCCESS)
LOG_ERROR("RK_MPI_VI_ReleaseChnFrame fail %x\n", ret);
} else {
LOG_ERROR("RK_MPI_VI_GetChnFrame timeout %x\n", ret);
}
send_jpeg_cnt--;
}
RK_TDE_Close();
RK_MPI_SYS_Free(dstBlk);
return NULL;
}
static int rga_nv12_border(rga_buffer_t buf, int x, int y, int width, int height, int line_pixel,
int color) {
im_rect rect_up = {x, y, width, line_pixel};
im_rect rect_buttom = {x, y + height - line_pixel, width, line_pixel};
im_rect rect_left = {x, y, line_pixel, height};
im_rect rect_right = {x + width - line_pixel, y, line_pixel, height};
IM_STATUS STATUS = imfill(buf, rect_up, color);
STATUS |= imfill(buf, rect_buttom, color);
STATUS |= imfill(buf, rect_left, color);
STATUS |= imfill(buf, rect_right, color);
return STATUS == IM_STATUS_SUCCESS ? 0 : 1;
}
static void *rkipc_get_vi_draw_send_venc(void *arg) {
LOG_DEBUG("#Start %s thread, arg:%p\n", __func__, arg);
prctl(PR_SET_NAME, "RkipcVi2Venc", 0, 0, 0);
VIDEO_FRAME_INFO_S stViFrame;
VI_CHN_STATUS_S stChnStatus;
int loopCount = 0;
int ret = 0;
int line_pixel = 2;
long long last_ba_result_time;
RockIvaBaResult ba_result;
im_handle_param_t param;
RockIvaBaObjectInfo *object;
rga_buffer_handle_t handle;
rga_buffer_t src;
memset(&ba_result, 0, sizeof(ba_result));
memset(&param, 0, sizeof(im_handle_param_t));
while (g_video_run_) {
// 5.get the frame
ret = RK_MPI_VI_GetChnFrame(pipe_id_, VIDEO_PIPE_1, &stViFrame, 1000);
if (ret == RK_SUCCESS) {
uint64_t phy_data = RK_MPI_MB_Handle2PhysAddr(stViFrame.stVFrame.pMbBlk);
// LOG_DEBUG("phy_data %p, loop:%d pts:%" PRId64 " ms\n", phy_data, loopCount,
// stViFrame.stVFrame.u64PTS / 1000);
ret = rkipc_rknn_object_get(&ba_result);
if ((!ret && ba_result.objNum) ||
((ret == -1) && (rkipc_get_curren_time_ms() - last_ba_result_time < 300))) {
// LOG_DEBUG("ret is %d, ba_result.objNum is %d\n", ret, ba_result.objNum);
handle = importbuffer_physicaladdr(phy_data, &param);
src = wrapbuffer_handle_t(handle, stViFrame.stVFrame.u32Width,
stViFrame.stVFrame.u32Height, stViFrame.stVFrame.u32Width,
stViFrame.stVFrame.u32Height, RK_FORMAT_YCbCr_420_SP);
if (!ret)
last_ba_result_time = rkipc_get_curren_time_ms();
for (int i = 0; i < ba_result.objNum; i++) {
int x, y, w, h;
object = &ba_result.triggerObjects[i];
LOG_DEBUG("topLeft:[%d,%d], bottomRight:[%d,%d],"
"objId is %d, frameId is %d, score is %d, type is %d\n",
object->objInfo.rect.topLeft.x, object->objInfo.rect.topLeft.y,
object->objInfo.rect.bottomRight.x,
object->objInfo.rect.bottomRight.y, object->objInfo.objId,
object->objInfo.frameId, object->objInfo.score, object->objInfo.type);
x = stViFrame.stVFrame.u32Width * object->objInfo.rect.topLeft.x / 10000;
y = stViFrame.stVFrame.u32Height * object->objInfo.rect.topLeft.y / 10000;
w = stViFrame.stVFrame.u32Width *
(object->objInfo.rect.bottomRight.x - object->objInfo.rect.topLeft.x) /
10000;
h = stViFrame.stVFrame.u32Height *
(object->objInfo.rect.bottomRight.y - object->objInfo.rect.topLeft.y) /
10000;
x = x / 2 * 2;
y = y / 2 * 2;
w = w / 2 * 2;
h = h / 2 * 2;
while (x + w + line_pixel >= stViFrame.stVFrame.u32Width) {
w -= 8;
}
while (y + h + line_pixel >= stViFrame.stVFrame.u32Height) {
h -= 8;
}
LOG_DEBUG("i is %d, x,y,w,h is %d,%d,%d,%d\n", i, x, y, w, h);
rga_nv12_border(src, x, y, w, h, line_pixel, 0x000000ff);
// LOG_INFO("draw rect time-consuming is %lld\n",(rkipc_get_curren_time_ms() -
// last_ba_result_time));
// LOG_INFO("triggerRules is %d, ruleID is %d, triggerType is %d\n",
// object->triggerRules,
// object->firstTrigger.ruleID,
// object->firstTrigger.triggerType);
}
releasebuffer_handle(handle);
}
// send venc
ret = RK_MPI_VENC_SendFrame(VIDEO_PIPE_1, &stViFrame, 1000);
if (ret)
LOG_ERROR("RK_MPI_VENC_SendFrame timeout %x\n", ret);
// 7.release the frame
ret = RK_MPI_VI_ReleaseChnFrame(pipe_id_, VIDEO_PIPE_1, &stViFrame);
if (ret != RK_SUCCESS) {
LOG_ERROR("RK_MPI_VI_ReleaseChnFrame fail %x\n", ret);
}
loopCount++;
} else {
LOG_ERROR("RK_MPI_VI_GetChnFrame timeout %x\n", ret);
}
}
return 0;
}
static void *rkipc_get_venc_1(void *arg) {
LOG_DEBUG("#Start %s thread, arg:%p\n", __func__, arg);
prctl(PR_SET_NAME, "RkipcVenc1", 0, 0, 0);
VENC_STREAM_S stFrame;
VI_CHN_STATUS_S stChnStatus;
int loopCount = 0;
int ret = 0;
stFrame.pstPack = malloc(sizeof(VENC_PACK_S));
while (g_video_run_) {
// 5.get the frame
ret = RK_MPI_VENC_GetStream(VIDEO_PIPE_1, &stFrame, 2500);
if (ret == RK_SUCCESS) {
void *data = RK_MPI_MB_Handle2VirAddr(stFrame.pstPack->pMbBlk);
// LOG_INFO("Count:%d, Len:%d, PTS is %" PRId64", enH264EType is %d\n", loopCount,
// stFrame.pstPack->u32Len, stFrame.pstPack->u64PTS,
// stFrame.pstPack->DataType.enH264EType);
rkipc_rtsp_write_video_frame(1, data, stFrame.pstPack->u32Len, stFrame.pstPack->u64PTS);
if ((stFrame.pstPack->DataType.enH264EType == H264E_NALU_IDRSLICE) ||
(stFrame.pstPack->DataType.enH264EType == H264E_NALU_ISLICE) ||
(stFrame.pstPack->DataType.enH265EType == H265E_NALU_IDRSLICE) ||
(stFrame.pstPack->DataType.enH265EType == H265E_NALU_ISLICE)) {
rk_storage_write_video_frame(1, data, stFrame.pstPack->u32Len,
stFrame.pstPack->u64PTS, 1);
if (enable_rtmp)
rk_rtmp_write_video_frame(1, data, stFrame.pstPack->u32Len,
stFrame.pstPack->u64PTS, 1);
} else {
rk_storage_write_video_frame(1, data, stFrame.pstPack->u32Len,
stFrame.pstPack->u64PTS, 0);
if (enable_rtmp)
rk_rtmp_write_video_frame(1, data, stFrame.pstPack->u32Len,
stFrame.pstPack->u64PTS, 0);
}
// 7.release the frame
ret = RK_MPI_VENC_ReleaseStream(VIDEO_PIPE_1, &stFrame);
if (ret != RK_SUCCESS)
LOG_ERROR("RK_MPI_VENC_ReleaseStream fail %x\n", ret);
loopCount++;
} else {
LOG_ERROR("RK_MPI_VENC_GetStream timeout %x\n", ret);
}
}
if (stFrame.pstPack)
free(stFrame.pstPack);
return 0;
}
static void *rkipc_get_jpeg(void *arg) {
LOG_DEBUG("#Start %s thread, arg:%p\n", __func__, arg);
prctl(PR_SET_NAME, "RkipcGetJpeg", 0, 0, 0);
VENC_STREAM_S stFrame;
int loopCount = 0;
int ret = 0;
char file_name[128] = {0};
char record_path[256];
memset(&record_path, 0, sizeof(record_path));
strcat(record_path, rk_param_get_string("storage:mount_path", "/userdata"));
strcat(record_path, "/");
strcat(record_path, rk_param_get_string("storage.0:folder_name", "video0"));
stFrame.pstPack = malloc(sizeof(VENC_PACK_S));
// drop first frame
ret = RK_MPI_VENC_GetStream(JPEG_VENC_CHN, &stFrame, 1000);
if (ret == RK_SUCCESS)
RK_MPI_VENC_ReleaseStream(JPEG_VENC_CHN, &stFrame);
else
LOG_ERROR("RK_MPI_VENC_GetStream timeout %x\n", ret);
while (g_video_run_) {
if (!get_jpeg_cnt) {
usleep(300 * 1000);
continue;
}
// get the frame
ret = RK_MPI_VENC_GetStream(JPEG_VENC_CHN, &stFrame, 1000);
if (ret == RK_SUCCESS) {
void *data = RK_MPI_MB_Handle2VirAddr(stFrame.pstPack->pMbBlk);
LOG_DEBUG("Count:%d, Len:%d, PTS is %" PRId64 ", enH264EType is %d\n", loopCount,
stFrame.pstPack->u32Len, stFrame.pstPack->u64PTS,
stFrame.pstPack->DataType.enH264EType);
// save jpeg file
time_t t = time(NULL);
struct tm tm = *localtime(&t);
snprintf(file_name, 128, "%s/%d%02d%02d%02d%02d%02d.jpeg", record_path,
tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min,
tm.tm_sec);
LOG_DEBUG("file_name is %s, u32Len is %d\n", file_name, stFrame.pstPack->u32Len);
FILE *fp = fopen(file_name, "wb");
if (fp == NULL) {
LOG_ERROR("fp is NULL\n");
} else {
fwrite(data, 1, stFrame.pstPack->u32Len, fp);
}
// release the frame
ret = RK_MPI_VENC_ReleaseStream(JPEG_VENC_CHN, &stFrame);
if (ret != RK_SUCCESS) {
LOG_ERROR("RK_MPI_VENC_ReleaseStream fail %x\n", ret);
}
if (fp) {
fflush(fp);
fclose(fp);
}
loopCount++;
} else {
LOG_ERROR("RK_MPI_VENC_GetStream timeout %x\n", ret);
}
get_jpeg_cnt--;
RK_MPI_VENC_StopRecvFrame(JPEG_VENC_CHN);
}
if (stFrame.pstPack)
free(stFrame.pstPack);
return 0;
}
static void *rkipc_cycle_snapshot(void *arg) {
LOG_INFO("start %s thread, arg:%p\n", __func__, arg);
prctl(PR_SET_NAME, "RkipcCycleSnapshot", 0, 0, 0);
while (g_video_run_ && cycle_snapshot_flag) {
usleep(rk_param_get_int("video.jpeg:snapshot_interval_ms", 1000) * 1000);
rk_take_photo();
}
LOG_INFO("exit %s thread, arg:%p\n", __func__, arg);
return 0;
}
static void *rkipc_get_vi_2_send(void *arg) {
LOG_DEBUG("#Start %s thread, arg:%p\n", __func__, arg);
prctl(PR_SET_NAME, "RkipcGetVi2", 0, 0, 0);
int ret;
int32_t loopCount = 0;
VIDEO_FRAME_INFO_S stViFrame;
int npu_cycle_time_ms = 1000 / rk_param_get_int("video.source:npu_fps", 10);
long long before_time, cost_time;
while (g_video_run_) {
before_time = rkipc_get_curren_time_ms();
ret = RK_MPI_VI_GetChnFrame(pipe_id_, VIDEO_PIPE_2, &stViFrame, 1000);
if (ret == RK_SUCCESS) {
void *data = RK_MPI_MB_Handle2VirAddr(stViFrame.stVFrame.pMbBlk);
uint8_t *phy_addr = (uint8_t *)RK_MPI_MB_Handle2PhysAddr(stViFrame.stVFrame.pMbBlk);
rkipc_rockiva_write_nv12_frame_by_phy_addr(
stViFrame.stVFrame.u32Width, stViFrame.stVFrame.u32Height, loopCount, phy_addr);
ret = RK_MPI_VI_ReleaseChnFrame(pipe_id_, VIDEO_PIPE_2, &stViFrame);
if (ret != RK_SUCCESS)
LOG_ERROR("RK_MPI_VI_ReleaseChnFrame fail %x", ret);
loopCount++;
} else {
LOG_ERROR("RK_MPI_VI_GetChnFrame timeout %x", ret);
}
cost_time = rkipc_get_curren_time_ms() - before_time;
if ((cost_time > 0) && (cost_time < npu_cycle_time_ms))
usleep((npu_cycle_time_ms - cost_time) * 1000);
}
return NULL;
}
static void *rkipc_get_vpss_bgr(void *arg) {
LOG_DEBUG("#Start %s thread, arg:%p\n", __func__, arg);
prctl(PR_SET_NAME, "RkipcGetVpssBgr", 0, 0, 0);
VIDEO_FRAME_INFO_S frame;
VI_CHN_STATUS_S stChnStatus;
int32_t loopCount = 0;
int ret = 0;
while (g_video_run_) {
ret = RK_MPI_VPSS_GetChnFrame(VPSS_BGR, 0, &frame, 1000);
if (ret == RK_SUCCESS) {
void *data = RK_MPI_MB_Handle2VirAddr(frame.stVFrame.pMbBlk);
// LOG_INFO("data:%p, u32Width:%d, u32Height:%d, PTS is %" PRId64 "\n", data,
// frame.stVFrame.u32Width, frame.stVFrame.u32Height, frame.stVFrame.u64PTS);
// rkipc_rockiva_write_rgb888_frame(frame.stVFrame.u32Width, frame.stVFrame.u32Height,
// data);
int32_t fd = RK_MPI_MB_Handle2Fd(frame.stVFrame.pMbBlk);
#if 0
FILE *fp = fopen("/data/test.bgr", "wb");
fwrite(data, 1, frame.stVFrame.u32Width * frame.stVFrame.u32Height * 3, fp);
fflush(fp);
fclose(fp);
exit(1);
#endif
// long long last_nn_time = rkipc_get_curren_time_ms();
rkipc_rockiva_write_rgb888_frame_by_fd(frame.stVFrame.u32Width,
frame.stVFrame.u32Height, loopCount, fd);
// LOG_DEBUG("nn time-consuming is %lld\n",(rkipc_get_curren_time_ms() - last_nn_time));
ret = RK_MPI_VPSS_ReleaseChnFrame(VPSS_BGR, 0, &frame);
if (ret != RK_SUCCESS)
LOG_ERROR("RK_MPI_VPSS_ReleaseChnFrame fail %x\n", ret);
loopCount++;
} else {
LOG_ERROR("RK_MPI_VPSS_GetChnFrame timeout %x\n", ret);
}
}
return 0;
}
static void *rkipc_ivs_get_results(void *arg) {
LOG_DEBUG("#Start %s thread, arg:%p\n", __func__, arg);
prctl(PR_SET_NAME, "RkipcGetIVS", 0, 0, 0);
int ret, i;
IVS_RESULT_INFO_S stResults;
int resultscount = 0;
int count = 0;
int md = rk_param_get_int("ivs:md", 0);
int od = rk_param_get_int("ivs:od", 0);
int width = rk_param_get_int("video.2:width", 960);
int height = rk_param_get_int("video.2:height", 540);
int md_area_threshold = width * height * 0.3;
while (g_video_run_) {
ret = RK_MPI_IVS_GetResults(0, &stResults, 1000);
if (ret >= 0) {
resultscount++;
if (md == 1) {
if (stResults.pstResults->stMdInfo.u32Square > md_area_threshold) {
LOG_INFO("MD: md_area is %d, md_area_threshold is %d\n",
stResults.pstResults->stMdInfo.u32Square, md_area_threshold);
}
}
if (od == 1) {
if (stResults.s32ResultNum > 0) {
if (stResults.pstResults->stOdInfo.u32Flag)
LOG_INFO("OD flag:%d\n", stResults.pstResults->stOdInfo.u32Flag);
}
}
RK_MPI_IVS_ReleaseResults(0, &stResults);
} else {
LOG_ERROR("get chn %d fail %d\n", 0, ret);
usleep(50000llu);
}
}
return NULL;
int color) {
im_rect rect_up = {x, y, width, line_pixel};
im_rect rect_buttom = {x, y + height - line_pixel, width, line_pixel};
im_rect rect_left = {x, y, line_pixel, height};
im_rect rect_right = {x + width - line_pixel, y, line_pixel, height};
IM_STATUS STATUS = imfill(buf, rect_up, color);
STATUS |= imfill(buf, rect_buttom, color);
STATUS |= imfill(buf, rect_left, color);
STATUS |= imfill(buf, rect_right, color);
return STATUS == IM_STATUS_SUCCESS ? 0 : 1;
}
int rkipc_rtmp_init() {
@ -1057,7 +475,7 @@ int rkipc_pipe_0_init() {
memset(&stRecvParam, 0, sizeof(VENC_RECV_PIC_PARAM_S));
stRecvParam.s32RecvPicNum = -1;
RK_MPI_VENC_StartRecvFrame(VIDEO_PIPE_0, &stRecvParam);
pthread_create(&venc_thread_0, NULL, rkipc_get_venc_0, NULL);
pthread_create(&venc_thread_0, NULL, saix_rtsp_stream_from_venc0, NULL);
// bind
vi_chn.enModId = RK_ID_VI;
vi_chn.s32DevId = 0;
@ -1434,9 +852,9 @@ int rkipc_pipe_1_init() {
memset(&stRecvParam, 0, sizeof(VENC_RECV_PIC_PARAM_S));
stRecvParam.s32RecvPicNum = -1;
RK_MPI_VENC_StartRecvFrame(VIDEO_PIPE_1, &stRecvParam);
pthread_create(&venc_thread_1, NULL, rkipc_get_venc_1, NULL);
pthread_create(&venc_thread_1, NULL, saix_venc1_dispatch, NULL);
// pthread_create(&vi_thread_1, NULL, rkipc_get_vi_draw_send_venc, NULL);
// pthread_create(&vi_thread_1, NULL, saix_vi_overlay_to_venc, NULL);
// bind
vi_chn.enModId = RK_ID_VI;
vi_chn.s32DevId = 0;
@ -1578,7 +996,7 @@ int rkipc_pipe_1_init() {
ret = RK_MPI_VO_SetChnAttr(VoLayer, 0, &VoChnAttr);
pthread_t thread_id;
pthread_create(&thread_id, NULL, get_vi_send_vo, NULL);
pthread_create(&thread_id, NULL, saix_send_cam_input_to_display, NULL);
#endif
return 0;
@ -1663,7 +1081,7 @@ int rkipc_pipe_2_init() {
LOG_DEBUG("Bind VI and IVS success\n");
}
if (enable_npu) {
pthread_create(&get_vi_2_send_thread, NULL, rkipc_get_vi_2_send, NULL);
pthread_create(&get_vi_2_send_thread, NULL, saix_push_vi2_to_iva, NULL);
rkipc_osd_draw_nn_init();
}
}
@ -1741,11 +1159,11 @@ int rkipc_pipe_jpeg_init() {
RK_MPI_VENC_StartRecvFrame(JPEG_VENC_CHN,
&stRecvParam); // must, for no streams callback running failed
pthread_create(&jpeg_venc_thread_id, NULL, rkipc_get_jpeg, NULL);
pthread_create(&get_vi_send_jpeg_thread_id, NULL, rkipc_get_vi_send_jpeg, NULL);
pthread_create(&jpeg_venc_thread_id, NULL, saix_process_jpeg, NULL);
pthread_create(&get_vi_send_jpeg_thread_id, NULL, saix_vi_snapshot_jpeg, NULL);
if (rk_param_get_int("video.jpeg:enable_cycle_snapshot", 0)) {
cycle_snapshot_flag = 1;
pthread_create(&cycle_snapshot_thread_id, NULL, rkipc_cycle_snapshot, NULL);
pthread_create(&cycle_snapshot_thread_id, NULL, saix_cycle_snapshot, NULL);
}
return ret;
@ -1823,7 +1241,7 @@ int rkipc_ivs_init() {
}
if (md == 1 || od == 1)
pthread_create(&get_ivs_result_thread, NULL, rkipc_ivs_get_results, NULL);
pthread_create(&get_ivs_result_thread, NULL, saix_fetch_ivs_results, NULL);
return 0;
}
@ -3110,7 +2528,7 @@ int rk_video_set_enable_cycle_snapshot(int value) {
rk_param_set_int(entry, value);
if (value && !cycle_snapshot_flag) {
cycle_snapshot_flag = 1;
pthread_create(&cycle_snapshot_thread_id, NULL, rkipc_cycle_snapshot, NULL);
pthread_create(&cycle_snapshot_thread_id, NULL, saix_cycle_snapshot, NULL);
} else if (!value && cycle_snapshot_flag) {
send_jpeg_cnt = 0;
get_jpeg_cnt = 0;

17
project/app/rkipc/rkipc/src/rv1106_ipc/video/video.h
View File

@ -1,6 +1,13 @@
// Copyright 2022 Rockchip Electronics Co., Ltd. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
/*
* Copyright (c) 2025 NeuralSense AI Private Limited
* Trading as swatah.ai. All rights reserved.
*
* This file is part of the swatah.ai software stack and is licensed under
* the terms defined in the accompanying LICENSE file. Unauthorized copying,
* distribution, or modification of this file, via any medium, is strictly prohibited.
*
* For more information, visit: https://swatah.ai
*/
#include "common.h"
#include "isp.h"
#include "osd.h"
@ -12,8 +19,8 @@
#include "storage.h"
#include <rga/im2d.h>
#include <rga/rga.h>
#include <rk_comm_tde.h>
//#include <rga/rga.h>
//#include <rk_comm_tde.h>
#include <rk_debug.h>
#include <rk_mpi_cal.h>
#include <rk_mpi_ivs.h>