Camera Integration: The Zero-Copy Pipeline

// Request 4 buffers from the kernel
v4l2_requestbuffers req = {
    .count = 4,
    .type = V4L2_BUF_TYPE_VIDEO_CAPTURE,
    .memory = V4L2_MEMORY_MMAP
};
if (ioctl(fd, VIDIOC_REQBUFS, &req) < 0) {
    perror("Requesting buffers failed");
}

# List all video devices and their capabilities
v4l2-ctl --list-devices
# List supported formats for /dev/video0
v4l2-ctl -d /dev/video0 --list-formats-ext

// 1. Open the camera device
// We open in Read/Write mode because ioctls require both.
int fd = open("/dev/video0", O_RDWR);

// 2. Set Format (YUYV 1080p)
// YUYV is the native format for most MIPI CSI cameras.
v4l2_format fmt = {
    .type = V4L2_BUF_TYPE_VIDEO_CAPTURE,
    .fmt = { .pix = {
        .width = 1920,
        .height = 1080,
        .pixelformat = V4L2_PIX_FMT_YUYV,
        .field = V4L2_FIELD_NONE
    }}
};
if (ioctl(fd, VIDIOC_S_FMT, &fmt) < 0) {
    throw std::runtime_error("Failed to set V4L2 format");
}

v4l2_exportbuffer expbuf = {
    .type = V4L2_BUF_TYPE_VIDEO_CAPTURE,
    .index = i // The index of the buffer (usually 0 to 3)
};
if (ioctl(fd, VIDIOC_EXPBUF, &expbuf) < 0) {
    perror("Exporting dmabuf failed");
}
// This integer is a standard File Descriptor.
// You can send it to other processes or drivers!
int sharedFd = expbuf.fd;

// 1. Define External Memory Properties
// We tell Vulkan that this memory comes from a DMA-BUF handle.
vk::ImportMemoryFdInfoKHR importInfo{
    .handleType = vk::ExternalMemoryHandleTypeFlagBits::eDmaBufEXT,
    .fd = sharedFd
};

vk::MemoryAllocateInfo allocInfo{
    .pNext = &importInfo,
    .allocationSize = bufferSize, // This must match the V4L2 buffer length
    .memoryTypeIndex = findDmabufCompatibleMemoryType()
};

// 2. Allocate 'External' Memory
// Note: Vulkan now 'shares' ownership of the memory with the kernel.
// It does not perform a copy; it simply maps the physical address.
vk::raii::DeviceMemory vulkanMemory = device.allocateMemory(allocInfo);

// 3. Bind to a Linear Image
// Camera sensors write data in 'Linear' order (pixel by pixel).
// Most GPUs prefer 'Tiled' order, but we MUST use linear here.
vk::ImageCreateInfo imageInfo{
    .imageType = vk::ImageType::e2D,
    .format = vk::Format::eG8B8G8R8G8B8G8R8_422_Unorm, // YUYV
    .tiling = vk::ImageTiling::eLinear,
    .usage = vk::ImageUsageFlagBits::eSampled,
    .extent = {1920, 1080, 1}
};
vk::raii::Image cameraImage = device.createImage(imageInfo);
cameraImage.bindMemory(*vulkanMemory, 0);

// Importing a multi-planar NV12 image
vk::ExternalFormatANDROID externalFormat{ .externalFormat = 0 }; // Vendor specific

vk::ImageCreateInfo imageInfo{
    .pNext = &externalFormat,
    .imageType = vk::ImageType::e2D,
    .format = vk::Format::eG8B8R8_2Plane420Unorm, // NV12
    .extent = {1920, 1080, 1},
    .usage = vk::ImageUsageFlagBits::eSampled
};

// 1. BEFORE GPU READS: Tell kernel to flush sensor caches
// This is mandatory on ARM Mali and Adreno GPUs.
struct dma_buf_sync sync_start = { .flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_READ };
ioctl(sharedFd, DMA_BUF_IOCTL_SYNC, &sync_start);

// 2. EXECUTE VULKAN COMPUTE JOB
// We use a timeline semaphore to signal when Vulkan is finished.
submitVulkanInference(cameraImage);

// 3. AFTER GPU FINISH: Tell kernel we are done
struct dma_buf_sync sync_end = { .flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_READ };
ioctl(sharedFd, DMA_BUF_IOCTL_SYNC, &sync_end);