Merge pull request #4930 from SteveGilvarry/codeql-int-overflow-camera-size

fix: compute camera frame buffer sizes in size_t to avoid int overflow
This commit is contained in:
Isaac Connor
2026-06-17 18:33:10 -04:00
committed by GitHub
3 changed files with 15 additions and 11 deletions

View File

@@ -80,7 +80,7 @@ void LibvlcUnlockBuffer(void* opaque, void* picture, void *const *planes) {
LibvlcPrivateData* data = reinterpret_cast<LibvlcPrivateData*>(opaque);
bool newFrame = false;
for( unsigned int i=0; i < data->bufferSize; i++ ) {
for( size_t i=0; i < data->bufferSize; i++ ) {
if ( data->buffer[i] != data->prevBuffer[i] ) {
newFrame = true;
break;
@@ -277,7 +277,7 @@ int LibvlcCamera::PrimeCapture() {
(*libvlc_video_set_format_f)(mLibvlcMediaPlayer, mTargetChroma.c_str(), width, height, width * mBpp);
(*libvlc_video_set_callbacks_f)(mLibvlcMediaPlayer, &LibvlcLockBuffer, &LibvlcUnlockBuffer, nullptr, &mLibvlcData);
mLibvlcData.bufferSize = width * height * mBpp;
mLibvlcData.bufferSize = static_cast<size_t>(width) * height * mBpp;
// Libvlc wants 32 byte alignment for images (should in theory do this for all image lines)
mLibvlcData.buffer = (uint8_t*)zm_mallocaligned(64, mLibvlcData.bufferSize);
mLibvlcData.prevBuffer = (uint8_t*)zm_mallocaligned(64, mLibvlcData.bufferSize);
@@ -307,7 +307,7 @@ int LibvlcCamera::Capture(std::shared_ptr<ZMPacket> &zm_packet) {
return 0;
mLibvlcData.mutex.lock();
zm_packet->image->Assign(width, height, colours, subpixelorder, mLibvlcData.buffer, width * height * mBpp);
zm_packet->image->Assign(width, height, colours, subpixelorder, mLibvlcData.buffer, mLibvlcData.bufferSize);
zm_packet->packet->stream_index = mVideoStreamId;
zm_packet->stream = mVideoStream;
mLibvlcData.mutex.unlock();

View File

@@ -36,7 +36,7 @@ struct LibvlcPrivateData {
uint8_t* buffer;
uint8_t* prevBuffer;
time_t prevTime;
uint32_t bufferSize;
size_t bufferSize;
std::mutex mutex;
bool newImage;

View File

@@ -73,11 +73,15 @@ static rfbBool resize(rfbClient* client) {
av_free(client->frameBuffer);
}
int bufferSize = 4*client->width*client->height;
size_t bufferSize = static_cast<size_t>(client->width) * client->height * 4;
// libVNC doesn't do alignment or padding in each line
//SWScale::GetBufferSize(AV_PIX_FMT_RGBA, client->width, client->height);
client->frameBuffer = (uint8_t *)av_malloc(bufferSize);
Debug(1, "Allocing new frame buffer %dx%d = %d", client->width, client->height, bufferSize);
if (!client->frameBuffer) {
Error("Failed to allocate %zu byte frame buffer for %dx%d", bufferSize, client->width, client->height);
return FALSE;
}
Debug(1, "Allocing new frame buffer %dx%d = %zu", client->width, client->height, bufferSize);
return TRUE;
}
@@ -225,10 +229,10 @@ int VncCamera::Capture(std::shared_ptr<ZMPacket> &zm_packet) {
zm_packet->stream = mVideoStream;
uint8_t *directbuffer = zm_packet->image->WriteBuffer(width, height, colours, subpixelorder);
Debug(1, "scale src %p, %d, dest %p %d %d %dx%d %dx%d", mVncData.buffer,
mRfb->si.framebufferWidth * mRfb->si.framebufferHeight * 4,
Debug(1, "scale src %p, %zu, dest %p %zu %d %dx%d %dx%d", mVncData.buffer,
static_cast<size_t>(mRfb->si.framebufferWidth) * mRfb->si.framebufferHeight * 4,
directbuffer,
width * height * colours,
static_cast<size_t>(width) * height * colours,
mImgPixFmt,
mRfb->si.framebufferWidth,
mRfb->si.framebufferHeight,
@@ -239,9 +243,9 @@ int VncCamera::Capture(std::shared_ptr<ZMPacket> &zm_packet) {
// Image buffer (WriteBuffer), which is always align-32.
int rc = scale.Convert(
mVncData.buffer,
mRfb->si.framebufferWidth * mRfb->si.framebufferHeight * 4,
static_cast<size_t>(mRfb->si.framebufferWidth) * mRfb->si.framebufferHeight * 4,
directbuffer,
width * height * colours,
static_cast<size_t>(width) * height * colours,
AV_PIX_FMT_RGBA,
mImgPixFmt,
mRfb->si.framebufferWidth,