diff --git a/src/blitter/32bpp_anim.cpp b/src/blitter/32bpp_anim.cpp
index 6a030fbf01..f78edcbbaa 100644
--- a/src/blitter/32bpp_anim.cpp
+++ b/src/blitter/32bpp_anim.cpp
@@ -323,7 +323,7 @@ void Blitter_32bppAnim::DrawColourMappingRect(void *dst, int width, int height,
 
 void Blitter_32bppAnim::SetPixel(void *video, int x, int y, PixelColour colour)
 {
-	*((Colour *)video + x + y * _screen.pitch) = LookupColourInPalette(colour.p);
+	*((Colour *)video + x + y * _screen.pitch) = colour.HasRGB() ? colour.ToColour() : LookupColourInPalette(colour.p);
 
 	/* Set the colour in the anim-buffer too, if we are rendering to the screen */
 	if (_screen_disable_anim) return;
@@ -333,7 +333,7 @@ void Blitter_32bppAnim::SetPixel(void *video, int x, int y, PixelColour colour)
 
 void Blitter_32bppAnim::DrawLine(void *video, int x, int y, int x2, int y2, int screen_width, int screen_height, PixelColour colour, int width, int dash)
 {
-	const Colour c = LookupColourInPalette(colour.p);
+	const Colour c = colour.HasRGB() ? colour.ToColour() : LookupColourInPalette(colour.p);
 
 	if (_screen_disable_anim)  {
 		this->DrawLineGeneric(x, y, x2, y2, screen_width, screen_height, width, dash, [&](int x, int y) {
@@ -357,7 +357,7 @@ void Blitter_32bppAnim::DrawRect(void *video, int width, int height, PixelColour
 		return;
 	}
 
-	Colour colour32 = LookupColourInPalette(colour.p);
+	Colour colour32 = colour.HasRGB() ? colour.ToColour() : LookupColourInPalette(colour.p);
 	uint16_t *anim_line = this->ScreenToAnimOffset((uint32_t *)video) + this->anim_buf;
 
 	do {
diff --git a/src/blitter/32bpp_base.cpp b/src/blitter/32bpp_base.cpp
index fdf780b982..811203da63 100644
--- a/src/blitter/32bpp_base.cpp
+++ b/src/blitter/32bpp_base.cpp
@@ -20,12 +20,13 @@ void *Blitter_32bppBase::MoveTo(void *video, int x, int y)
 
 void Blitter_32bppBase::SetPixel(void *video, int x, int y, PixelColour colour)
 {
-	*((Colour *)video + x + y * _screen.pitch) = LookupColourInPalette(colour.p);
+	const Colour c = colour.HasRGB() ? colour.ToColour() : LookupColourInPalette(colour.p);
+	*((Colour *)video + x + y * _screen.pitch) = c;
 }
 
 void Blitter_32bppBase::DrawLine(void *video, int x, int y, int x2, int y2, int screen_width, int screen_height, PixelColour colour, int width, int dash)
 {
-	const Colour c = LookupColourInPalette(colour.p);
+	const Colour c = colour.HasRGB() ? colour.ToColour() : LookupColourInPalette(colour.p);
 	this->DrawLineGeneric(x, y, x2, y2, screen_width, screen_height, width, dash, [=](int x, int y) {
 		*((Colour *)video + x + y * _screen.pitch) = c;
 	});
@@ -33,7 +34,7 @@ void Blitter_32bppBase::DrawLine(void *video, int x, int y, int x2, int y2, int
 
 void Blitter_32bppBase::DrawRect(void *video, int width, int height, PixelColour colour)
 {
-	Colour colour32 = LookupColourInPalette(colour.p);
+	Colour colour32 = colour.HasRGB() ? colour.ToColour() : LookupColourInPalette(colour.p);
 
 	do {
 		Colour *dst = (Colour *)video;
diff --git a/src/blitter/40bpp_anim.cpp b/src/blitter/40bpp_anim.cpp
index 52c2ec7678..59a3294173 100644
--- a/src/blitter/40bpp_anim.cpp
+++ b/src/blitter/40bpp_anim.cpp
@@ -32,10 +32,14 @@ void Blitter_40bppAnim::SetPixel(void *video, int x, int y, PixelColour colour)
 	if (_screen_disable_anim) {
 		Blitter_32bppOptimized::SetPixel(video, x, y, colour);
 	} else {
-		size_t y_offset = static_cast<size_t>(y) * _screen.pitch;
-		*((Colour *)video + x + y_offset) = _black_colour;
+		bool has_rgb = colour.HasRGB();
+		const Colour colour32 = has_rgb ? colour.ToColour() : _black_colour;
+		const uint8_t colour8 = has_rgb ? 0 : colour.p;
 
-		VideoDriver::GetInstance()->GetAnimBuffer()[((uint32_t *)video - (uint32_t *)_screen.dst_ptr) + x + y_offset] = colour.p;
+		size_t y_offset = static_cast<size_t>(y) * _screen.pitch;
+		*((Colour *)video + x + y_offset) = colour32;
+
+		VideoDriver::GetInstance()->GetAnimBuffer()[((uint32_t *)video - (uint32_t *)_screen.dst_ptr) + x + y_offset] = colour8;
 	}
 }
 
@@ -47,6 +51,10 @@ void Blitter_40bppAnim::DrawRect(void *video, int width, int height, PixelColour
 		return;
 	}
 
+	bool has_rgb = colour.HasRGB();
+	const Colour colour32 = has_rgb ? colour.ToColour() : _black_colour;
+	const uint8_t colour8 = has_rgb ? 0 : colour.p;
+
 	assert(VideoDriver::GetInstance()->GetAnimBuffer() != nullptr);
 	uint8_t *anim_line = ((uint32_t *)video - (uint32_t *)_screen.dst_ptr) + VideoDriver::GetInstance()->GetAnimBuffer();
 
@@ -55,8 +63,8 @@ void Blitter_40bppAnim::DrawRect(void *video, int width, int height, PixelColour
 		uint8_t *anim = anim_line;
 
 		for (int i = width; i > 0; i--) {
-			*dst = _black_colour;
-			*anim = colour.p;
+			*dst = colour32;
+			*anim = colour8;
 			dst++;
 			anim++;
 		}
@@ -73,12 +81,16 @@ void Blitter_40bppAnim::DrawLine(void *video, int x, int y, int x2, int y2, int
 		return;
 	}
 
+	bool has_rgb = colour.HasRGB();
+	const Colour colour32 = has_rgb ? colour.ToColour() : _black_colour;
+	const uint8_t colour8 = has_rgb ? 0 : colour.p;
+
 	assert(VideoDriver::GetInstance()->GetAnimBuffer() != nullptr);
 	uint8_t *anim = ((uint32_t *)video - (uint32_t *)_screen.dst_ptr) + VideoDriver::GetInstance()->GetAnimBuffer();
 
 	this->DrawLineGeneric(x, y, x2, y2, screen_width, screen_height, width, dash, [=](int x, int y) {
-		*((Colour *)video + x + y * _screen.pitch) = _black_colour;
-		*(anim + x + y * _screen.pitch) = colour.p;
+		*((Colour *)video + x + y * _screen.pitch) = colour32;
+		*(anim + x + y * _screen.pitch) = colour8;
 	});
 }
 
diff --git a/src/gfx_type.h b/src/gfx_type.h
index 4434690bdd..a45d68c5ba 100644
--- a/src/gfx_type.h
+++ b/src/gfx_type.h
@@ -404,11 +404,18 @@ DECLARE_ENUM_AS_BIT_SET(StringAlignment)
 
 /** Colour for pixel/line drawing. */
 struct PixelColour {
-	uint8_t p; ///< Palette index.
+	uint8_t p = 0; ///< Palette index.
+	uint8_t r = 0; ///< Red component.
+	uint8_t g = 0; ///< Green component.
+	uint8_t b = 0; ///< Blue component.
 
-	constexpr PixelColour() : p(0) {}
+	constexpr PixelColour() {}
 	explicit constexpr PixelColour(uint8_t p) : p(p) {}
+	constexpr PixelColour(uint8_t p, Colour colour) : p(p), r(colour.r), g(colour.g), b(colour.b) {}
+	PixelColour(Colour colour);
 
+	constexpr inline bool HasRGB() const { return (this->r | this->g | this->b) != 0; }
+	constexpr inline Colour ToColour() const { return {this->r, this->g, this->b}; }
 	constexpr inline TextColour ToTextColour() const { return static_cast<TextColour>(this->p) | TC_IS_PALETTE_COLOUR; }
 };
 
diff --git a/src/palette.cpp b/src/palette.cpp
index b22ac326ea..44a2ae8329 100644
--- a/src/palette.cpp
+++ b/src/palette.cpp
@@ -402,3 +402,8 @@ void SetColourGradient(Colours colour, ColourShade shade, PixelColour palette_in
 	assert(shade < SHADE_END);
 	ColourGradients::gradient[colour % COLOUR_END][shade % SHADE_END] = palette_index;
 }
+
+PixelColour::PixelColour(Colour colour) : r(colour.r), g(colour.g), b(colour.b)
+{
+	this->p = GetNearestColourIndex(colour);
+}