diff --git a/src/fontcache/freetypefontcache.cpp b/src/fontcache/freetypefontcache.cpp
index 51187047c7..57a07ffb13 100644
--- a/src/fontcache/freetypefontcache.cpp
+++ b/src/fontcache/freetypefontcache.cpp
@@ -273,14 +273,14 @@ const Sprite *FreeTypeFontCache::InternalGetGlyph(GlyphID key, bool aa)
 		}
 	}
 
+	UniquePtrSpriteAllocator allocator;
+	BlitterFactory::GetCurrentBlitter()->Encode(spritecollection, allocator);
+
 	GlyphEntry new_glyph;
-	SimpleSpriteAllocator allocator;
-	new_glyph.sprite = BlitterFactory::GetCurrentBlitter()->Encode(spritecollection, allocator);
-	new_glyph.width  = slot->advance.x >> 6;
+	new_glyph.data = std::move(allocator.data);
+	new_glyph.width = slot->advance.x >> 6;
 
-	this->SetGlyphPtr(key, &new_glyph);
-
-	return new_glyph.sprite;
+	return this->SetGlyphPtr(key, std::move(new_glyph)).GetSprite();
 }
 
 
diff --git a/src/fontcache/truetypefontcache.cpp b/src/fontcache/truetypefontcache.cpp
index df07f802e0..890464e039 100644
--- a/src/fontcache/truetypefontcache.cpp
+++ b/src/fontcache/truetypefontcache.cpp
@@ -21,7 +21,7 @@
  * @param fs     The font size that is going to be cached.
  * @param pixels The number of pixels this font should be high.
  */
-TrueTypeFontCache::TrueTypeFontCache(FontSize fs, int pixels) : FontCache(fs), req_size(pixels), glyph_to_sprite(nullptr)
+TrueTypeFontCache::TrueTypeFontCache(FontSize fs, int pixels) : FontCache(fs), req_size(pixels)
 {
 }
 
@@ -39,47 +39,22 @@ TrueTypeFontCache::~TrueTypeFontCache()
  */
 void TrueTypeFontCache::ClearFontCache()
 {
-	if (this->glyph_to_sprite == nullptr) return;
-
-	for (int i = 0; i < 256; i++) {
-		if (this->glyph_to_sprite[i] == nullptr) continue;
-
-		for (int j = 0; j < 256; j++) {
-			free(this->glyph_to_sprite[i][j].sprite);
-		}
-
-		free(this->glyph_to_sprite[i]);
-	}
-
-	free(this->glyph_to_sprite);
-	this->glyph_to_sprite = nullptr;
-
+	this->glyph_to_sprite_map.clear();
 	Layouter::ResetFontCache(this->fs);
 }
 
 
 TrueTypeFontCache::GlyphEntry *TrueTypeFontCache::GetGlyphPtr(GlyphID key)
 {
-	if (this->glyph_to_sprite == nullptr) return nullptr;
-	if (this->glyph_to_sprite[GB(key, 8, 8)] == nullptr) return nullptr;
-	return &this->glyph_to_sprite[GB(key, 8, 8)][GB(key, 0, 8)];
+	auto found = this->glyph_to_sprite_map.find(key);
+	if (found == std::end(this->glyph_to_sprite_map)) return nullptr;
+	return &found->second;
 }
 
-void TrueTypeFontCache::SetGlyphPtr(GlyphID key, const GlyphEntry *glyph)
+TrueTypeFontCache::GlyphEntry &TrueTypeFontCache::SetGlyphPtr(GlyphID key, GlyphEntry &&glyph)
 {
-	if (this->glyph_to_sprite == nullptr) {
-		Debug(fontcache, 3, "Allocating root glyph cache for size {}", this->fs);
-		this->glyph_to_sprite = CallocT<GlyphEntry*>(256);
-	}
-
-	if (this->glyph_to_sprite[GB(key, 8, 8)] == nullptr) {
-		Debug(fontcache, 3, "Allocating glyph cache for range 0x{:02X}00, size {}", GB(key, 8, 8), this->fs);
-		this->glyph_to_sprite[GB(key, 8, 8)] = CallocT<GlyphEntry>(256);
-	}
-
-	Debug(fontcache, 4, "Set glyph for unicode character 0x{:04X}, size {}", key, this->fs);
-	this->glyph_to_sprite[GB(key, 8, 8)][GB(key, 0, 8)].sprite = glyph->sprite;
-	this->glyph_to_sprite[GB(key, 8, 8)][GB(key, 0, 8)].width = glyph->width;
+	this->glyph_to_sprite_map[key] = std::move(glyph);
+	return this->glyph_to_sprite_map[key];
 }
 
 bool TrueTypeFontCache::GetDrawGlyphShadow()
@@ -92,7 +67,7 @@ uint TrueTypeFontCache::GetGlyphWidth(GlyphID key)
 	if ((key & SPRITE_GLYPH) != 0) return this->parent->GetGlyphWidth(key);
 
 	GlyphEntry *glyph = this->GetGlyphPtr(key);
-	if (glyph == nullptr || glyph->sprite == nullptr) {
+	if (glyph == nullptr || glyph->data == nullptr) {
 		this->GetGlyph(key);
 		glyph = this->GetGlyphPtr(key);
 	}
@@ -106,7 +81,7 @@ const Sprite *TrueTypeFontCache::GetGlyph(GlyphID key)
 
 	/* Check for the glyph in our cache */
 	GlyphEntry *glyph = this->GetGlyphPtr(key);
-	if (glyph != nullptr && glyph->sprite != nullptr) return glyph->sprite;
+	if (glyph != nullptr && glyph->data != nullptr) return glyph->GetSprite();
 
 	return this->InternalGetGlyph(key, GetFontAAState());
 }
diff --git a/src/fontcache/truetypefontcache.h b/src/fontcache/truetypefontcache.h
index 6661b72ee3..3eb275df6e 100644
--- a/src/fontcache/truetypefontcache.h
+++ b/src/fontcache/truetypefontcache.h
@@ -29,27 +29,16 @@ protected:
 
 	/** Container for information about a glyph. */
 	struct GlyphEntry {
-		Sprite *sprite; ///< The loaded sprite.
-		uint8_t width; ///< The width of the glyph.
+		std::unique_ptr<uint8_t[]> data; ///< The loaded sprite.
+		uint8_t width = 0; ///< The width of the glyph.
+
+		Sprite *GetSprite() { return reinterpret_cast<Sprite *>(data.get()); }
 	};
 
-	/**
-	 * The glyph cache. This is structured to reduce memory consumption.
-	 * 1) There is a 'segment' table for each font size.
-	 * 2) Each segment table is a discrete block of characters.
-	 * 3) Each block contains 256 (aligned) characters sequential characters.
-	 *
-	 * The cache is accessed in the following way:
-	 * For character 0x0041  ('A'): glyph_to_sprite[0x00][0x41]
-	 * For character 0x20AC (Euro): glyph_to_sprite[0x20][0xAC]
-	 *
-	 * Currently only 256 segments are allocated, "limiting" us to 65536 characters.
-	 * This can be simply changed in the two functions Get & SetGlyphPtr.
-	 */
-	GlyphEntry **glyph_to_sprite;
+	std::unordered_map<GlyphID, GlyphEntry> glyph_to_sprite_map{};
 
 	GlyphEntry *GetGlyphPtr(GlyphID key);
-	void SetGlyphPtr(GlyphID key, const GlyphEntry *glyph);
+	GlyphEntry &SetGlyphPtr(GlyphID key, GlyphEntry &&glyph);
 
 	virtual const Sprite *InternalGetGlyph(GlyphID key, bool aa) = 0;
 
diff --git a/src/os/macosx/font_osx.cpp b/src/os/macosx/font_osx.cpp
index ab54485929..db4d4c1f83 100644
--- a/src/os/macosx/font_osx.cpp
+++ b/src/os/macosx/font_osx.cpp
@@ -276,13 +276,14 @@ const Sprite *CoreTextFontCache::InternalGetGlyph(GlyphID key, bool use_aa)
 		}
 	}
 
-	GlyphEntry new_glyph;
-	SimpleSpriteAllocator allocator;
-	new_glyph.sprite = BlitterFactory::GetCurrentBlitter()->Encode(spritecollection, allocator);
-	new_glyph.width = (uint8_t)std::round(CTFontGetAdvancesForGlyphs(this->font.get(), kCTFontOrientationDefault, &glyph, nullptr, 1));
-	this->SetGlyphPtr(key, &new_glyph);
+	UniquePtrSpriteAllocator allocator;
+	BlitterFactory::GetCurrentBlitter()->Encode(spritecollection, allocator);
 
-	return new_glyph.sprite;
+	GlyphEntry new_glyph;
+	new_glyph.data = std::move(allocator.data);
+	new_glyph.width = (uint8_t)std::round(CTFontGetAdvancesForGlyphs(this->font.get(), kCTFontOrientationDefault, &glyph, nullptr, 1));
+
+	return this->SetGlyphPtr(key, std::move(new_glyph)).GetSprite();
 }
 
 static CTFontDescriptorRef LoadFontFromFile(const std::string &font_name)
diff --git a/src/os/windows/font_win32.cpp b/src/os/windows/font_win32.cpp
index dbab716f1a..b2eac91314 100644
--- a/src/os/windows/font_win32.cpp
+++ b/src/os/windows/font_win32.cpp
@@ -252,14 +252,14 @@ void Win32FontCache::ClearFontCache()
 		}
 	}
 
+	UniquePtrSpriteAllocator allocator;
+	BlitterFactory::GetCurrentBlitter()->Encode(spritecollection, allocator);
+
 	GlyphEntry new_glyph;
-	SimpleSpriteAllocator allocator;
-	new_glyph.sprite = BlitterFactory::GetCurrentBlitter()->Encode(spritecollection, allocator);
+	new_glyph.data = std::move(allocator.data);
 	new_glyph.width = gm.gmCellIncX;
 
-	this->SetGlyphPtr(key, &new_glyph);
-
-	return new_glyph.sprite;
+	return this->SetGlyphPtr(key, std::move(new_glyph)).GetSprite();
 }
 
 /* virtual */ GlyphID Win32FontCache::MapCharToGlyph(char32_t key, bool allow_fallback)
diff --git a/src/spritecache.cpp b/src/spritecache.cpp
index 1c688c2c38..a1f99de871 100644
--- a/src/spritecache.cpp
+++ b/src/spritecache.cpp
@@ -894,6 +894,12 @@ void *SimpleSpriteAllocator::AllocatePtr(size_t size)
 	return MallocT<uint8_t>(size);
 }
 
+void *UniquePtrSpriteAllocator::AllocatePtr(size_t size)
+{
+	this->data = std::make_unique<uint8_t[]>(size);
+	return this->data.get();
+}
+
 /**
  * Handles the case when a sprite of different type is requested than is present in the SpriteCache.
  * For SpriteType::Font sprites, it is normal. In other cases, default sprite is loaded instead.
diff --git a/src/spritecache.h b/src/spritecache.h
index c2be62bdec..7aa01765d4 100644
--- a/src/spritecache.h
+++ b/src/spritecache.h
@@ -37,6 +37,14 @@ protected:
 	void *AllocatePtr(size_t size) override;
 };
 
+/** SpriteAllocator that allocates memory via a unique_ptr array. */
+class UniquePtrSpriteAllocator : public SpriteAllocator {
+public:
+	std::unique_ptr<uint8_t[]> data;
+protected:
+	void *AllocatePtr(size_t size) override;
+};
+
 void *GetRawSprite(SpriteID sprite, SpriteType type, SpriteAllocator *allocator = nullptr, SpriteEncoder *encoder = nullptr);
 bool SpriteExists(SpriteID sprite);