diff --git a/src/town_cmd.cpp b/src/town_cmd.cpp
index ee2f49d930..0ae79367cc 100644
--- a/src/town_cmd.cpp
+++ b/src/town_cmd.cpp
@@ -423,7 +423,7 @@ static void ChangePopulation(Town *t, int mod)
 {
 	t->cache.population += mod;
 	InvalidateWindowData(WC_TOWN_VIEW, t->index); // Cargo requirements may appear/vanish for small populations
-	t->UpdateVirtCoord();
+	if (_settings_client.gui.population_in_label) t->UpdateVirtCoord();
 
 	InvalidateWindowData(WC_TOWN_DIRECTORY, 0, 1);
 }