diff --git a/esphome/components/display/display_buffer.cpp b/esphome/components/display/display_buffer.cpp
index 9fe4137a14..85ebd2567b 100644
--- a/esphome/components/display/display_buffer.cpp
+++ b/esphome/components/display/display_buffer.cpp
@@ -256,7 +256,7 @@ void DisplayBuffer::print(int x, int y, Font *font, Color color, TextAlign align
     if (glyph_n < 0) {
       // Unknown char, skip
       ESP_LOGW(TAG, "Encountered character without representation in font: '%c'", text[i]);
-      if (!font->get_glyphs().empty()) {
+      if (font->get_glyphs_size() > 0) {
         uint8_t glyph_width = font->get_glyphs()[0].glyph_data_->width;
         for (int glyph_x = 0; glyph_x < glyph_width; glyph_x++) {
           for (int glyph_y = 0; glyph_y < height; glyph_y++)
@@ -557,7 +557,7 @@ void Glyph::scan_area(int *x1, int *y1, int *width, int *height) const {
 }
 int Font::match_next_glyph(const char *str, int *match_length) {
   int lo = 0;
-  int hi = this->glyphs_.size() - 1;
+  int hi = this->glyphs_size_ - 1;
   while (lo != hi) {
     int mid = (lo + hi + 1) / 2;
     if (this->glyphs_[mid].compare_to(str)) {
@@ -583,7 +583,7 @@ void Font::measure(const char *str, int *width, int *x_offset, int *baseline, in
     int glyph_n = this->match_next_glyph(str + i, &match_length);
     if (glyph_n < 0) {
       // Unknown char, skip
-      if (!this->get_glyphs().empty())
+      if (this->glyphs_size_ > 0)
         x += this->get_glyphs()[0].glyph_data_->width;
       i++;
       continue;
@@ -603,10 +603,17 @@ void Font::measure(const char *str, int *width, int *x_offset, int *baseline, in
   *x_offset = min_x;
   *width = x - min_x;
 }
-const std::vector<Glyph> &Font::get_glyphs() const { return this->glyphs_; }
 Font::Font(const GlyphData *data, int data_nr, int baseline, int height) : baseline_(baseline), height_(height) {
-  for (int i = 0; i < data_nr; ++i)
-    glyphs_.emplace_back(data + i);
+  ExternalRAMAllocator<Glyph> allocator(ExternalRAMAllocator<Glyph>::ALLOW_FAILURE);
+  this->glyphs_ = allocator.allocate(data_nr);
+  if (this->glyphs_ == nullptr) {
+    ESP_LOGE(TAG, "Could not allocate buffer for Glyphs!");
+    return;
+  }
+  for (int i = 0; i < data_nr; ++i) {
+    this->glyphs_[i] = Glyph(data + i);
+  }
+  this->glyphs_size_ = data_nr;
 }
 
 bool Image::get_pixel(int x, int y) const {
diff --git a/esphome/components/display/display_buffer.h b/esphome/components/display/display_buffer.h
index 3763da8041..815ba8d2e1 100644
--- a/esphome/components/display/display_buffer.h
+++ b/esphome/components/display/display_buffer.h
@@ -526,10 +526,12 @@ class Font {
   inline int get_baseline() { return this->baseline_; }
   inline int get_height() { return this->height_; }
 
-  const std::vector<Glyph> &get_glyphs() const;
+  Glyph *&get_glyphs() { return this->glyphs_; }
+  const u_int16_t &get_glyphs_size() const { return this->glyphs_size_; }
 
  protected:
-  std::vector<Glyph> glyphs_;
+  Glyph *glyphs_{nullptr};
+  u_int16_t glyphs_size_;
   int baseline_;
   int height_;
 };