diff --git a/pkg/model/loader.go b/pkg/model/loader.go
index 27ea78b34..00b452bd7 100644
--- a/pkg/model/loader.go
+++ b/pkg/model/loader.go
@@ -120,6 +120,8 @@ var knownModelsNameSuffixToSkip []string = []string{
 	".",
 	".safetensors",
 	".bin",
+	".gguf",
+	".ggml",
 	".partial",
 	".tar.gz",
 }
diff --git a/pkg/model/loader_test.go b/pkg/model/loader_test.go
index 2c296c2ea..ec8436203 100644
--- a/pkg/model/loader_test.go
+++ b/pkg/model/loader_test.go
@@ -61,11 +61,13 @@ var _ = Describe("ModelLoader", func() {
 	Context("ListFilesInModelPath", func() {
 		It("should list all valid model files in the model path", func() {
 			os.Create(filepath.Join(modelPath, "test.model"))
+			os.Create(filepath.Join(modelPath, "model.gguf"))
 			os.Create(filepath.Join(modelPath, "README.md"))
 
 			files, err := modelLoader.ListFilesInModelPath()
 			Expect(err).To(BeNil())
 			Expect(files).To(ContainElement("test.model"))
+			Expect(files).ToNot(ContainElement("model.gguf"))
 			Expect(files).ToNot(ContainElement("README.md"))
 		})
 	})