Improve the performance of the importer

Signed-off-by: Max Magorsch <arzano@gentoo.org>

1 files changed, 77 insertions, 3 deletions

diff --git a/pkg/importer/importer.go b/pkg/importer/importer.go
index cdb278d..76ba8e7 100644
--- a/pkg/importer/importer.go
+++ b/pkg/importer/importer.go
@@ -2,14 +2,88 @@ package importer
 
 import (
 	"archives/pkg/config"
+	"archives/pkg/database"
+	"archives/pkg/models"
 	"fmt"
+	"os"
 	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
 )
 
+var overAllcounter int
+var importedCounter int
+var startTime time.Time
+
+
 func FullImport() {
+
+	fmt.Println("Init import...")
+	filepath.Walk(config.MailDirPath(), initImport)
+
+	overAllcounter = 0
+	importedCounter = 0
+	startTime = time.Now()
+	filepath.Walk(config.MailDirPath(), func(path string, info os.FileInfo, err error) error {
+		if overAllcounter % 1000 == 0 {
+			fmt.Println(strconv.Itoa(overAllcounter) + ": " + time.Now().Sub(startTime).String())
+		}
+		overAllcounter++
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() && getDepth(path, config.MailDirPath()) >= 1 && isPublicList(path) {
+			importedCounter++
+			importMail(path, info.Name())
+		}
+		return nil
+	})
+
+	fmt.Println("Finished full import. Imported " + strconv.Itoa(importedCounter) + " messages.")
+}
+
+func IncrementalImport() {
+	var messages []*models.Message
+	err := database.DBCon.Model(&messages).
+		Column("filename").
+		Select()
+
+	if err != nil {
+		fmt.Println("Problem during import, aborting:")
+		fmt.Println(err)
+		return
+	}
+
 	fmt.Println("Init import...")
 	filepath.Walk(config.MailDirPath(), initImport)
-	fmt.Println("Start import...")
-	filepath.Walk(config.MailDirPath(), importMail)
-	fmt.Println("Finished import.")
+
+	overAllcounter = 0
+	importedCounter = 0
+	startTime = time.Now()
+	filepath.Walk(config.MailDirPath(), func(path string, info os.FileInfo, err error) error {
+		if overAllcounter % 1000 == 0 {
+			fmt.Println(strconv.Itoa(overAllcounter) + ": " + time.Now().Sub(startTime).String())
+		}
+		overAllcounter++
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() && getDepth(path, config.MailDirPath()) >= 1 && isPublicList(path) && !fileIsAlreadyPresent(path, messages) {
+			importedCounter++
+			importMail(path, info.Name())
+		}
+		return nil
+	})
+
+	fmt.Println("Finished incremental import. Imported " + strconv.Itoa(importedCounter) + " new messages.")
 }
+
+func fileIsAlreadyPresent(path string, messages []*models.Message) bool {
+	for _, message := range messages {
+		if strings.Contains(strings.TrimRight(path, ",S"), strings.TrimRight(message.Filename, ",S")){
+			return true
+		}
+	}
+	return false
+}
\ No newline at end of file