aboutsummaryrefslogtreecommitdiff
blob: 8e4f10c9fc1d220942ff57ac0812312cde48a256 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
package importer

import (
	"archives/pkg/config"
	"archives/pkg/database"
	"archives/pkg/models"
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

var overAllcounter int
var importedCounter int
var startTime time.Time


func FullImport() {

	fmt.Println("Init import...")
	filepath.Walk(config.MailDirPath(), initImport)

	overAllcounter = 0
	importedCounter = 0
	startTime = time.Now()
	filepath.Walk(config.MailDirPath(), func(path string, info os.FileInfo, err error) error {
		if overAllcounter % 1000 == 0 {
			fmt.Println(strconv.Itoa(overAllcounter) + ": " + time.Now().Sub(startTime).String())
		}
		overAllcounter++
		if err != nil {
			return err
		}
		if !info.IsDir() && getDepth(path, config.MailDirPath()) >= 1 && isPublicList(path) {
			importedCounter++
			importMail(path, info.Name())
		}
		return nil
	})

	fmt.Println("Finished full import. Imported " + strconv.Itoa(importedCounter) + " messages.")
}

func IncrementalImport() {
	var messages []*models.Message
	err := database.DBCon.Model(&messages).
		Column("filename").
		Select()

	if err != nil {
		fmt.Println("Problem during import, aborting:")
		fmt.Println(err)
		return
	}

	fmt.Println("Init import...")
	filepath.Walk(config.MailDirPath(), initImport)

	overAllcounter = 0
	importedCounter = 0
	startTime = time.Now()
	filepath.Walk(config.MailDirPath(), func(path string, info os.FileInfo, err error) error {
		if overAllcounter % 1000 == 0 {
			fmt.Println(strconv.Itoa(overAllcounter) + ": " + time.Now().Sub(startTime).String())
		}
		overAllcounter++
		if err != nil {
			return err
		}
		if !info.IsDir() && getDepth(path, config.MailDirPath()) >= 1 && isPublicList(path) && !fileIsAlreadyPresent(path, messages) {
			importedCounter++
			importMail(path, info.Name())
		}
		return nil
	})

	fmt.Println("Finished incremental import. Imported " + strconv.Itoa(importedCounter) + " new messages.")
}

func RecomputeThreads() {

	fmt.Println("Init thread computation...")
	filepath.Walk(config.MailDirPath(), initImport)

	fmt.Println("Import thread references into database...")
	overallLength := len(mails)
	counter := 0
	for _, mail := range mails {
		if counter % 1000 == 0 {
			fmt.Println("Imported thread references for " + strconv.Itoa(counter) +  " of " + strconv.Itoa(overallLength) + " mails.")
		}
		counter++
		insertReferencesToMail(mail.RawReferences, mail.Id, mail.From)
	}

	fmt.Println("Finished thread computation.")
}

// utility methods

func fileIsAlreadyPresent(path string, messages []*models.Message) bool {
	for _, message := range messages {
		if strings.Contains(strings.TrimRight(path, ",S"), strings.TrimRight(message.Filename, ",S")){
			return true
		}
	}
	return false
}