1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
|
package importer
import (
"archives/pkg/config"
"archives/pkg/database"
"archives/pkg/models"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"time"
)
var overAllcounter int
var importedCounter int
var startTime time.Time
func FullImport() {
fmt.Println("Init import...")
filepath.Walk(config.MailDirPath(), initImport)
overAllcounter = 0
importedCounter = 0
startTime = time.Now()
filepath.Walk(config.MailDirPath(), func(path string, info os.FileInfo, err error) error {
if overAllcounter % 1000 == 0 {
fmt.Println(strconv.Itoa(overAllcounter) + ": " + time.Now().Sub(startTime).String())
}
overAllcounter++
if err != nil {
return err
}
if !info.IsDir() && getDepth(path, config.MailDirPath()) >= 1 && isPublicList(path) {
importedCounter++
importMail(path, info.Name())
}
return nil
})
fmt.Println("Finished full import. Imported " + strconv.Itoa(importedCounter) + " messages.")
}
func IncrementalImport() {
var messages []*models.Message
err := database.DBCon.Model(&messages).
Column("filename").
Select()
if err != nil {
fmt.Println("Problem during import, aborting:")
fmt.Println(err)
return
}
fmt.Println("Init import...")
filepath.Walk(config.MailDirPath(), initImport)
overAllcounter = 0
importedCounter = 0
startTime = time.Now()
filepath.Walk(config.MailDirPath(), func(path string, info os.FileInfo, err error) error {
if overAllcounter % 1000 == 0 {
fmt.Println(strconv.Itoa(overAllcounter) + ": " + time.Now().Sub(startTime).String())
}
overAllcounter++
if err != nil {
return err
}
if !info.IsDir() && getDepth(path, config.MailDirPath()) >= 1 && isPublicList(path) && !fileIsAlreadyPresent(path, messages) {
importedCounter++
importMail(path, info.Name())
}
return nil
})
fmt.Println("Finished incremental import. Imported " + strconv.Itoa(importedCounter) + " new messages.")
}
func RecomputeThreads() {
fmt.Println("Init thread computation...")
filepath.Walk(config.MailDirPath(), initImport)
fmt.Println("Import thread references into database...")
overallLength := len(mails)
counter := 0
for _, mail := range mails {
if counter % 1000 == 0 {
fmt.Println("Imported thread references for " + strconv.Itoa(counter) + " of " + strconv.Itoa(overallLength) + " mails.")
}
counter++
insertReferencesToMail(mail.RawReferences, mail.Id, mail.From)
}
fmt.Println("Finished thread computation.")
}
// utility methods
func fileIsAlreadyPresent(path string, messages []*models.Message) bool {
for _, message := range messages {
if strings.Contains(strings.TrimRight(path, ",S"), strings.TrimRight(message.Filename, ",S")){
return true
}
}
return false
}
|
GitWeb