aboutsummaryrefslogtreecommitdiff
path: root/pkg
diff options
context:
space:
mode:
authorMax Magorsch <arzano@gentoo.org>2020-06-19 15:51:41 +0200
committerMax Magorsch <arzano@gentoo.org>2020-06-19 15:51:41 +0200
commit21181c518cf41828917d36005b726f9452fde657 (patch)
tree38fab1b3c86a41383e48be6b2686d92efd86db62 /pkg
downloadarchives-21181c518cf41828917d36005b726f9452fde657.tar.gz
archives-21181c518cf41828917d36005b726f9452fde657.tar.bz2
archives-21181c518cf41828917d36005b726f9452fde657.zip
Initial version
Signed-off-by: Max Magorsch <arzano@gentoo.org>
Diffstat (limited to 'pkg')
-rw-r--r--pkg/app/home/home.go71
-rw-r--r--pkg/app/home/utils.go64
-rw-r--r--pkg/app/list/browse.go40
-rw-r--r--pkg/app/list/messages.go59
-rw-r--r--pkg/app/list/show.go44
-rw-r--r--pkg/app/list/threads.go60
-rw-r--r--pkg/app/list/utils.go137
-rw-r--r--pkg/app/message/show.go49
-rw-r--r--pkg/app/message/utils.go67
-rw-r--r--pkg/app/popular/threads.go14
-rw-r--r--pkg/app/popular/utils.go57
-rw-r--r--pkg/app/search/search.go98
-rw-r--r--pkg/app/search/utils.go90
-rw-r--r--pkg/app/serve.go60
-rw-r--r--pkg/config/config.go75
-rw-r--r--pkg/database/connection.go78
-rw-r--r--pkg/importer/importer.go24
-rw-r--r--pkg/importer/utils.go125
-rw-r--r--pkg/models/mailinglist.go8
-rw-r--r--pkg/models/message.go155
-rw-r--r--pkg/models/thread.go8
21 files changed, 1383 insertions, 0 deletions
diff --git a/pkg/app/home/home.go b/pkg/app/home/home.go
new file mode 100644
index 0000000..e2d3955
--- /dev/null
+++ b/pkg/app/home/home.go
@@ -0,0 +1,71 @@
+// Used to show the landing page of the application
+
+package home
+
+import (
+ "archives/pkg/app/popular"
+ "archives/pkg/config"
+ "archives/pkg/database"
+ "archives/pkg/models"
+ "github.com/go-pg/pg/v10/orm"
+ "net/http"
+ "time"
+)
+
+// Show renders a template to show the landing page of the application
+func Show(w http.ResponseWriter, r *http.Request) {
+
+ var mailingLists []models.MailingList
+
+ for _, mailingList := range config.IndexMailingLists() {
+ var messages []*models.Message
+ database.DBCon.Model(&messages).
+ WhereGroup(func(q *orm.Query) (*orm.Query, error) {
+ q = q.WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE '[` + mailingList[0] + `]%'`).
+ WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE 'Re: [` + mailingList[0] + `]%'`)
+ return q, nil
+ }).
+ WhereGroup(func(q *orm.Query) (*orm.Query, error) {
+ q = q.WhereOr(`headers::jsonb->>'To' LIKE '%` + mailingList[0] + `@lists.gentoo.org%'`).
+ WhereOr(`headers::jsonb->>'Cc' LIKE '%` + mailingList[0] + `@lists.gentoo.org%'`).
+ WhereOr(`headers::jsonb->>'To' LIKE '%` + mailingList[0] + `@gentoo.org%'`).
+ WhereOr(`headers::jsonb->>'Cc' LIKE '%` + mailingList[0] + `@gentoo.org%'`)
+ return q, nil
+ }).
+ Order("date DESC").
+ Limit(5).
+ Select()
+
+ mailingLists = append(mailingLists, models.MailingList{
+ Name: mailingList[0],
+ Description: mailingList[1],
+ Messages: messages,
+ })
+ }
+
+ //
+ // Get popular threads
+ //
+ popularThreads, err := popular.GetPopularThreads(10, "2020-06-01")
+ if err != nil {
+ http.NotFound(w, r)
+ return
+ }
+ if len(popularThreads) > 5 {
+ popularThreads = popularThreads[:5]
+ }
+
+ templateData := struct {
+ MailingLists []models.MailingList
+ PopularThreads models.Threads
+ MessageCount string
+ CurrentMonth string
+ }{
+ MailingLists: mailingLists,
+ PopularThreads: popularThreads,
+ MessageCount: formatMessageCount(getAllMessagesCount()),
+ CurrentMonth: time.Now().Format("2006-01"),
+ }
+
+ renderIndexTemplate(w, templateData)
+}
diff --git a/pkg/app/home/utils.go b/pkg/app/home/utils.go
new file mode 100644
index 0000000..daab2de
--- /dev/null
+++ b/pkg/app/home/utils.go
@@ -0,0 +1,64 @@
+// miscellaneous utility functions used for the landing page of the application
+
+package home
+
+import (
+ "archives/pkg/database"
+ "archives/pkg/models"
+ "github.com/go-pg/pg/v10"
+ "html/template"
+ "net/http"
+ "strconv"
+)
+
+// renderIndexTemplate renders all templates used for the landing page
+func renderIndexTemplate(w http.ResponseWriter, templateData interface{}) {
+ templates := template.Must(
+ template.Must(
+ template.New("Show").
+ Funcs(template.FuncMap{
+ "makeMessage" : func(headers map[string][]string) models.Message {
+ return models.Message{
+ Headers: headers,
+ }
+ },
+ }).
+ ParseGlob("web/templates/layout/*.tmpl")).
+ ParseGlob("web/templates/home/*.tmpl"))
+
+ templates.ExecuteTemplate(w, "home.tmpl", templateData)
+}
+
+// utility methods
+
+func getAllMessagesCount() int {
+ var messsageCount int
+ database.DBCon.Model((*models.Message)(nil)).QueryOne(pg.Scan(&messsageCount), `
+ SELECT
+ count(DISTINCT messages.headers->>'Message-Id')
+ FROM
+ messages;
+ `)
+ return messsageCount
+}
+
+// formatMessageCount returns the formatted number of
+// messages containing a thousands comma
+func formatMessageCount(messageCount int) string {
+ packages := strconv.Itoa(messageCount)
+ if len(string(messageCount)) == 9 {
+ return packages[:3] + "," + packages[3:6] + "," + packages[6:]
+ } else if len(packages) == 8 {
+ return packages[:2] + "," + packages[2:5] + "," + packages[5:]
+ } else if len(packages) == 7 {
+ return packages[:1] + "," + packages[1:4] + "," + packages[4:]
+ } else if len(packages) == 6 {
+ return packages[:3] + "," + packages[3:]
+ } else if len(packages) == 5 {
+ return packages[:2] + "," + packages[2:]
+ } else if len(packages) == 4 {
+ return packages[:1] + "," + packages[1:]
+ } else {
+ return packages
+ }
+}
diff --git a/pkg/app/list/browse.go b/pkg/app/list/browse.go
new file mode 100644
index 0000000..7d046e6
--- /dev/null
+++ b/pkg/app/list/browse.go
@@ -0,0 +1,40 @@
+package list
+
+import (
+ "archives/pkg/config"
+ "archives/pkg/models"
+ "net/http"
+)
+
+func Browse(w http.ResponseWriter, r *http.Request) {
+
+ // Count number of messages in the current mailing lists
+ var currentMailingLists []models.MailingList
+ for _, listName := range config.CurrentMailingLists() {
+ messageCount, _ := countMessages(listName)
+ currentMailingLists = append(currentMailingLists, models.MailingList{
+ Name: listName,
+ MessageCount: messageCount,
+ })
+ }
+
+ // Count number of messages in the frozen archives
+ var frozenArchives []models.MailingList
+ for _, listName := range config.FrozenArchives() {
+ messageCount, _ := countMessages(listName)
+ frozenArchives = append(frozenArchives, models.MailingList{
+ Name: listName,
+ MessageCount: messageCount,
+ })
+ }
+
+ browseData := struct {
+ CurrentMailingLists []models.MailingList
+ FrozenArchives []models.MailingList
+ }{
+ CurrentMailingLists: currentMailingLists,
+ FrozenArchives: frozenArchives,
+ }
+
+ renderBrowseTemplate(w, browseData)
+}
diff --git a/pkg/app/list/messages.go b/pkg/app/list/messages.go
new file mode 100644
index 0000000..383e891
--- /dev/null
+++ b/pkg/app/list/messages.go
@@ -0,0 +1,59 @@
+package list
+
+import (
+ "archives/pkg/database"
+ "archives/pkg/models"
+ "github.com/go-pg/pg/v10/orm"
+ "math"
+ "net/http"
+ "strconv"
+ "strings"
+)
+
+func Messages(w http.ResponseWriter, r *http.Request) {
+
+ urlParts := strings.Split(r.URL.Path, "/messages/")
+ if len(urlParts) != 2 {
+ http.NotFound(w, r)
+ return
+ }
+
+ listName := strings.ReplaceAll(urlParts[0], "/", "")
+
+ trailingUrlParts := strings.Split(urlParts[1], "/")
+ combinedDate := trailingUrlParts[0]
+ currentPage := 1
+ if len(trailingUrlParts) > 1 {
+ parsedCurrentPage, err := strconv.Atoi(trailingUrlParts[1])
+ if err == nil {
+ currentPage = parsedCurrentPage
+ }
+ }
+ offset := (currentPage - 1) * 50
+
+ var messages []*models.Message
+ query := database.DBCon.Model(&messages).
+ Column("id", "headers", "date").
+ Where("to_char(date, 'YYYY-MM') = ?", combinedDate).
+ WhereGroup(func(q *orm.Query) (*orm.Query, error) {
+ q = q.WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE '[` + listName + `]%'`).
+ WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE 'Re: [` + listName + `]%'`)
+ return q, nil
+ }).
+ WhereGroup(func(q *orm.Query) (*orm.Query, error) {
+ q = q.WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@lists.gentoo.org%'`).
+ WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@lists.gentoo.org%'`).
+ WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@gentoo.org%'`).
+ WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@gentoo.org%'`)
+ return q, nil
+ }).
+ Order("date DESC")
+
+ messagesCount, _ := query.Count()
+ query.Limit(50).Offset(offset).Select()
+
+ maxPages := int(math.Ceil(float64(messagesCount) / float64(50)))
+
+ renderMessagesTemplate(w, listName, combinedDate, currentPage, maxPages, messages)
+
+}
diff --git a/pkg/app/list/show.go b/pkg/app/list/show.go
new file mode 100644
index 0000000..8db8778
--- /dev/null
+++ b/pkg/app/list/show.go
@@ -0,0 +1,44 @@
+package list
+
+import (
+ "archives/pkg/database"
+ "archives/pkg/models"
+ "github.com/go-pg/pg/v10/orm"
+ "net/http"
+ "strings"
+)
+
+func Show(w http.ResponseWriter, r *http.Request) {
+
+ listName := strings.ReplaceAll(r.URL.Path, "/", "")
+
+ var res []struct {
+ CombinedDate string
+ MessageCount int
+ }
+ err := database.DBCon.Model((*models.Message)(nil)).
+ WhereGroup(func(q *orm.Query) (*orm.Query, error) {
+ q = q.WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE '[` + listName + `]%'`).
+ WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE 'Re: [` + listName + `]%'`)
+ return q, nil
+ }).
+ WhereGroup(func(q *orm.Query) (*orm.Query, error) {
+ q = q.WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@lists.gentoo.org%'`).
+ WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@lists.gentoo.org%'`).
+ WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@gentoo.org%'`).
+ WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@gentoo.org%'`)
+ return q, nil
+ }).
+ ColumnExpr("to_char(date, 'YYYY-MM') AS combined_date").
+ ColumnExpr("count(*) AS message_count").
+ Group("combined_date").
+ Order("combined_date DESC").
+ Select(&res)
+
+ if err != nil {
+ http.NotFound(w, r)
+ return
+ }
+
+ renderShowTemplate(w, listName, res)
+}
diff --git a/pkg/app/list/threads.go b/pkg/app/list/threads.go
new file mode 100644
index 0000000..33ade3c
--- /dev/null
+++ b/pkg/app/list/threads.go
@@ -0,0 +1,60 @@
+package list
+
+import (
+ "archives/pkg/database"
+ "archives/pkg/models"
+ "github.com/go-pg/pg/v10/orm"
+ "math"
+ "net/http"
+ "strconv"
+ "strings"
+)
+
+func Threads(w http.ResponseWriter, r *http.Request) {
+
+ urlParts := strings.Split(r.URL.Path, "/threads/")
+ if len(urlParts) != 2 {
+ http.NotFound(w, r)
+ return
+ }
+
+ listName := strings.ReplaceAll(urlParts[0], "/", "")
+ trailingUrlParts := strings.Split(urlParts[1], "/")
+ combinedDate := trailingUrlParts[0]
+ currentPage := 1
+ if len(trailingUrlParts) > 1 {
+ parsedCurrentPage, err := strconv.Atoi(trailingUrlParts[1])
+ if err == nil {
+ currentPage = parsedCurrentPage
+ }
+ }
+ offset := (currentPage - 1) * 50
+
+ var messages []*models.Message
+ query := database.DBCon.Model(&messages).
+ Column("id", "headers", "date").
+ Where("to_char(date, 'YYYY-MM') = ?", combinedDate).
+ Where(`NOT headers::jsonb ? 'References'`).
+ Where(`NOT headers::jsonb ? 'In-Reply-To'`).
+ WhereGroup(func(q *orm.Query) (*orm.Query, error) {
+ q = q.WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE '[` + listName + `]%'`).
+ WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE 'Re: [` + listName + `]%'`)
+ return q, nil
+ }).
+ WhereGroup(func(q *orm.Query) (*orm.Query, error) {
+ q = q.WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@lists.gentoo.org%'`).
+ WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@lists.gentoo.org%'`).
+ WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@gentoo.org%'`).
+ WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@gentoo.org%'`)
+ return q, nil
+ }).
+ Order("date DESC")
+
+ messagesCount, _ := query.Count()
+ query.Limit(50).Offset(offset).Select()
+
+ maxPages := int(math.Ceil(float64(messagesCount) / float64(50)))
+
+ renderThreadsTemplate(w, listName, combinedDate, currentPage, maxPages, messages)
+
+}
diff --git a/pkg/app/list/utils.go b/pkg/app/list/utils.go
new file mode 100644
index 0000000..07dbc65
--- /dev/null
+++ b/pkg/app/list/utils.go
@@ -0,0 +1,137 @@
+// miscellaneous utility functions used for the landing page of the application
+
+package list
+
+import (
+ "archives/pkg/database"
+ "archives/pkg/models"
+ "github.com/go-pg/pg/v10/orm"
+ "html/template"
+ "net/http"
+)
+
+type ListData struct {
+ ListName string
+ Date string
+ CurrentPage int
+ MaxPages int
+ Messages []*models.Message
+}
+
+// renderIndexTemplate renders all templates used for the landing page
+func renderShowTemplate(w http.ResponseWriter, listName string, messageData interface{}) {
+ templates := template.Must(
+ template.Must(
+ template.New("Show").
+ ParseGlob("web/templates/layout/*.tmpl")).
+ ParseGlob("web/templates/list/*.tmpl"))
+
+ templteData := struct {
+ ListName string
+ MessageData interface{}
+ }{
+ ListName: listName,
+ MessageData: messageData,
+ }
+
+ templates.ExecuteTemplate(w, "show.tmpl", templteData)
+}
+
+// renderIndexTemplate renders all templates used for the landing page
+func renderMessagesTemplate(w http.ResponseWriter, listName string, date string, currentPage int, maxPages int, messages []*models.Message) {
+ templates := template.Must(
+ template.Must(
+ template.Must(
+ template.New("Show").
+ Funcs(getFuncMap()).
+ ParseGlob("web/templates/layout/*.tmpl")).
+ ParseGlob("web/templates/list/components/*.tmpl")).
+ ParseGlob("web/templates/list/*.tmpl"))
+
+ templates.ExecuteTemplate(w, "messages.tmpl", buildListData(listName, date, currentPage, maxPages, messages))
+}
+
+// renderIndexTemplate renders all templates used for the landing page
+func renderThreadsTemplate(w http.ResponseWriter, listName string, date string, currentPage int, maxPages int, messages []*models.Message) {
+ templates := template.Must(
+ template.Must(
+ template.Must(
+ template.New("Show").
+ Funcs(getFuncMap()).
+ ParseGlob("web/templates/layout/*.tmpl")).
+ ParseGlob("web/templates/list/components/*.tmpl")).
+ ParseGlob("web/templates/list/*.tmpl"))
+
+ templates.ExecuteTemplate(w, "threads.tmpl", buildListData(listName, date, currentPage, maxPages, messages))
+}
+
+// renderIndexTemplate renders all templates used for the landing page
+func renderBrowseTemplate(w http.ResponseWriter, lists interface{}) {
+ templates := template.Must(
+ template.Must(
+ template.New("Show").
+ ParseGlob("web/templates/layout/*.tmpl")).
+ ParseGlob("web/templates/list/*.tmpl"))
+
+ templates.ExecuteTemplate(w, "browse.tmpl", lists)
+}
+
+// utility methods
+
+func getFuncMap() template.FuncMap {
+ return template.FuncMap{
+ "min": func(a, b int) int {
+ if a < b {
+ return a
+ }
+ return b
+ },
+ "max": func(a, b int) int {
+ if a < b {
+ return b
+ }
+ return a
+ },
+ "add": func(a, b int) int {
+ return a + b
+ },
+ "sub": func(a, b int) int {
+ return a - b
+ },
+ "makeRange": makeRange,
+ }
+}
+
+func buildListData(listName string, date string, currentPage int, maxPages int, messages []*models.Message) ListData {
+ return ListData{
+ ListName: listName,
+ Date: date,
+ CurrentPage: currentPage,
+ MaxPages: maxPages,
+ Messages: messages,
+ }
+}
+
+func makeRange(min, max int) []int {
+ a := make([]int, max-min+1)
+ for i := range a {
+ a[i] = min + i
+ }
+ return a
+}
+
+func countMessages(listName string) (int, error) {
+ return database.DBCon.Model((*models.Message)(nil)).
+ WhereGroup(func(q *orm.Query) (*orm.Query, error) {
+ q = q.WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE '[` + listName + `]%'`).
+ WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE 'Re: [` + listName + `]%'`)
+ return q, nil
+ }).
+ WhereGroup(func(q *orm.Query) (*orm.Query, error) {
+ q = q.WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@lists.gentoo.org%'`).
+ WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@lists.gentoo.org%'`).
+ WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@gentoo.org%'`).
+ WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@gentoo.org%'`)
+ return q, nil
+ }).Count()
+}
diff --git a/pkg/app/message/show.go b/pkg/app/message/show.go
new file mode 100644
index 0000000..a027596
--- /dev/null
+++ b/pkg/app/message/show.go
@@ -0,0 +1,49 @@
+// Used to show the landing page of the application
+
+package message
+
+import (
+ "archives/pkg/database"
+ "archives/pkg/models"
+ "net/http"
+ "strings"
+)
+
+// Show renders a template to show the landing page of the application
+func Show(w http.ResponseWriter, r *http.Request) {
+
+ urlParts := strings.Split(r.URL.Path, "/")
+ listName := urlParts[1]
+ messageHash := urlParts[len(urlParts)-1]
+
+ message := &models.Message{Id: messageHash}
+ err := database.DBCon.Select(message)
+
+ if err != nil {
+ http.NotFound(w, r)
+ return
+ }
+
+ var inReplyTos []*models.Message
+ var inReplyTo *models.Message
+ if message.HasHeaderField("In-Reply-To") {
+ err = database.DBCon.Model(&inReplyTos).
+ Where(`(headers::jsonb->>'Message-Id')::jsonb ? '` + message.GetHeaderField("In-Reply-To") + `'`).
+ Select()
+ if err != nil || len(inReplyTos) < 1 {
+ inReplyTo = nil
+ } else {
+ inReplyTo = inReplyTos[0]
+ }
+ } else {
+ inReplyTo = nil
+ }
+
+ var replies []*models.Message
+ database.DBCon.Model(&replies).
+ Where(`(headers::jsonb->>'References')::jsonb ? '` + message.GetHeaderField("Message-Id") + `'`).
+ WhereOr(`(headers::jsonb->>'In-Reply-To')::jsonb ? '` + message.GetHeaderField("Message-Id") + `'`).
+ Order("date ASC").Select()
+
+ renderMessageTemplate(w, listName, message, inReplyTo, replies)
+}
diff --git a/pkg/app/message/utils.go b/pkg/app/message/utils.go
new file mode 100644
index 0000000..0cb40f5
--- /dev/null
+++ b/pkg/app/message/utils.go
@@ -0,0 +1,67 @@
+// miscellaneous utility functions used for the landing page of the application
+
+package message
+
+import (
+ "archives/pkg/models"
+ "html/template"
+ "net/http"
+ "strings"
+)
+
+// renderIndexTemplate renders all templates used for the landing page
+func renderMessageTemplate(w http.ResponseWriter, listName string, message *models.Message, inReplyTo *models.Message, replies []*models.Message) {
+ templates := template.Must(
+ template.Must(
+ template.New("Show").
+ Funcs(getFuncMap()).
+ ParseGlob("web/templates/layout/*.tmpl")).
+ ParseGlob("web/templates/message/*.tmpl"))
+
+ templateData := struct {
+ ListName string
+ Message *models.Message
+ InReplyTo *models.Message
+ Replies []*models.Message
+ }{
+ ListName: listName,
+ Message: message,
+ InReplyTo: inReplyTo,
+ Replies: replies,
+ }
+
+ templates.ExecuteTemplate(w, "show.tmpl", templateData)
+}
+
+func getFuncMap() template.FuncMap {
+ return template.FuncMap{
+ "formatAddr": func(addr string) string {
+ if strings.Contains(addr, "@lists.gentoo.org") || strings.Contains(addr, "@gentoo.org") {
+ addr = strings.ReplaceAll(addr, "@lists.gentoo.org", "@l.g.o")
+ addr = strings.ReplaceAll(addr, "@gentoo.org", "@g.o")
+ } else {
+ start := false
+ for i := len(addr) - 1; i > 0; i-- {
+ if addr[i] == '@' {
+ break
+ }
+ if start {
+ out := []rune(addr)
+ out[i] = '×'
+ addr = string(out)
+ }
+ if addr[i] == '.' {
+ start = true
+ }
+ }
+ }
+ return addr
+ },
+ }
+}
+
+func replaceAtIndex(in string, r rune, i int) string {
+ out := []rune(in)
+ out[i] = r
+ return string(out)
+}
diff --git a/pkg/app/popular/threads.go b/pkg/app/popular/threads.go
new file mode 100644
index 0000000..5c12c8a
--- /dev/null
+++ b/pkg/app/popular/threads.go
@@ -0,0 +1,14 @@
+package popular
+
+import (
+ "net/http"
+)
+
+func Threads(w http.ResponseWriter, r *http.Request) {
+ threads, err := GetPopularThreads(25, "2020-06-01")
+ if err != nil {
+ http.NotFound(w, r)
+ return
+ }
+ renderPopularThreads(w, threads)
+}
diff --git a/pkg/app/popular/utils.go b/pkg/app/popular/utils.go
new file mode 100644
index 0000000..6cfeff7
--- /dev/null
+++ b/pkg/app/popular/utils.go
@@ -0,0 +1,57 @@
+package popular
+
+import (
+ "archives/pkg/database"
+ "archives/pkg/models"
+ "html/template"
+ "net/http"
+ "strconv"
+)
+
+// renderIndexTemplate renders all templates used for the landing page
+func renderPopularThreads(w http.ResponseWriter, templateData interface{}) {
+ templates := template.Must(
+ template.Must(
+ template.New("Popular").
+ Funcs(template.FuncMap{
+ "makeMessage" : func(headers map[string][]string) models.Message {
+ return models.Message{
+ Headers: headers,
+ }
+ },
+ }).
+ ParseGlob("web/templates/layout/*.tmpl")).
+ ParseGlob("web/templates/popular/*.tmpl"))
+
+ templates.ExecuteTemplate(w, "threads.tmpl", templateData)
+}
+
+// utility methods
+
+func GetPopularThreads(n int, date string) (models.Threads, error) {
+ var popularThreads models.Threads
+ err := database.DBCon.Model(&popularThreads).
+ TableExpr(`(SELECT id, headers, regexp_replace(regexp_replace(regexp_replace(regexp_replace(headers::jsonb->>'Subject','^\["',''),'"\]$',''),'^Re:\s',''), '^\[.*\]', '') AS c FROM messages WHERE date >= '2020-06-12'::date) t`).
+ ColumnExpr(`c as Subject, jsonb_agg(id)->>0 as Id, jsonb_agg(headers)->>0 as Headers, Count(*) as Count`).
+ GroupExpr(`c`).
+ OrderExpr(`count DESC`).
+ Limit(n).
+ Select()
+
+ return popularThreads, err
+}
+
+func GetMessagesFromPopularThreads(threads models.Threads) []*models.Message {
+ var popularThreads []*models.Message
+ for _, thread := range threads {
+ var messages []*models.Message
+ err := database.DBCon.Model(&messages).
+ Where(`headers::jsonb->>'Subject' LIKE '%` + thread.Id + `%'`).
+ Select()
+ if err == nil && len(messages) > 0 {
+ messages[0].Comment = strconv.Itoa(thread.Count)
+ popularThreads = append(popularThreads, messages[0])
+ }
+ }
+ return popularThreads
+}
diff --git a/pkg/app/search/search.go b/pkg/app/search/search.go
new file mode 100644
index 0000000..f6498e9
--- /dev/null
+++ b/pkg/app/search/search.go
@@ -0,0 +1,98 @@
+package search
+
+import (
+ "archives/pkg/config"
+ "archives/pkg/database"
+ "archives/pkg/models"
+ "math"
+ "net/http"
+ "strconv"
+ "strings"
+)
+
+func Search(w http.ResponseWriter, r *http.Request) {
+
+ //
+ // Parse search params
+ //
+ searchTerm := getParameterValue("q", r)
+ showThreads := getParameterValue("threads", r) != ""
+ page, err := strconv.Atoi(getParameterValue("page", r))
+ var currentPage int
+ var offset int
+
+ if err != nil {
+ currentPage = 1
+ offset = 0
+ } else {
+ currentPage = page
+ offset = 50 * (page - 1)
+ }
+
+ //
+ // Step 1: Search for List with the same name and redirect
+ //
+ for _, list := range config.AllPublicMailingLists() {
+ if strings.TrimSpace(searchTerm) == list {
+ http.Redirect(w, r, "/"+list+"/", http.StatusMovedPermanently)
+ return
+ }
+ }
+
+ //
+ // Step 2: Search by Author
+ //
+ var searchResults []*models.Message
+ query := database.DBCon.Model(&searchResults).
+ WhereOr(`headers::jsonb->>'From' LIKE ?`, "%"+searchTerm+"%").
+ Order("date DESC")
+ if showThreads {
+ query = query.Where(`NOT headers::jsonb ? 'References'`).Where(`NOT headers::jsonb ? 'In-Reply-To'`)
+ }
+
+ messagesCount, _ := query.Count()
+ err = query.Limit(50).Offset(offset).Select()
+
+ if err == nil && messagesCount > 0 && strings.TrimSpace(searchTerm) != "gentoo" {
+ maxPages := int(math.Ceil(float64(messagesCount) / float64(50)))
+ renderSearchTemplate(w, showThreads, searchTerm, messagesCount, currentPage, maxPages, searchResults)
+ return
+ }
+
+ //
+ // Step 3: Search by Subject
+ //
+ query = database.DBCon.Model(&searchResults).
+ Where(`tsv_subject @@ to_tsquery(''?'')`, searchTerm)
+ if showThreads {
+ query = query.Where(`NOT headers::jsonb ? 'References'`).Where(`NOT headers::jsonb ? 'In-Reply-To'`)
+ }
+
+ messagesCount, _ = query.Count()
+ err = query.Limit(50).Offset(offset).Select()
+
+ if err == nil && messagesCount > 0 {
+ maxPages := int(math.Ceil(float64(messagesCount) / float64(50)))
+ renderSearchTemplate(w, showThreads, searchTerm, messagesCount, currentPage, maxPages, searchResults)
+ return
+ }
+
+ //
+ // Step 4: Search by Message Body
+ //
+ query = database.DBCon.Model(&searchResults).
+ Where(`tsv_body @@ to_tsquery(''?'')`, searchTerm)
+ if showThreads {
+ query = query.Where(`NOT headers::jsonb ? 'References'`).Where(`NOT headers::jsonb ? 'In-Reply-To'`)
+ }
+
+ messagesCount, _ = query.Count()
+ err = query.Limit(50).Offset(offset).Select()
+
+ if err != nil {
+ http.NotFound(w, r)
+ return
+ }
+ maxPages := int(math.Ceil(float64(messagesCount) / float64(50)))
+ renderSearchTemplate(w, showThreads, searchTerm, messagesCount, currentPage, maxPages, searchResults)
+}
diff --git a/pkg/app/search/utils.go b/pkg/app/search/utils.go
new file mode 100644
index 0000000..0fa4285
--- /dev/null
+++ b/pkg/app/search/utils.go
@@ -0,0 +1,90 @@
+package search
+
+import (
+ "archives/pkg/models"
+ "html/template"
+ "net/http"
+)
+
+type SearchData struct {
+ SearchQuery string
+ ShowThreads bool
+ SearchResultsCount int
+ CurrentPage int
+ MaxPages int
+ Messages []*models.Message
+}
+
+// renderIndexTemplate renders all templates used for the landing page
+func renderSearchTemplate(w http.ResponseWriter, showThreads bool, searchQuery string, messagesCount int, currentPage int, maxPages int, messages []*models.Message) {
+ templates := template.Must(
+ template.Must(
+ template.Must(
+ template.New("Show").
+ Funcs(getFuncMap()).
+ ParseGlob("web/templates/layout/*.tmpl")).
+ ParseGlob("web/templates/search/components/pagination.tmpl")).
+ ParseGlob("web/templates/search/*.tmpl"))
+
+ templates.ExecuteTemplate(w, "searchresults.tmpl", buildSearchData(showThreads, searchQuery, messagesCount, currentPage, maxPages, messages))
+}
+
+// utility methods
+
+func getFuncMap() template.FuncMap {
+ return template.FuncMap{
+ "min": func(a, b int) int {
+ if a < b {
+ return a
+ }
+ return b
+ },
+ "max": func(a, b int) int {
+ if a < b {
+ return b
+ }
+ return a
+ },
+ "add": func(a, b int) int {
+ return a + b
+ },
+ "sub": func(a, b int) int {
+ return a - b
+ },
+ "mul": func(a, b int) int {
+ return a * b
+ },
+ "makeRange": makeRange,
+ }
+}
+
+func buildSearchData(showThreads bool, searchQuery string, messagesCount int, currentPage int, maxPages int, messages []*models.Message) SearchData {
+ return SearchData{
+ SearchQuery: searchQuery,
+ ShowThreads: showThreads,
+ SearchResultsCount: messagesCount,
+ CurrentPage: currentPage,
+ MaxPages: maxPages,
+ Messages: messages,
+ }
+}
+
+func makeRange(min, max int) []int {
+ a := make([]int, max-min+1)
+ for i := range a {
+ a[i] = min + i
+ }
+ return a
+}
+
+// getParameterValue returns the value of a given parameter
+func getParameterValue(parameterName string, r *http.Request) string {
+ results, ok := r.URL.Query()[parameterName]
+ if !ok {
+ return ""
+ }
+ if len(results) == 0 {
+ return ""
+ }
+ return results[0]
+}
diff --git a/pkg/app/serve.go b/pkg/app/serve.go
new file mode 100644
index 0000000..62eac33
--- /dev/null
+++ b/pkg/app/serve.go
@@ -0,0 +1,60 @@
+// Entrypoint for the web application
+
+package app
+
+import (
+ "archives/pkg/app/home"
+ "archives/pkg/app/list"
+ "archives/pkg/app/message"
+ "archives/pkg/app/popular"
+ "archives/pkg/app/search"
+ "archives/pkg/config"
+ "fmt"
+ "log"
+ "net/http"
+)
+
+// Serve is used to serve the web application
+func Serve() {
+
+ fmt.Println("Serving on Port " + config.Port())
+
+ for _, mailingList := range config.AllPublicMailingLists() {
+ setRoute("/"+mailingList+"/message/", message.Show)
+ setRoute("/"+mailingList+"/messages/", list.Messages)
+ setRoute("/"+mailingList+"/threads/", list.Threads)
+ setRoute("/"+mailingList+"/", list.Show)
+ }
+
+ setRoute("/lists", list.Browse)
+
+ setRoute("/popular", popular.Threads)
+
+ setRoute("/search", search.Search)
+
+ setRoute("/", home.Show)
+
+ fs := http.StripPrefix("/assets/", http.FileServer(http.Dir("assets")))
+ http.Handle("/assets/", fs)
+
+ log.Fatal(http.ListenAndServe(":"+config.Port(), nil))
+
+}
+
+// define a route using the default middleware and the given handler
+func setRoute(path string, handler http.HandlerFunc) {
+ http.HandleFunc(path, mw(handler))
+}
+
+// mw is used as default middleware to set the default headers
+func mw(handler http.HandlerFunc) http.HandlerFunc {
+ return func(w http.ResponseWriter, r *http.Request) {
+ setDefaultHeaders(w)
+ handler(w, r)
+ }
+}
+
+// setDefaultHeaders sets the default headers that apply for all pages
+func setDefaultHeaders(w http.ResponseWriter) {
+ w.Header().Set("Cache-Control", config.CacheControl())
+}
diff --git a/pkg/config/config.go b/pkg/config/config.go
new file mode 100644
index 0000000..70ab9cc
--- /dev/null
+++ b/pkg/config/config.go
@@ -0,0 +1,75 @@
+package config
+
+import (
+ "os"
+ "strings"
+)
+
+func MailDirPath() string {
+ mailDir := getEnv("ARCHIVES_MAILDIR_PATH", "/var/archives/.maildir/")
+ if !strings.HasSuffix(mailDir, "/") {
+ mailDir = mailDir + "/"
+ }
+ return mailDir
+}
+
+func Port() string {
+ return getEnv("ARCHIVES_PORT", "5000")
+}
+
+func PostgresUser() string {
+ return getEnv("ARCHIVES_POSTGRES_USER", "admin")
+}
+
+func PostgresPass() string {
+ return getEnv("ARCHIVES_POSTGRES_PASS", "admin")
+}
+
+func PostgresDb() string {
+ return getEnv("ARCHIVES_POSTGRES_DB", "garchives")
+}
+
+func PostgresHost() string {
+ return getEnv("ARCHIVES_POSTGRES_HOST", "localhost")
+}
+
+func PostgresPort() string {
+ return getEnv("ARCHIVES_POSTGRES_PORT", "5432")
+}
+
+func CacheControl() string {
+ return getEnv("ARCHIVES_CACHE_CONTROL", "max-age=300")
+}
+
+func IndexMailingLists() [][]string {
+ return [][]string{
+ {"gentoo-dev", "is the main technical development mailing list of Gentoo"},
+ {"gentoo-project", "contains non-technical discussion and propositions for the Gentoo Council"},
+ {"gentoo-announce", "contains important news for all Gentoo stakeholders"},
+ {"gentoo-user", "is our main support and Gentoo-related talk mailing list"},
+ {"gentoo-commits", " - Lots of commits"},
+ {"gentoo-dev-announce", "conveys important changes to all developers and interested users"}}
+}
+
+func AllPublicMailingLists() []string {
+ var allMailingLists []string
+ allMailingLists = append(allMailingLists, CurrentMailingLists()...)
+ allMailingLists = append(allMailingLists, FrozenArchives()...)
+ return allMailingLists
+}
+
+func CurrentMailingLists() []string {
+ return []string{"gentoo-announce", "gentoo-commits", "gentoo-dev", "gentoo-dev-announce", "gentoo-nfp", "gentoo-project", "gentoo-user"}
+}
+
+func FrozenArchives() []string {
+ return []string{"gentoo-arm", "gentoo-au", "gentoo-council", "gentoo-cygwin", "gentoo-desktop-research"}
+}
+
+func getEnv(key string, fallback string) string {
+ if os.Getenv(key) != "" {
+ return os.Getenv(key)
+ } else {
+ return fallback
+ }
+}
diff --git a/pkg/database/connection.go b/pkg/database/connection.go
new file mode 100644
index 0000000..18fa2e6
--- /dev/null
+++ b/pkg/database/connection.go
@@ -0,0 +1,78 @@
+// Contains utility functions around the database
+
+package database
+
+import (
+ "archives/pkg/config"
+ "archives/pkg/models"
+ "context"
+ "github.com/go-pg/pg/v10"
+ "github.com/go-pg/pg/v10/orm"
+)
+
+// DBCon is the connection handle
+// for the database
+var (
+ DBCon *pg.DB
+)
+
+// CreateSchema creates the tables in the database
+// in case they don't alreay exist
+func CreateSchema() error {
+ if !tableExists("messages") {
+
+ err := DBCon.CreateTable((*models.Message)(nil), &orm.CreateTableOptions{
+ IfNotExists: true,
+ })
+
+ // Add tsvector column for subjects
+ DBCon.Exec("ALTER TABLE messages ADD COLUMN tsv_subject tsvector;")
+ DBCon.Exec("CREATE INDEX subject_idx ON messages USING gin(tsv_subject);")
+
+ // Add tsvector column for bodies
+ DBCon.Exec("ALTER TABLE messages ADD COLUMN tsv_body tsvector;")
+ DBCon.Exec("CREATE INDEX body_idx ON messages USING gin(tsv_body);")
+
+ return err
+ }
+ return nil
+}
+
+type dbLogger struct{}
+
+func (d dbLogger) BeforeQuery(c context.Context, q *pg.QueryEvent) (context.Context, error) {
+ return c, nil
+}
+
+// AfterQuery is used to log SQL queries
+func (d dbLogger) AfterQuery(c context.Context, q *pg.QueryEvent) error {
+ // logger.Debug.Println(q.FormattedQuery())
+ return nil
+}
+
+// Connect is used to connect to the database
+// and turn on logging if desired
+func Connect() {
+ DBCon = pg.Connect(&pg.Options{
+ User: config.PostgresUser(),
+ Password: config.PostgresPass(),
+ Database: config.PostgresDb(),
+ Addr: config.PostgresHost() + ":" + config.PostgresPort(),
+ })
+
+ DBCon.AddQueryHook(dbLogger{})
+
+ err := CreateSchema()
+ if err != nil {
+ // logger.Error.Println("ERROR: Could not create database schema")
+ // logger.Error.Println(err)
+ }
+
+}
+
+// utility methods
+
+func tableExists(tableName string) bool {
+ _, err := DBCon.Exec("select * from " + tableName + ";")
+ return err == nil
+}
diff --git a/pkg/importer/importer.go b/pkg/importer/importer.go
new file mode 100644
index 0000000..a989238
--- /dev/null
+++ b/pkg/importer/importer.go
@@ -0,0 +1,24 @@
+package importer
+
+import (
+ "archives/pkg/config"
+ "log"
+ "os"
+ "path/filepath"
+)
+
+func FullImport() {
+ err := filepath.Walk(config.MailDirPath(),
+ func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ if !info.IsDir() && getDepth(path, config.MailDirPath()) >= 1 {
+ importMail(info.Name(), path, config.MailDirPath())
+ }
+ return nil
+ })
+ if err != nil {
+ log.Println(err)
+ }
+}
diff --git a/pkg/importer/utils.go b/pkg/importer/utils.go
new file mode 100644
index 0000000..5672577
--- /dev/null
+++ b/pkg/importer/utils.go
@@ -0,0 +1,125 @@
+package importer
+
+import (
+ "archives/pkg/database"
+ "archives/pkg/models"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "log"
+ "mime/multipart"
+ "net/mail"
+ "os"
+ "regexp"
+ "strings"
+ "time"
+)
+
+func importMail(name, path, maildirPath string) {
+ file, _ := os.Open(path)
+ m, _ := mail.ReadMessage(file)
+
+ msg := models.Message{
+ Id: m.Header.Get("X-Archives-Hash"),
+ Filename: name,
+ Headers: m.Header,
+ Attachments: nil,
+ Body: getBody(m.Header, m.Body),
+ Date: getDate(m.Header),
+ Lists: getLists(m.Header),
+ Comment: "",
+ Hidden: false,
+ }
+
+ err := insertMessage(msg)
+
+ if err != nil {
+ fmt.Println("Error during importing Mail")
+ fmt.Println(err)
+ }
+}
+
+func getDepth(path, maildirPath string) int {
+ return strings.Count(strings.ReplaceAll(path, maildirPath, ""), "/")
+}
+
+func getBody(header mail.Header, body io.Reader) map[string]string {
+ if isMultipartMail(header) {
+ boundary := regexp.MustCompile(`boundary="(.*?)"`).
+ FindStringSubmatch(
+ header.Get("Content-Type"))
+ if len(boundary) != 2 {
+ //err
+ return map[string]string{
+ "text/plain": "",
+ }
+ }
+ return getBodyParts(body, boundary[1])
+ } else {
+ content, _ := ioutil.ReadAll(body)
+ return map[string]string{
+ getContentType(header): string(content),
+ }
+ }
+}
+
+func getBodyParts(body io.Reader, boundary string) map[string]string {
+ bodyParts := make(map[string]string)
+ mr := multipart.NewReader(body, boundary)
+ for {
+ p, err := mr.NextPart()
+ if err != nil {
+ return bodyParts
+ }
+ slurp, err := ioutil.ReadAll(p)
+ if err != nil {
+ log.Fatal(err)
+ }
+ bodyParts[p.Header.Get("Content-Type")] = string(slurp)
+ }
+ return bodyParts
+}
+
+func getContentType(header mail.Header) string {
+ contentTypes := regexp.MustCompile(`(.*?);`).
+ FindStringSubmatch(
+ header.Get("Content-Type"))
+ if len(contentTypes) < 2 {
+ // assume text/plain if we don't find a Content-Type header e.g. for git patches
+ return "text/plain"
+ }
+ return contentTypes[1]
+}
+
+func getDate(header mail.Header) time.Time {
+ date, _ := header.Date()
+ return date
+}
+
+func isMultipartMail(header mail.Header) bool {
+ return strings.Contains(getContentType(header), "multipart")
+}
+
+func getLists(header mail.Header) []string {
+ var lists []string
+ // To
+ adr, _ := mail.ParseAddressList(header.Get("To"))
+ for _, v := range adr {
+ lists = append(lists, v.Address)
+ }
+ // Cc
+ adr, _ = mail.ParseAddressList(header.Get("Cc"))
+ for _, v := range adr {
+ lists = append(lists, v.Address)
+ }
+ return lists
+}
+
+func insertMessage(message models.Message) error {
+ _, err := database.DBCon.Model(&message).
+ Value("tsv_subject", "to_tsvector(?)", message.GetSubject()).
+ Value("tsv_body", "to_tsvector(?)", message.GetBody()).
+ OnConflict("(id) DO NOTHING").
+ Insert()
+ return err
+}
diff --git a/pkg/models/mailinglist.go b/pkg/models/mailinglist.go
new file mode 100644
index 0000000..b090aed
--- /dev/null
+++ b/pkg/models/mailinglist.go
@@ -0,0 +1,8 @@
+package models
+
+type MailingList struct {
+ Name string
+ Description string
+ Messages []*Message
+ MessageCount int
+}
diff --git a/pkg/models/message.go b/pkg/models/message.go
new file mode 100644
index 0000000..17bbb9d
--- /dev/null
+++ b/pkg/models/message.go
@@ -0,0 +1,155 @@
+package models
+
+import (
+ "mime"
+ "net/mail"
+ "strings"
+ "time"
+)
+
+type Message struct {
+ Id string `pg:",pk"`
+ Filename string
+
+ Headers map[string][]string
+ Body map[string]string
+ Attachments []Attachment
+
+ Lists []string
+ Date time.Time
+
+ //Search types.ValueAppender // tsvector
+
+ Comment string
+ Hidden bool
+
+ //ParentId string
+ //Parent Message -> pg fk?
+}
+
+type Header struct {
+ Name string
+ Content string
+}
+
+type Body struct {
+ ContentType string
+ Content string
+}
+
+type Attachment struct {
+ Filename string
+ Mime string
+ Content string
+}
+
+func (m Message) GetSubject() string {
+ return m.GetHeaderField("Subject")
+}
+
+func (m Message) GetListNameFromSubject() string {
+ subject := m.GetSubject()
+ listName := strings.Split(subject, "]")[0]
+ listName = strings.ReplaceAll(listName, "[", "")
+ listName = strings.ReplaceAll(listName, "Re:", "")
+ listName = strings.TrimSpace(listName)
+ return listName
+}
+
+func (m Message) GetAuthorName() string {
+ addr, err := mail.ParseAddress(m.GetHeaderField("From"))
+ if err != nil {
+ return ""
+ }
+ return addr.Name
+}
+
+func (m Message) GetMessageId() string {
+ messageId := m.GetHeaderField("Message-Id")
+ messageId = strings.ReplaceAll(messageId, "<", "")
+ messageId = strings.ReplaceAll(messageId, ">", "")
+ messageId = strings.ReplaceAll(messageId, "\"", "")
+ return messageId
+}
+
+func (m Message) GetInReplyTo() string {
+ inReplyTo := m.GetHeaderField("In-Reply-To")
+ inReplyTo = strings.ReplaceAll(inReplyTo, "<", "")
+ inReplyTo = strings.ReplaceAll(inReplyTo, ">", "")
+ inReplyTo = strings.ReplaceAll(inReplyTo, " ", "")
+ return inReplyTo
+}
+
+func (m Message) GetHeaderField(key string) string {
+ subject, found := m.Headers[key]
+ if !found {
+ return ""
+ }
+ header := strings.Join(subject, " ")
+ if strings.Contains(header, "=?") {
+ dec := new(mime.WordDecoder)
+ decodedHeader, err := dec.DecodeHeader(header)
+ if err != nil {
+ return ""
+ }
+ return decodedHeader
+ }
+ return header
+}
+
+func (m Message) HasHeaderField(key string) bool {
+ _, found := m.Headers[key]
+ return found
+}
+
+func (m Message) GetBody() string {
+ // Get text/plain body
+ for contentType, content := range m.Body {
+ if strings.Contains(contentType, "text/plain") {
+ return content
+ }
+ }
+
+ // If text/plain is not present, fall back to html
+ for contentType, content := range m.Body {
+ if strings.Contains(contentType, "text/html") {
+ return content
+ }
+ }
+
+ // If neither text/plain nor text/html is available return nothing
+ return ""
+}
+
+func (m Message) HasAttachments() bool {
+ for key, _ := range m.Body {
+ if !(strings.Contains(key, "text/plain") || strings.Contains(key, "text/plain")) {
+ return true
+ }
+ }
+ return false
+}
+
+func (m Message) GetAttachments() []Attachment {
+ var attachments []Attachment
+ for key, content := range m.Body {
+ if !(strings.Contains(key, "text/plain") || strings.Contains(key, "text/plain")) {
+ attachments = append(attachments, Attachment{
+ Filename: getAttachmentFileName(key),
+ Mime: strings.Split(key, ";")[0],
+ Content: content,
+ })
+ }
+ }
+ return attachments
+}
+
+// utility methods
+
+func getAttachmentFileName(contentTypeHeader string) string {
+ parts := strings.Split(contentTypeHeader, "name=")
+ if len(parts) < 2 {
+ return "unknown"
+ }
+ return strings.ReplaceAll(parts[1], "\"", "")
+}
diff --git a/pkg/models/thread.go b/pkg/models/thread.go
new file mode 100644
index 0000000..ebeff2a
--- /dev/null
+++ b/pkg/models/thread.go
@@ -0,0 +1,8 @@
+package models
+
+type Threads []struct {
+ Id string
+ Headers map[string][]string
+ Subject string
+ Count int
+}