From 88c17bacb8f3c8d1e04637f8a494698a9aee807a Mon Sep 17 00:00:00 2001 From: Jan-Lukas Else Date: Thu, 12 Dec 2019 16:29:26 +0100 Subject: [PATCH] Handle webmention receiving --- entry.go | 2 +- gitea.go | 100 +++++++++++++++++++++++++++++++++----- main.go | 84 +------------------------------- micropub.go | 87 ++++++++++++++++++++++++++++++++++ webmention.go | 129 ++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 308 insertions(+), 94 deletions(-) create mode 100644 micropub.go diff --git a/entry.go b/entry.go index 54e24ae..15fd2d9 100644 --- a/entry.go +++ b/entry.go @@ -206,7 +206,7 @@ func ReadEntry(url string) (entry *Entry, err error) { if err != nil { return } - fileContent, err := ReadFile(filePath) + fileContent, err := ReadFileContent(filePath) if err != nil { return } diff --git a/gitea.go b/gitea.go index b77226b..4081a39 100644 --- a/gitea.go +++ b/gitea.go @@ -8,18 +8,34 @@ import ( "io/ioutil" "net/http" "net/url" + "strings" ) -func CreateFile(path string, file string, name string) error { - message := map[string]interface{}{ - "message": name, - "content": base64.StdEncoding.EncodeToString([]byte(file)), +type GiteaCommitRequest struct { + Content string `json:"content"` + Message string `json:"message"` + SHA string `json:"sha,omitempty"` +} + +type GiteaReadRequest struct { + Type string `json:"type"` + Content string `json:"content"` + SHA string `json:"sha"` +} + +type GiteaErrorResponse struct { + Message string `json:"message"` +} + +func CreateFile(path string, file string, message string) error { + request := &GiteaCommitRequest{ + Content: base64.StdEncoding.EncodeToString([]byte(file)), + Message: message, } - bytesRepresentation, err := json.Marshal(message) + bytesRepresentation, err := json.Marshal(request) if err != nil { return errors.New("failed to marshal json before committing") } - // TODO: handle file updating resp, err := http.Post(GiteaEndpoint+url.QueryEscape(path)+"?access_token="+GiteaToken, "application/json", bytes.NewBuffer(bytesRepresentation)) if err != nil || resp.StatusCode != 201 { return errors.New("failed to create file in repo") @@ -27,26 +43,88 @@ func CreateFile(path string, file string, name string) error { return nil } -func ReadFile(path string) (fileContent string, err error) { +func UpdateFile(path string, file string, message string) error { + existingFile, exists, err := ReadFile(path) + if err != nil { + return err + } + if !exists { + // File doesn't exist, create it + return CreateFile(path, file, message) + } + request := &GiteaCommitRequest{ + Content: base64.StdEncoding.EncodeToString([]byte(file)), + Message: message, + SHA: existingFile.SHA, + } + bytesRepresentation, err := json.Marshal(request) + if err != nil { + return errors.New("failed to marshal json before committing") + } + req, err := http.NewRequest(http.MethodPut, GiteaEndpoint+url.QueryEscape(path)+"?access_token="+GiteaToken, bytes.NewBuffer(bytesRepresentation)) + if err != nil { + return errors.New("error making update request") + } + req.Header.Set("Content-type", "application/json") + client := &http.Client{} + resp, err := client.Do(req) + if err != nil || resp.StatusCode != 200 { + return errors.New("failed to update file in repo") + } + return nil +} + +func ReadFile(path string) (response *GiteaReadRequest, exists bool, err error) { + exists = false resp, err := http.Get(GiteaEndpoint + url.QueryEscape(path) + "?access_token=" + GiteaToken) if err != nil || resp.StatusCode != 200 { + if resp != nil { + defer resp.Body.Close() + body, readErr := ioutil.ReadAll(resp.Body) + if readErr != nil { + err = errors.New("failed reading Gitea error response") + return + } + errorResponse := &GiteaErrorResponse{} + marshalErr := json.Unmarshal(body, &errorResponse) + if marshalErr != nil { + err = errors.New("failed parsing Gitea error response") + return + } + exists = !strings.Contains(errorResponse.Message, "does not exist") + if !exists { + return + } + } err = errors.New("failed to read file in repo") return } + exists = true defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { err = errors.New("failed reading file in repo") return } - giteaResponseBody := struct { - Content string - }{} - err = json.Unmarshal(body, &giteaResponseBody) + response = &GiteaReadRequest{} + err = json.Unmarshal(body, &response) if err != nil { err = errors.New("failed parsing Gitea response") return } + return +} + +func ReadFileContent(path string) (fileContent string, err error) { + giteaResponseBody, exists, err := ReadFile(path) + if err != nil { + return + } + if !exists { + // File doesn't exist, nothing to read + err = errors.New("file does not exist") + return + } decodedBytes, err := base64.StdEncoding.DecodeString(giteaResponseBody.Content) if err != nil { err = errors.New("failed decoding file content") diff --git a/main.go b/main.go index f88ebb5..1227427 100644 --- a/main.go +++ b/main.go @@ -3,96 +3,16 @@ package main import ( "log" "net/http" - "strconv" "strings" "time" ) -func handleMicroPub(w http.ResponseWriter, r *http.Request) { - // a handler for GET requests, used for troubleshooting - if r.Method == "GET" { - if q := r.URL.Query().Get("q"); q == "syndicate-to" { - w.Header().Add("Content-type", "application/json") - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte("[]")) - return - } else if url := r.URL.Query().Get("url"); q == "source" { - limit := r.URL.Query().Get("limit") - limitInt, err := strconv.Atoi(limit) - jsonBytes, err := QueryURL(url, limitInt) - if err != nil { - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte(err.Error())) - return - } - w.Header().Add("Content-type", "application/json") - w.WriteHeader(http.StatusOK) - _, _ = w.Write(jsonBytes) - return - } else { - w.Header().Add("Content-type", "application/json") - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte("{}")) - return - } - } - // check if the request is a POST - if r.Method != "POST" { - w.WriteHeader(http.StatusMethodNotAllowed) - _, _ = w.Write([]byte("The HTTP method is not allowed, make a POST request")) - return - } - // check content type - contentType, err := GetContentType(r.Header.Get("content-type")) - if err != nil { - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte(err.Error())) - return - } - // Create entry - entry, err := CreateEntry(contentType, r) - if err != nil || entry == nil { - w.WriteHeader(http.StatusBadRequest) - if err != nil { - _, _ = w.Write([]byte(err.Error())) - } else { - _, _ = w.Write([]byte("There was an error creating the entry")) - } - return - } - if CheckAuthorization(entry, r.Header.Get("authorization")) { - location, err := WriteEntry(entry) - if err != nil { - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte("There was an error committing the entry to the repository")) - return - } else { - w.Header().Add("Location", location) - w.WriteHeader(http.StatusAccepted) - // Purge BunnyCDN in 10 seconds - go func() { - time.Sleep(10 * time.Second) - Purge(location) - // Send webmentions - go func() { - time.Sleep(3 * time.Second) - SendWebmentions(location) - }() - }() - return - } - } else { - w.WriteHeader(http.StatusForbidden) - _, _ = w.Write([]byte("Forbidden, there was a problem with the provided access token")) - return - } -} - func main() { - http.HandleFunc("/", handleMicroPub) log.Println("Starting micropub server...") log.Println("Current time: " + time.Now().Format(time.RFC3339)) log.Println("Blog URL: " + BlogUrl) log.Println("Ignored URLs for Webmention: " + strings.Join(IgnoredWebmentionUrls, ", ")) + http.HandleFunc("/micropub", HandleMicroPub) + http.HandleFunc("/webmention", HandleWebmention) log.Fatal(http.ListenAndServe(":5555", nil)) } diff --git a/micropub.go b/micropub.go new file mode 100644 index 0000000..f32a2dc --- /dev/null +++ b/micropub.go @@ -0,0 +1,87 @@ +package main + +import ( + "net/http" + "strconv" + "time" +) + +func HandleMicroPub(w http.ResponseWriter, r *http.Request) { + // a handler for GET requests, used for troubleshooting + if r.Method == "GET" { + if q := r.URL.Query().Get("q"); q == "syndicate-to" { + w.Header().Add("Content-type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("[]")) + return + } else if url := r.URL.Query().Get("url"); q == "source" { + limit := r.URL.Query().Get("limit") + limitInt, err := strconv.Atoi(limit) + jsonBytes, err := QueryURL(url, limitInt) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(err.Error())) + return + } + w.Header().Add("Content-type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(jsonBytes) + return + } else { + w.Header().Add("Content-type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + return + } + } + // check if the request is a POST + if r.Method != "POST" { + w.WriteHeader(http.StatusMethodNotAllowed) + _, _ = w.Write([]byte("The HTTP method is not allowed, make a POST request")) + return + } + // check content type + contentType, err := GetContentType(r.Header.Get("content-type")) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(err.Error())) + return + } + // Create entry + entry, err := CreateEntry(contentType, r) + if err != nil || entry == nil { + w.WriteHeader(http.StatusBadRequest) + if err != nil { + _, _ = w.Write([]byte(err.Error())) + } else { + _, _ = w.Write([]byte("There was an error creating the entry")) + } + return + } + if CheckAuthorization(entry, r.Header.Get("authorization")) { + location, err := WriteEntry(entry) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte("There was an error committing the entry to the repository")) + return + } else { + w.Header().Add("Location", location) + w.WriteHeader(http.StatusAccepted) + // Purge BunnyCDN in 10 seconds + go func() { + time.Sleep(10 * time.Second) + Purge(location) + // Send webmentions + go func() { + time.Sleep(3 * time.Second) + SendWebmentions(location) + }() + }() + return + } + } else { + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte("Forbidden, there was a problem with the provided access token")) + return + } +} diff --git a/webmention.go b/webmention.go index f305201..5608e51 100644 --- a/webmention.go +++ b/webmention.go @@ -1,11 +1,24 @@ package main import ( + "crypto/md5" + "encoding/json" + "errors" + "fmt" "log" + "net/http" + "net/url" "strings" + "time" "willnorris.com/go/webmention" ) +type Mention struct { + Source string `json:"source"` + Target string `json:"target"` + Date string `json:"date"` +} + func SendWebmentions(url string) { client := webmention.New(nil) dl, err := client.DiscoverLinks(url, ".h-entry") @@ -42,4 +55,120 @@ LINKFILTER: filteredLinks = append(filteredLinks, link) } return filteredLinks +} + +func HandleWebmention(w http.ResponseWriter, r *http.Request) { + if r.Method != "POST" { + w.WriteHeader(http.StatusMethodNotAllowed) + _, _ = w.Write([]byte("The HTTP method is not allowed, make a POST request")) + return + } + sourceUrl, err := url.Parse(r.FormValue("source")) + if err != nil || !(sourceUrl.Scheme == "http" || sourceUrl.Scheme == "https") { + err = errors.New("failed to parse webmention source URL") + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(err.Error())) + return + } + targetUrl, err := url.Parse(r.FormValue("target")) + if err != nil || !(sourceUrl.Scheme == "http" || sourceUrl.Scheme == "https") { + err = errors.New("failed to parse webmention target URL") + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(err.Error())) + return + } + // Check if urls don't equal + if sourceUrl.String() == targetUrl.String() { + err = errors.New("source and target equal") + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(err.Error())) + return + } + // Check if target is blog + if !strings.HasPrefix(targetUrl.String(), BlogUrl) { + err = errors.New("wrong webmention target") + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(err.Error())) + return + } + // Check response code for source + respCode, err := responseCodeForSource(sourceUrl.String()) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + if respCode < 200 || respCode >= 300 { + if respCode == 410 { + // Delete mention, because source is gone + // TODO: Implement deleteion + w.Header().Add("Location", targetUrl.String()) + w.WriteHeader(http.StatusOK) + return + } else { + err = errors.New("source returned error") + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(err.Error())) + return + } + } + // Check if source mentions target + if !sourceMentionsTarget(sourceUrl.String(), targetUrl.String()) { + err = errors.New("source doesn't mention target") + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(err.Error())) + return + } + mention := &Mention{ + Source: sourceUrl.String(), + Target: targetUrl.String(), + Date: time.Now().Format(time.RFC3339), + } + err = saveWebmention(mention) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + w.Header().Add("Location", targetUrl.String()) + w.WriteHeader(http.StatusOK) + // Purge BunnyCDN in 10 seconds + go func() { + time.Sleep(10 * time.Second) + Purge(targetUrl.String()) + }() + return +} + +func responseCodeForSource(source string) (int, error) { + client := &http.Client{} + resp, err := client.Get(source) + if err != nil || resp == nil { + return 0, err + } + return resp.StatusCode, nil +} + +func sourceMentionsTarget(source string, target string) bool { + client := webmention.New(nil) + dl, err := client.DiscoverLinks(source, "") + if err != nil { + return false + } + for _, link := range dl { + if link == target { + return true + } + } + return false +} + +func saveWebmention(mention *Mention) (err error) { + bytesRepresentation, err := json.Marshal(mention) + if err != nil { + return errors.New("failed to marshal json before committing") + } + filePath := fmt.Sprintf("data/mentions/%x/%x.json", md5.Sum([]byte(mention.Target)), md5.Sum([]byte(mention.Source))) + err = UpdateFile(filePath, string(bytesRepresentation), "New webmention from "+mention.Source) + return } \ No newline at end of file