package main import ( "encoding/json" "fmt" "log" "net" "net/http" "net/url" "regexp" "strconv" "strings" "time" ) var ( API_AVAILABILITY string = "https://archive.org/wayback/available?url=" API_SAVE string = "https://web.archive.org/save" TIMEOUT time.Duration = 10 client *http.Client = &http.Client{Timeout: TIMEOUT * time.Second} ) type Save struct { URL string `json:"url"` Job string `json:"job_id"` Message string `json:"message"` } type Wayback struct { Snapshots Snapshot `json:"archived_snapshots,omitempty"` } type Snapshot struct { Recent Closest `json:"closest"` } type Closest struct { Available bool `json:"available"` Status string `json:"status"` } func isIgnored(regex []string, url string) bool { for _, r := range regex { if v := regexp.MustCompile(r); v.MatchString(url) { return true } } return false } func isArchived(url string) (bool, int) { req, err := http.NewRequest("GET", API_AVAILABILITY+url, nil) resp, err := client.Do(req) if err != nil { log.Println("isArchived: ", err) return false, 0 } defer resp.Body.Close() av := &Wayback{} decoder := json.NewDecoder(resp.Body) if err := decoder.Decode(av); err != nil { log.Println("isArchived:", err) return false, 0 } status, _ := strconv.Atoi(av.Snapshots.Recent.Status) return av.Snapshots.Recent.Available, status } func archive(accessKey string, secretKey string, URL string) (int, string, string) { params := url.Values{} params.Set("url", URL) req, _ := http.NewRequest(http.MethodPost, API_SAVE, strings.NewReader(params.Encode()), ) req.Header.Set("Accept", "application/json") auth := fmt.Sprintf("LOW %s:%s", accessKey, secretKey) req.Header.Set("Authorization", auth) req.Header.Set("Content-Type", "application/x-www-form-urlencoded") resp, err := client.Do(req) if err != nil { if e, _ := err.(net.Error); !e.Timeout() { log.Println("SAVE", err) } return 0, "", "" } defer resp.Body.Close() save := &Save{} decoder := json.NewDecoder(resp.Body) if err := decoder.Decode(save); err != nil { log.Println("SAVE", err) return 0, "", "" } return resp.StatusCode, save.Job, save.Message }