1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
|
package main
import (
"encoding/json"
"fmt"
"log"
"net"
"net/http"
"net/url"
"regexp"
"strconv"
"strings"
"time"
)
var (
API_AVAILABILITY string = "https://archive.org/wayback/available?url="
API_SAVE string = "https://web.archive.org/save"
TIMEOUT time.Duration = 10
client *http.Client = &http.Client{Timeout: TIMEOUT * time.Second}
)
type Save struct {
URL string `json:"url"`
Job string `json:"job_id"`
Message string `json:"message"`
}
type Wayback struct {
Snapshots Snapshot `json:"archived_snapshots,omitempty"`
}
type Snapshot struct {
Recent Closest `json:"closest"`
}
type Closest struct {
Available bool `json:"available"`
Status string `json:"status"`
}
func isIgnored(regex []string, url string) bool {
for _, r := range regex {
if v := regexp.MustCompile(r); v.MatchString(url) {
return true
}
}
return false
}
func isArchived(url string) (bool, int) {
req, err := http.NewRequest("GET", API_AVAILABILITY+url, nil)
resp, err := client.Do(req)
if err != nil {
log.Println("isArchived: ", err)
return false, 0
}
defer resp.Body.Close()
av := &Wayback{}
decoder := json.NewDecoder(resp.Body)
if err := decoder.Decode(av); err != nil {
log.Println("isArchived:", err)
return false, 0
}
status, _ := strconv.Atoi(av.Snapshots.Recent.Status)
return av.Snapshots.Recent.Available, status
}
func archive(accessKey string, secretKey string, URL string) (int, string, string) {
params := url.Values{}
params.Set("url", URL)
req, _ := http.NewRequest(http.MethodPost, API_SAVE,
strings.NewReader(params.Encode()),
)
req.Header.Set("Accept", "application/json")
auth := fmt.Sprintf("LOW %s:%s", accessKey, secretKey)
req.Header.Set("Authorization", auth)
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
resp, err := client.Do(req)
if err != nil {
if e, _ := err.(net.Error); !e.Timeout() {
log.Println("SAVE", err)
}
return 0, "", ""
}
defer resp.Body.Close()
save := &Save{}
decoder := json.NewDecoder(resp.Body)
if err := decoder.Decode(save); err != nil {
log.Println("SAVE", err)
return 0, "", ""
}
return resp.StatusCode, save.Job, save.Message
}
|
