aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJordan <me@jordan.im>2022-01-15 10:09:56 -0700
committerJordan <me@jordan.im>2022-01-15 10:09:56 -0700
commit4a92c26fbc739734ce5227c1be8efd440306cebf (patch)
tree3a009e9ae3543725a597b77a838d0b41690b2e0e
parent995a8fad1461734108b5266f51a3c913936c7203 (diff)
downloadkeep-4a92c26fbc739734ce5227c1be8efd440306cebf.tar.gz
keep-4a92c26fbc739734ce5227c1be8efd440306cebf.zip
keep, archive: normalize URLs (RFC 3986), improve logging
-rw-r--r--Makefile2
-rw-r--r--archive.go6
-rw-r--r--go.mod12
-rw-r--r--go.sum10
-rw-r--r--keep.go25
-rw-r--r--normalize/LICENSE12
-rw-r--r--normalize/normalize.go379
-rw-r--r--normalize/normalize_test.go782
-rw-r--r--urlesc/LICENSE27
-rw-r--r--urlesc/urlesc.go180
-rw-r--r--urlesc/urlesc_test.go641
11 files changed, 2058 insertions, 18 deletions
diff --git a/Makefile b/Makefile
index ddff8eb..1745ef2 100644
--- a/Makefile
+++ b/Makefile
@@ -3,7 +3,7 @@
GO = go
RM = rm
-GOFLAGS =
+GOFLAGS = -o keep
PREFIX = /usr/local
BINDIR = $(PREFIX)/bin
CONFIGDIR = $(HOME)/.keep
diff --git a/archive.go b/archive.go
index 768453e..7c38c51 100644
--- a/archive.go
+++ b/archive.go
@@ -47,13 +47,13 @@ func isArchived(url string) (bool, int) {
req, err := http.NewRequest("GET", API_AVAILABILITY+url, nil)
resp, err := client.Do(req)
if err != nil {
- log.Println(err)
+ log.Println("isArchived: ", err)
return false, 0
}
av := &Wayback{}
decoder := json.NewDecoder(resp.Body)
if err := decoder.Decode(av); err != nil {
- log.Println(err)
+ log.Println("isArchived:", err)
return false, 0
}
status, _ := strconv.Atoi(av.Snapshots.Recent.Status)
@@ -66,7 +66,7 @@ func archive(url string) int {
resp, err := client.Do(req)
if err != nil {
if e, _ := err.(net.Error); !e.Timeout() {
- log.Println(err)
+ log.Println("archive:", err)
}
return 0
}
diff --git a/go.mod b/go.mod
index ef29ca4..bd15d88 100644
--- a/go.mod
+++ b/go.mod
@@ -1,9 +1,15 @@
module keep
-go 1.16
+go 1.17
require (
github.com/bwmarrin/discordgo v0.23.2
- github.com/mattn/go-sqlite3 v1.14.9
- golang.org/x/net v0.0.0-20211205041911-012df41ee64c
+ github.com/mattn/go-sqlite3 v1.14.10
+ golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d
+ golang.org/x/text v0.3.7
+)
+
+require (
+ github.com/gorilla/websocket v1.4.0 // indirect
+ golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16 // indirect
)
diff --git a/go.sum b/go.sum
index f022b29..2874832 100644
--- a/go.sum
+++ b/go.sum
@@ -2,14 +2,16 @@ github.com/bwmarrin/discordgo v0.23.2 h1:BzrtTktixGHIu9Tt7dEE6diysEF9HWnXeHuoJEt
github.com/bwmarrin/discordgo v0.23.2/go.mod h1:c1WtWUGN6nREDmzIpyTp/iD3VYt4Fpx+bVyfBG7JE+M=
github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA=
-github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
+github.com/mattn/go-sqlite3 v1.14.10 h1:MLn+5bFRlWMGoSRmJour3CL1w/qL96mvipqpwQW/Sfk=
+github.com/mattn/go-sqlite3 v1.14.10/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16 h1:y6ce7gCWtnH+m3dCjzQ1PCuwl28DDIc3VNnvY29DlIA=
golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/net v0.0.0-20211205041911-012df41ee64c h1:7SfqwP5fxEtl/P02w5IhKc86ziJ+A25yFrkVgoy2FT8=
-golang.org/x/net v0.0.0-20211205041911-012df41ee64c/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d h1:1n1fc535VhN8SYtD4cDUyNlfpAF2ROMM9+11equK3hs=
+golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
diff --git a/keep.go b/keep.go
index 0c0ad49..109ff5d 100644
--- a/keep.go
+++ b/keep.go
@@ -19,6 +19,7 @@ import (
"github.com/bwmarrin/discordgo"
"golang.org/x/net/publicsuffix"
+ "keep/normalize"
)
type Config struct {
@@ -125,9 +126,10 @@ func archiver(db *SqliteDB) {
// Blocks until URL is received
message := <-messageChan
- // Skip if we have URL in database
- cached, _ := db.IsCached(message.URL)
+ // Skip if we've already seen URL (cached)
+ cached, status_code := db.IsCached(message.URL)
if cached {
+ log.Println("SEEN", status_code, message.URL)
continue
}
@@ -135,14 +137,14 @@ func archiver(db *SqliteDB) {
archived, status_code := isArchived(message.URL)
if archived && status_code == http.StatusOK {
db.AddArchived(message, status_code)
- log.Printf("SKIP %d %s", status_code, message.URL)
+ log.Println("SKIP", status_code, message.URL)
continue
}
// Archive, URL is not present in cache or IA
status_code = archive(message.URL)
db.AddArchived(message, status_code)
- log.Printf("SAVE %d %s", status_code, message.URL)
+ log.Println("SAVE", status_code, message.URL)
// Limit requests to Wayback API to 15-second intervals
time.Sleep(15 * time.Second)
@@ -157,7 +159,7 @@ func messageCreate(s *discordgo.Session, m *discordgo.MessageCreate) {
if m.Content == "" {
chanMsgs, err := s.ChannelMessages(m.ChannelID, 1, "", "", m.ID)
if err != nil {
- log.Printf("Unable to get messages: %s", err)
+ log.Println("Unable to get messages:", err)
return
}
if len(chanMsgs) > 0 {
@@ -185,14 +187,23 @@ func messageCreate(s *discordgo.Session, m *discordgo.MessageCreate) {
continue
}
+ // Normalize URL (RFC 3986)
+ uStr := normalize.NormalizeURL(u,
+ normalize.FlagsSafe|normalize.FlagRemoveDotSegments|
+ normalize.FlagRemoveDuplicateSlashes|
+ normalize.FlagRemoveFragment|
+ normalize.FlagSortQuery)
+
+ log.Println(uStr)
+
// Ensure host is not present in ignoreList set
- if isIgnored(config.Ignore, w) {
+ if isIgnored(config.Ignore, uStr) {
continue
}
// Send message attributes/URL over the channel
message := Message{
- URL: w,
+ URL: uStr,
Author: m.Author.ID,
Guild: m.GuildID,
Channel: m.ChannelID,
diff --git a/normalize/LICENSE b/normalize/LICENSE
new file mode 100644
index 0000000..4b9986d
--- /dev/null
+++ b/normalize/LICENSE
@@ -0,0 +1,12 @@
+Copyright (c) 2012, Martin Angers
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+* Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/normalize/normalize.go b/normalize/normalize.go
new file mode 100644
index 0000000..0220f65
--- /dev/null
+++ b/normalize/normalize.go
@@ -0,0 +1,379 @@
+/*
+Package normalize offers URL normalization as described on the wikipedia page:
+http://en.wikipedia.org/wiki/URL_normalization
+*/
+package normalize
+
+import (
+ "bytes"
+ "fmt"
+ "net/url"
+ "regexp"
+ "sort"
+ "strconv"
+ "strings"
+
+ "golang.org/x/net/idna"
+ "golang.org/x/text/unicode/norm"
+ "golang.org/x/text/width"
+ "keep/urlesc"
+)
+
+// A set of normalization flags determines how a URL will
+// be normalized.
+type NormalizationFlags uint
+
+const (
+ // Safe normalizations
+ FlagLowercaseScheme NormalizationFlags = 1 << iota // HTTP://host -> http://host, applied by default in Go1.1
+ FlagLowercaseHost // http://HOST -> http://host
+ FlagUppercaseEscapes // http://host/t%ef -> http://host/t%EF
+ FlagDecodeUnnecessaryEscapes // http://host/t%41 -> http://host/tA
+ FlagEncodeNecessaryEscapes // http://host/!"#$ -> http://host/%21%22#$
+ FlagRemoveDefaultPort // http://host:80 -> http://host
+ FlagRemoveEmptyQuerySeparator // http://host/path? -> http://host/path
+
+ // Usually safe normalizations
+ FlagRemoveTrailingSlash // http://host/path/ -> http://host/path
+ FlagAddTrailingSlash // http://host/path -> http://host/path/ (should choose only one of these add/remove trailing slash flags)
+ FlagRemoveDotSegments // http://host/path/./a/b/../c -> http://host/path/a/c
+
+ // Unsafe normalizations
+ FlagRemoveDirectoryIndex // http://host/path/index.html -> http://host/path/
+ FlagRemoveFragment // http://host/path#fragment -> http://host/path
+ FlagForceHTTP // https://host -> http://host
+ FlagRemoveDuplicateSlashes // http://host/path//a///b -> http://host/path/a/b
+ FlagRemoveWWW // http://www.host/ -> http://host/
+ FlagAddWWW // http://host/ -> http://www.host/ (should choose only one of these add/remove WWW flags)
+ FlagSortQuery // http://host/path?c=3&b=2&a=1&b=1 -> http://host/path?a=1&b=1&b=2&c=3
+
+ // Normalizations not in the wikipedia article, required to cover tests cases
+ // submitted by jehiah
+ FlagDecodeDWORDHost // http://1113982867 -> http://66.102.7.147
+ FlagDecodeOctalHost // http://0102.0146.07.0223 -> http://66.102.7.147
+ FlagDecodeHexHost // http://0x42660793 -> http://66.102.7.147
+ FlagRemoveUnnecessaryHostDots // http://.host../path -> http://host/path
+ FlagRemoveEmptyPortSeparator // http://host:/path -> http://host/path
+
+ // Convenience set of safe normalizations
+ FlagsSafe NormalizationFlags = FlagLowercaseHost | FlagLowercaseScheme | FlagUppercaseEscapes | FlagDecodeUnnecessaryEscapes | FlagEncodeNecessaryEscapes | FlagRemoveDefaultPort | FlagRemoveEmptyQuerySeparator
+
+ // For convenience sets, "greedy" uses the "remove trailing slash" and "remove www. prefix" flags,
+ // while "non-greedy" uses the "add (or keep) the trailing slash" and "add www. prefix".
+
+ // Convenience set of usually safe normalizations (includes FlagsSafe)
+ FlagsUsuallySafeGreedy NormalizationFlags = FlagsSafe | FlagRemoveTrailingSlash | FlagRemoveDotSegments
+ FlagsUsuallySafeNonGreedy NormalizationFlags = FlagsSafe | FlagAddTrailingSlash | FlagRemoveDotSegments
+
+ // Convenience set of unsafe normalizations (includes FlagsUsuallySafe)
+ FlagsUnsafeGreedy NormalizationFlags = FlagsUsuallySafeGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagRemoveWWW | FlagSortQuery
+ FlagsUnsafeNonGreedy NormalizationFlags = FlagsUsuallySafeNonGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagAddWWW | FlagSortQuery
+
+ // Convenience set of all available flags
+ FlagsAllGreedy = FlagsUnsafeGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator
+ FlagsAllNonGreedy = FlagsUnsafeNonGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator
+)
+
+const (
+ defaultHttpPort = ":80"
+ defaultHttpsPort = ":443"
+)
+
+// Regular expressions used by the normalizations
+var rxPort = regexp.MustCompile(`(:\d+)/?$`)
+var rxDirIndex = regexp.MustCompile(`(^|/)((?:default|index)\.\w{1,4})$`)
+var rxDupSlashes = regexp.MustCompile(`/{2,}`)
+var rxDWORDHost = regexp.MustCompile(`^(\d+)((?:\.+)?(?:\:\d*)?)$`)
+var rxOctalHost = regexp.MustCompile(`^(0\d*)\.(0\d*)\.(0\d*)\.(0\d*)((?:\.+)?(?:\:\d*)?)$`)
+var rxHexHost = regexp.MustCompile(`^0x([0-9A-Fa-f]+)((?:\.+)?(?:\:\d*)?)$`)
+var rxHostDots = regexp.MustCompile(`^(.+?)(:\d+)?$`)
+var rxEmptyPort = regexp.MustCompile(`:+$`)
+
+// Map of flags to implementation function.
+// FlagDecodeUnnecessaryEscapes has no action, since it is done automatically
+// by parsing the string as an URL. Same for FlagUppercaseEscapes and FlagRemoveEmptyQuerySeparator.
+
+// Since maps have undefined traversing order, make a slice of ordered keys
+var flagsOrder = []NormalizationFlags{
+ FlagLowercaseScheme,
+ FlagLowercaseHost,
+ FlagRemoveDefaultPort,
+ FlagRemoveDirectoryIndex,
+ FlagRemoveDotSegments,
+ FlagRemoveFragment,
+ FlagForceHTTP, // Must be after remove default port (because https=443/http=80)
+ FlagRemoveDuplicateSlashes,
+ FlagRemoveWWW,
+ FlagAddWWW,
+ FlagSortQuery,
+ FlagDecodeDWORDHost,
+ FlagDecodeOctalHost,
+ FlagDecodeHexHost,
+ FlagRemoveUnnecessaryHostDots,
+ FlagRemoveEmptyPortSeparator,
+ FlagRemoveTrailingSlash, // These two (add/remove trailing slash) must be last
+ FlagAddTrailingSlash,
+}
+
+// ... and then the map, where order is unimportant
+var flags = map[NormalizationFlags]func(*url.URL){
+ FlagLowercaseScheme: lowercaseScheme,
+ FlagLowercaseHost: lowercaseHost,
+ FlagRemoveDefaultPort: removeDefaultPort,
+ FlagRemoveDirectoryIndex: removeDirectoryIndex,
+ FlagRemoveDotSegments: removeDotSegments,
+ FlagRemoveFragment: removeFragment,
+ FlagForceHTTP: forceHTTP,
+ FlagRemoveDuplicateSlashes: removeDuplicateSlashes,
+ FlagRemoveWWW: removeWWW,
+ FlagAddWWW: addWWW,
+ FlagSortQuery: sortQuery,
+ FlagDecodeDWORDHost: decodeDWORDHost,
+ FlagDecodeOctalHost: decodeOctalHost,
+ FlagDecodeHexHost: decodeHexHost,
+ FlagRemoveUnnecessaryHostDots: removeUnncessaryHostDots,
+ FlagRemoveEmptyPortSeparator: removeEmptyPortSeparator,
+ FlagRemoveTrailingSlash: removeTrailingSlash,
+ FlagAddTrailingSlash: addTrailingSlash,
+}
+
+// MustNormalizeURLString returns the normalized string, and panics if an error occurs.
+// It takes an URL string as input, as well as the normalization flags.
+func MustNormalizeURLString(u string, f NormalizationFlags) string {
+ result, e := NormalizeURLString(u, f)
+ if e != nil {
+ panic(e)
+ }
+ return result
+}
+
+// NormalizeURLString returns the normalized string, or an error if it can't be parsed into an URL object.
+// It takes an URL string as input, as well as the normalization flags.
+func NormalizeURLString(u string, f NormalizationFlags) (string, error) {
+ parsed, err := url.Parse(u)
+ if err != nil {
+ return "", err
+ }
+
+ if f&FlagLowercaseHost == FlagLowercaseHost {
+ parsed.Host = strings.ToLower(parsed.Host)
+ }
+
+ // The idna package doesn't fully conform to RFC 5895
+ // (https://tools.ietf.org/html/rfc5895), so we do it here.
+ // Taken from Go 1.8 cycle source, courtesy of bradfitz.
+ // TODO: Remove when (if?) idna package conforms to RFC 5895.
+ parsed.Host = width.Fold.String(parsed.Host)
+ parsed.Host = norm.NFC.String(parsed.Host)
+ if parsed.Host, err = idna.ToASCII(parsed.Host); err != nil {
+ return "", err
+ }
+
+ return NormalizeURL(parsed, f), nil
+}
+
+// NormalizeURL returns the normalized string.
+// It takes a parsed URL object as input, as well as the normalization flags.
+func NormalizeURL(u *url.URL, f NormalizationFlags) string {
+ for _, k := range flagsOrder {
+ if f&k == k {
+ flags[k](u)
+ }
+ }
+ return urlesc.Escape(u)
+}
+
+func lowercaseScheme(u *url.URL) {
+ if len(u.Scheme) > 0 {
+ u.Scheme = strings.ToLower(u.Scheme)
+ }
+}
+
+func lowercaseHost(u *url.URL) {
+ if len(u.Host) > 0 {
+ u.Host = strings.ToLower(u.Host)
+ }
+}
+
+func removeDefaultPort(u *url.URL) {
+ if len(u.Host) > 0 {
+ scheme := strings.ToLower(u.Scheme)
+ u.Host = rxPort.ReplaceAllStringFunc(u.Host, func(val string) string {
+ if (scheme == "http" && val == defaultHttpPort) || (scheme == "https" && val == defaultHttpsPort) {
+ return ""
+ }
+ return val
+ })
+ }
+}
+
+func removeTrailingSlash(u *url.URL) {
+ if l := len(u.Path); l > 0 {
+ if strings.HasSuffix(u.Path, "/") {
+ u.Path = u.Path[:l-1]
+ }
+ } else if l = len(u.Host); l > 0 {
+ if strings.HasSuffix(u.Host, "/") {
+ u.Host = u.Host[:l-1]
+ }
+ }
+}
+
+func addTrailingSlash(u *url.URL) {
+ if l := len(u.Path); l > 0 {
+ if !strings.HasSuffix(u.Path, "/") {
+ u.Path += "/"
+ }
+ } else if l = len(u.Host); l > 0 {
+ if !strings.HasSuffix(u.Host, "/") {
+ u.Host += "/"
+ }
+ }
+}
+
+func removeDotSegments(u *url.URL) {
+ if len(u.Path) > 0 {
+ var dotFree []string
+ var lastIsDot bool
+
+ sections := strings.Split(u.Path, "/")
+ for _, s := range sections {
+ if s == ".." {
+ if len(dotFree) > 0 {
+ dotFree = dotFree[:len(dotFree)-1]
+ }
+ } else if s != "." {
+ dotFree = append(dotFree, s)
+ }
+ lastIsDot = (s == "." || s == "..")
+ }
+ // Special case if host does not end with / and new path does not begin with /
+ u.Path = strings.Join(dotFree, "/")
+ if u.Host != "" && !strings.HasSuffix(u.Host, "/") && !strings.HasPrefix(u.Path, "/") {
+ u.Path = "/" + u.Path
+ }
+ // Special case if the last segment was a dot, make sure the path ends with a slash
+ if lastIsDot && !strings.HasSuffix(u.Path, "/") {
+ u.Path += "/"
+ }
+ }
+}
+
+func removeDirectoryIndex(u *url.URL) {
+ if len(u.Path) > 0 {
+ u.Path = rxDirIndex.ReplaceAllString(u.Path, "$1")
+ }
+}
+
+func removeFragment(u *url.URL) {
+ u.Fragment = ""
+}
+
+func forceHTTP(u *url.URL) {
+ if strings.ToLower(u.Scheme) == "https" {
+ u.Scheme = "http"
+ }
+}
+
+func removeDuplicateSlashes(u *url.URL) {
+ if len(u.Path) > 0 {
+ u.Path = rxDupSlashes.ReplaceAllString(u.Path, "/")
+ }
+}
+
+func removeWWW(u *url.URL) {
+ if len(u.Host) > 0 && strings.HasPrefix(strings.ToLower(u.Host), "www.") {
+ u.Host = u.Host[4:]
+ }
+}
+
+func addWWW(u *url.URL) {
+ if len(u.Host) > 0 && !strings.HasPrefix(strings.ToLower(u.Host), "www.") {
+ u.Host = "www." + u.Host
+ }
+}
+
+func sortQuery(u *url.URL) {
+ q := u.Query()
+
+ if len(q) > 0 {
+ arKeys := make([]string, len(q))
+ i := 0
+ for k := range q {
+ arKeys[i] = k
+ i++
+ }
+ sort.Strings(arKeys)
+ buf := new(bytes.Buffer)
+ for _, k := range arKeys {
+ sort.Strings(q[k])
+ for _, v := range q[k] {
+ if buf.Len() > 0 {
+ buf.WriteRune('&')
+ }
+ buf.WriteString(fmt.Sprintf("%s=%s", k, urlesc.QueryEscape(v)))
+ }
+ }
+
+ // Rebuild the raw query string
+ u.RawQuery = buf.String()
+ }
+}
+
+func decodeDWORDHost(u *url.URL) {
+ if len(u.Host) > 0 {
+ if matches := rxDWORDHost.FindStringSubmatch(u.Host); len(matches) > 2 {
+ var parts [4]int64
+
+ dword, _ := strconv.ParseInt(matches[1], 10, 0)
+ for i, shift := range []uint{24, 16, 8, 0} {
+ parts[i] = dword >> shift & 0xFF
+ }
+ u.Host = fmt.Sprintf("%d.%d.%d.%d%s", parts[0], parts[1], parts[2], parts[3], matches[2])
+ }
+ }
+}
+
+func decodeOctalHost(u *url.URL) {
+ if len(u.Host) > 0 {
+ if matches := rxOctalHost.FindStringSubmatch(u.Host); len(matches) > 5 {
+ var parts [4]int64
+
+ for i := 1; i <= 4; i++ {
+ parts[i-1], _ = strconv.ParseInt(matches[i], 8, 0)
+ }
+ u.Host = fmt.Sprintf("%d.%d.%d.%d%s", parts[0], parts[1], parts[2], parts[3], matches[5])
+ }
+ }
+}
+
+func decodeHexHost(u *url.URL) {
+ if len(u.Host) > 0 {
+ if matches := rxHexHost.FindStringSubmatch(u.Host); len(matches) > 2 {
+ // Conversion is safe because of regex validation
+ parsed, _ := strconv.ParseInt(matches[1], 16, 0)
+ // Set host as DWORD (base 10) encoded host
+ u.Host = fmt.Sprintf("%d%s", parsed, matches[2])
+ // The rest is the same as decoding a DWORD host
+ decodeDWORDHost(u)
+ }
+ }
+}
+
+func removeUnncessaryHostDots(u *url.URL) {
+ if len(u.Host) > 0 {
+ if matches := rxHostDots.FindStringSubmatch(u.Host); len(matches) > 1 {
+ // Trim the leading and trailing dots
+ u.Host = strings.Trim(matches[1], ".")
+ if len(matches) > 2 {
+ u.Host += matches[2]
+ }
+ }
+ }
+}
+
+func removeEmptyPortSeparator(u *url.URL) {
+ if len(u.Host) > 0 {
+ u.Host = rxEmptyPort.ReplaceAllString(u.Host, "")
+ }
+}
diff --git a/normalize/normalize_test.go b/normalize/normalize_test.go
new file mode 100644
index 0000000..177ae01
--- /dev/null
+++ b/normalize/normalize_test.go
@@ -0,0 +1,782 @@
+package normalize
+
+import (
+ "fmt"
+ "net/url"
+ "testing"
+ "unicode"
+)
+
+type testCase struct {
+ nm string
+ src string
+ flgs NormalizationFlags
+ res string
+ parsed bool
+}
+
+var (
+ cases = [...]*testCase{
+ {
+ "LowerScheme",
+ "HTTP://www.SRC.ca",
+ FlagLowercaseScheme,
+ "http://www.SRC.ca",
+ false,
+ },
+ {
+ "LowerScheme2",
+ "http://www.SRC.ca",
+ FlagLowercaseScheme,
+ "http://www.SRC.ca",
+ false,
+ },
+ {
+ "LowerHost",
+ "HTTP://www.SRC.ca/",
+ FlagLowercaseHost,
+ "http://www.src.ca/", // Since Go1.1, scheme is automatically lowercased
+ false,
+ },
+ {
+ "UpperEscapes",
+ `http://www.whatever.com/Some%aa%20Special%8Ecases/`,
+ FlagUppercaseEscapes,
+ "http://www.whatever.com/Some%AA%20Special%8Ecases/",
+ false,
+ },
+ {
+ "UnnecessaryEscapes",
+ `http://www.toto.com/%41%42%2E%44/%32%33%52%2D/%5f%7E`,
+ FlagDecodeUnnecessaryEscapes,
+ "http://www.toto.com/AB.D/23R-/_~",
+ false,
+ },
+ {
+ "RemoveDefaultPort",
+ "HTTP://www.SRC.ca:80/",
+ FlagRemoveDefaultPort,
+ "http://www.SRC.ca/", // Since Go1.1, scheme is automatically lowercased
+ false,
+ },
+ {
+ "RemoveDefaultPort2",
+ "HTTP://www.SRC.ca:80",
+ FlagRemoveDefaultPort,
+ "http://www.SRC.ca", // Since Go1.1, scheme is automatically lowercased
+ false,
+ },
+ {
+ "RemoveDefaultPort3",
+ "HTTP://www.SRC.ca:8080",
+ FlagRemoveDefaultPort,
+ "http://www.SRC.ca:8080", // Since Go1.1, scheme is automatically lowercased
+ false,
+ },
+ {
+ "Safe",
+ "HTTP://www.SRC.ca:80/to%1ato%8b%ee/OKnow%41%42%43%7e",
+ FlagsSafe,
+ "http://www.src.ca/to%1Ato%8B%EE/OKnowABC~",
+ false,
+ },
+ {
+ "BothLower",
+ "HTTP://www.SRC.ca:80/to%1ato%8b%ee/OKnow%41%42%43%7e",
+ FlagLowercaseHost | FlagLowercaseScheme,
+ "http://www.src.ca:80/to%1Ato%8B%EE/OKnowABC~",
+ false,
+ },
+ {
+ "RemoveTrailingSlash",
+ "HTTP://www.SRC.ca:80/",
+ FlagRemoveTrailingSlash,
+ "http://www.SRC.ca:80", // Since Go1.1, scheme is automatically lowercased
+ false,
+ },
+ {
+ "RemoveTrailingSlash2",
+ "HTTP://www.SRC.ca:80/toto/titi/",
+ FlagRemoveTrailingSlash,
+ "http://www.SRC.ca:80/toto/titi", // Since Go1.1, scheme is automatically lowercased
+ false,
+ },
+ {
+ "RemoveTrailingSlash3",
+ "HTTP://www.SRC.ca:80/toto/titi/fin/?a=1",
+ FlagRemoveTrailingSlash,
+ "http://www.SRC.ca:80/toto/titi/fin?a=1", // Since Go1.1, scheme is automatically lowercased
+ false,
+ },
+ {
+ "AddTrailingSlash",
+ "HTTP://www.SRC.ca:80",
+ FlagAddTrailingSlash,
+ "http://www.SRC.ca:80/", // Since Go1.1, scheme is automatically lowercased
+ false,
+ },
+ {
+ "AddTrailingSlash2",
+ "HTTP://www.SRC.ca:80/toto/titi.html",
+ FlagAddTrailingSlash,
+ "http://www.SRC.ca:80/toto/titi.html/", // Since Go1.1, scheme is automatically lowercased
+ false,
+ },
+ {
+ "AddTrailingSlash3",
+ "HTTP://www.SRC.ca:80/toto/titi/fin?a=1",
+ FlagAddTrailingSlash,
+ "http://www.SRC.ca:80/toto/titi/fin/?a=1", // Since Go1.1, scheme is automatically lowercased
+ false,
+ },
+ {
+ "RemoveDotSegments",
+ "HTTP://root/a/b/./../../c/",
+ FlagRemoveDotSegments,
+ "http://root/c/", // Since Go1.1, scheme is automatically lowercased
+ false,
+ },
+ {
+ "RemoveDotSegments2",
+ "HTTP://root/../a/b/./../c/../d",
+ FlagRemoveDotSegments,
+ "http://root/a/d", // Since Go1.1, scheme is automatically lowercased
+ false,
+ },
+ {
+ "UsuallySafe",
+ "HTTP://www.SRC.ca:80/to%1ato%8b%ee/./c/d/../OKnow%41%42%43%7e/?a=b#test",
+ FlagsUsuallySafeGreedy,
+ "http://www.src.ca/to%1Ato%8B%EE/c/OKnowABC~?a=b#test",
+ false,
+ },
+ {
+ "RemoveDirectoryIndex",
+ "HTTP://root/a/b/c/default.aspx",
+ FlagRemoveDirectoryIndex,
+ "http://root/a/b/c/", // Since Go1.1, scheme is automatically lowercased
+ false,
+ },
+ {
+ "RemoveDirectoryIndex2",
+ "HTTP://root/a/b/c/default#a=b",
+ FlagRemoveDirectoryIndex,
+ "http://root/a/b/c/default#a=b", // Since Go1.1, scheme is automatically lowercased
+ false,
+ },
+ {
+ "RemoveFragment",
+ "HTTP://root/a/b/c/default#toto=tata",
+ FlagRemoveFragment,
+ "http://root/a/b/c/default", // Since Go1.1, scheme is automatically lowercased
+ false,
+ },
+ {
+ "ForceHTTP",
+ "https://root/a/b/c/default#toto=tata",
+ FlagForceHTTP,
+ "http://root/a/b/c/default#toto=tata",
+ false,
+ },
+ {
+ "RemoveDuplicateSlashes",
+ "https://root/a//b///c////default#toto=tata",
+ FlagRemoveDuplicateSlashes,
+ "https://root/a/b/c/default#toto=tata",
+ false,
+ },
+ {
+ "RemoveDuplicateSlashes2",
+ "https://root//a//b///c////default#toto=tata",
+ FlagRemoveDuplicateSlashes,
+ "https://root/a/b/c/default#toto=tata",
+ false,
+ },
+ {
+ "RemoveWWW",
+ "https://www.root/a/b/c/",
+ FlagRemoveWWW,
+ "https://root/a/b/c/",
+ false,
+ },
+ {
+ "RemoveWWW2",
+ "https://WwW.Root/a/b/c/",
+ FlagRemoveWWW,
+ "https://Root/a/b/c/",
+ false,
+ },
+ {
+ "AddWWW",
+ "https://Root/a/b/c/",
+ FlagAddWWW,
+ "https://www.Root/a/b/c/",
+ false,
+ },
+ {
+ "SortQuery",
+ "http://root/toto/?b=4&a=1&c=3&b=2&a=5",
+ FlagSortQuery,
+ "http://root/toto/?a=1&a=5&b=2&b=4&c=3",
+ false,
+ },
+ {
+ "RemoveEmptyQuerySeparator",
+ "http://root/toto/?",
+ FlagRemoveEmptyQuerySeparator,
+ "http://root/toto/",
+ false,
+ },
+ {
+ "Unsafe",
+ "HTTPS://www.RooT.com/toto/t%45%1f///a/./b/../c/?z=3&w=2&a=4&w=1#invalid",
+ FlagsUnsafeGreedy,
+ "http://root.com/toto/tE%1F/a/c?a=4&w=1&w=2&z=3",
+ false,
+ },
+ {
+ "Safe2",
+ "HTTPS://www.RooT.com/toto/t%45%1f///a/./b/../c/?z=3&w=2&a=4&w=1#invalid",
+ FlagsSafe,
+ "https://www.root.com/toto/tE%1F///a/./b/../c/?z=3&w=2&a=4&w=1#invalid",
+ false,
+ },
+ {
+ "UsuallySafe2",
+ "HTTPS://www.RooT.com/toto/t%45%1f///a/./b/../c/?z=3&w=2&a=4&w=1#invalid",
+ FlagsUsuallySafeGreedy,
+ "https://www.root.com/toto/tE%1F///a/c?z=3&w=2&a=4&w=1#invalid",
+ false,
+ },
+ {
+ "AddTrailingSlashBug",
+ "http://src.ca/",
+ FlagsAllNonGreedy,
+ "http://www.src.ca/",
+ false,
+ },
+ {
+ "SourceModified",
+ "HTTPS://www.RooT.com/toto/t%45%1f///a/./b/../c/?z=3&w=2&a=4&w=1#invalid",
+ FlagsUnsafeGreedy,
+ "http://root.com/toto/tE%1F/a/c?a=4&w=1&w=2&z=3",
+ true,
+ },
+ {
+ "IPv6-1",
+ "http://[2001:db8:1f70::999:de8:7648:6e8]/test",
+ FlagsSafe | FlagRemoveDotSegments,
+ "http://[2001:db8:1f70::999:de8:7648:6e8]/test",
+ false,
+ },
+ {
+ "IPv6-2",
+ "http://[::ffff:192.168.1.1]/test",
+ FlagsSafe | FlagRemoveDotSegments,
+ "http://[::ffff:192.168.1.1]/test",
+ false,
+ },
+ {
+ "IPv6-3",
+ "http://[::ffff:192.168.1.1]:80/test",
+ FlagsSafe | FlagRemoveDotSegments,
+ "http://[::ffff:192.168.1.1]/test",
+ false,
+ },
+ {
+ "IPv6-4",
+ "htTps://[::fFff:192.168.1.1]:443/test",
+ FlagsSafe | FlagRemoveDotSegments,
+ "https://[::ffff:192.168.1.1]/test",
+ false,
+ },
+ {
+ "FTP",
+ "ftp://user:pass@ftp.foo.net/foo/bar",
+ FlagsSafe | FlagRemoveDotSegments,
+ "ftp://user:pass@ftp.foo.net/foo/bar",
+ false,
+ },
+ {
+ "Standard-1",
+ "http://www.foo.com:80/foo",
+ FlagsSafe | FlagRemoveDotSegments,
+ "http://www.foo.com/foo",
+ false,
+ },
+ {
+ "Standard-2",
+ "http://www.foo.com:8000/foo",
+ FlagsSafe | FlagRemoveDotSegments,
+ "http://www.foo.com:8000/foo",
+ false,
+ },
+ {
+ "Standard-3",
+ "http://www.foo.com/%7ebar",
+ FlagsSafe | FlagRemoveDotSegments,
+ "http://www.foo.com/~bar",
+ false,
+ },
+ {
+ "Standard-4",
+ "http://www.foo.com/%7Ebar",
+ FlagsSafe | FlagRemoveDotSegments,
+ "http://www.foo.com/~bar",
+ false,
+ },
+ {
+ "Standard-5",
+ "http://USER:pass@www.Example.COM/foo/bar",
+ FlagsSafe | FlagRemoveDotSegments,
+ "http://USER:pass@www.example.com/foo/bar",
+ false,
+ },
+ {
+ "Standard-6",
+ "http://test.example/?a=%26&b=1",
+ FlagsSafe | FlagRemoveDotSegments,
+ "http://test.example/?a=%26&b=1",
+ false,
+ },
+ {
+ "Standard-7",
+ "http://test.example/%25/?p=%20val%20%25",
+ FlagsSafe | FlagRemoveDotSegments,
+ "http://test.example/%25/?p=%20val%20%25",
+ false,
+ },
+ {
+ "Standard-8",
+ "http://test.example/path/with a%20space+/",
+ FlagsSafe | FlagRemoveDotSegments,
+ "http://test.example/path/with%20a%20space+/",
+ false,
+ },
+ {
+ "Standard-9",
+ "http://test.example/?",
+ FlagsSafe | FlagRemoveDotSegments,
+ "http://test.example/",
+ false,
+ },
+ {
+ "Standard-10",
+ "http://a.COM/path/?b&a",
+ FlagsSafe | FlagRemoveDotSegments,
+ "http://a.com/path/?b&a",
+ false,
+ },
+ {
+ "StandardCasesAddTrailingSlash",
+ "http://test.example?",
+ FlagsSafe | FlagAddTrailingSlash,
+ "http://test.example/",
+ false,
+ },
+ {
+ "OctalIP-1",
+ "http://0123.011.0.4/",
+ FlagsSafe | FlagDecodeOctalHost,
+ "http://0123.011.0.4/",
+ false,
+ },
+ {
+ "OctalIP-2",
+ "http://0102.0146.07.0223/",
+ FlagsSafe | FlagDecodeOctalHost,
+ "http://66.102.7.147/",
+ false,
+ },
+ {
+ "OctalIP-3",
+ "http://0102.0146.07.0223.:23/",
+ FlagsSafe | FlagDecodeOctalHost,
+ "http://66.102.7.147.:23/",
+ false,
+ },
+ {
+ "OctalIP-4",
+ "http://USER:pass@0102.0146.07.0223../",
+ FlagsSafe | FlagDecodeOctalHost,
+ "http://USER:pass@66.102.7.147../",
+ false,
+ },
+ {
+ "DWORDIP-1",
+ "http://123.1113982867/",
+ FlagsSafe | FlagDecodeDWORDHost,
+ "http://123.1113982867/",
+ false,
+ },
+ {
+ "DWORDIP-2",
+ "http://1113982867/",
+ FlagsSafe | FlagDecodeDWORDHost,
+ "http://66.102.7.147/",
+ false,
+ },
+ {
+ "DWORDIP-3",
+ "http://1113982867.:23/",
+ FlagsSafe | FlagDecodeDWORDHost,
+ "http://66.102.7.147.:23/",
+ false,
+ },
+ {
+ "DWORDIP-4",
+ "http://USER:pass@1113982867../",
+ FlagsSafe | FlagDecodeDWORDHost,
+ "http://USER:pass@66.102.7.147../",
+ false,
+ },
+ {
+ "HexIP-1",
+ "http://0x123.1113982867/",
+ FlagsSafe | FlagDecodeHexHost,
+ "http://0x123.1113982867/",
+ false,
+ },
+ {
+ "HexIP-2",
+ "http://0x42660793/",
+ FlagsSafe | FlagDecodeHexHost,
+ "http://66.102.7.147/",
+ false,
+ },
+ {
+ "HexIP-3",
+ "http://0x42660793.:23/",
+ FlagsSafe | FlagDecodeHexHost,
+ "http://66.102.7.147.:23/",
+ false,
+ },
+ {
+ "HexIP-4",
+ "http://USER:pass@0x42660793../",
+ FlagsSafe | FlagDecodeHexHost,
+ "http://USER:pass@66.102.7.147../",
+ false,
+ },
+ {
+ "UnnecessaryHostDots-1",
+ "http://.www.foo.com../foo/bar.html",
+ FlagsSafe | FlagRemoveUnnecessaryHostDots,
+ "http://www.foo.com/foo/bar.html",
+ false,
+ },
+ {
+ "UnnecessaryHostDots-2",
+ "http://www.foo.com./foo/bar.html",
+ FlagsSafe | FlagRemoveUnnecessaryHostDots,
+ "http://www.foo.com/foo/bar.html",
+ false,
+ },
+ {
+ "UnnecessaryHostDots-3",
+ "http://www.foo.com.:81/foo",
+ FlagsSafe | FlagRemoveUnnecessaryHostDots,
+ "http://www.foo.com:81/foo",
+ false,
+ },
+ {
+ "UnnecessaryHostDots-4",
+ "http://www.example.com./",
+ FlagsSafe | FlagRemoveUnnecessaryHostDots,
+ "http://www.example.com/",
+ false,
+ },
+ {
+ "EmptyPort-1",
+ "http://www.thedraymin.co.uk:/main/?p=308",
+ FlagsSafe | FlagRemoveEmptyPortSeparator,
+ "http://www.thedraymin.co.uk/main/?p=308",
+ false,
+ },
+ {
+ "EmptyPort-2",
+ "http://www.src.ca:",
+ FlagsSafe | FlagRemoveEmptyPortSeparator,
+ "http://www.src.ca",
+ false,
+ },
+ {
+ "Slashes-1",
+ "http://test.example/foo/bar/.",
+ FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+ "http://test.example/foo/bar/",
+ false,
+ },
+ {
+ "Slashes-2",
+ "http://test.example/foo/bar/./",
+ FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+ "http://test.example/foo/bar/",
+ false,
+ },
+ {
+ "Slashes-3",
+ "http://test.example/foo/bar/..",
+ FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+ "http://test.example/foo/",
+ false,
+ },
+ {
+ "Slashes-4",
+ "http://test.example/foo/bar/../",
+ FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+ "http://test.example/foo/",
+ false,
+ },
+ {
+ "Slashes-5",
+ "http://test.example/foo/bar/../baz",
+ FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+ "http://test.example/foo/baz",
+ false,
+ },
+ {
+ "Slashes-6",
+ "http://test.example/foo/bar/../..",
+ FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+ "http://test.example/",
+ false,
+ },
+ {
+ "Slashes-7",
+ "http://test.example/foo/bar/../../",
+ FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+ "http://test.example/",
+ false,
+ },
+ {
+ "Slashes-8",
+ "http://test.example/foo/bar/../../baz",
+ FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+ "http://test.example/baz",
+ false,
+ },
+ {
+ "Slashes-9",
+ "http://test.example/foo/bar/../../../baz",
+ FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+ "http://test.example/baz",
+ false,
+ },
+ {
+ "Slashes-10",
+ "http://test.example/foo/bar/../../../../baz",
+ FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+ "http://test.example/baz",
+ false,
+ },
+ {
+ "Slashes-11",
+ "http://test.example/./foo",
+ FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+ "http://test.example/foo",
+ false,
+ },
+ {
+ "Slashes-12",
+ "http://test.example/../foo",
+ FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+ "http://test.example/foo",
+ false,
+ },
+ {
+ "Slashes-13",
+ "http://test.example/foo.",
+ FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+ "http://test.example/foo.",
+ false,
+ },
+ {
+ "Slashes-14",
+ "http://test.example/.foo",
+ FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+ "http://test.example/.foo",
+ false,
+ },
+ {
+ "Slashes-15",
+ "http://test.example/foo..",
+ FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+ "http://test.example/foo..",
+ false,
+ },
+ {
+ "Slashes-16",
+ "http://test.example/..foo",
+ FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+ "http://test.example/..foo",
+ false,
+ },
+ {
+ "Slashes-17",
+ "http://test.example/./../foo",
+ FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+ "http://test.example/foo",
+ false,
+ },
+ {
+ "Slashes-18",
+ "http://test.example/./foo/.",
+ FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+ "http://test.example/foo/",
+ false,
+ },
+ {
+ "Slashes-19",
+ "http://test.example/foo/./bar",
+ FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+ "http://test.example/foo/bar",
+ false,
+ },
+ {
+ "Slashes-20",
+ "http://test.example/foo/../bar",
+ FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+ "http://test.example/bar",
+ false,
+ },
+ {
+ "Slashes-21",
+ "http://test.example/foo//",
+ FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+ "http://test.example/foo/",
+ false,
+ },
+ {
+ "Slashes-22",
+ "http://test.example/foo///bar//",
+ FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+ "http://test.example/foo/bar/",
+ false,
+ },
+ {
+ "Relative",
+ "foo/bar",
+ FlagsAllGreedy,
+ "foo/bar",
+ false,
+ },
+ {
+ "Relative-1",
+ "./../foo",
+ FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+ "foo",
+ false,
+ },
+ {
+ "Relative-2",
+ "./foo/bar/../baz/../bang/..",
+ FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+ "foo/",
+ false,
+ },
+ {
+ "Relative-3",
+ "foo///bar//",
+ FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+ "foo/bar/",
+ false,
+ },
+ {
+ "Relative-4",
+ "www.youtube.com",
+ FlagsUsuallySafeGreedy,
+ "www.youtube.com",
+ false,
+ },
+ {
+ "Issue-#24",
+ "///foo///bar///",
+ FlagRemoveDuplicateSlashes | FlagRemoveTrailingSlash,
+ "/foo/bar",
+ false,
+ },
+ /*&testCase{
+ "UrlNorm-5",
+ "http://ja.wikipedia.org/wiki/%E3%82%AD%E3%83%A3%E3%82%BF%E3%83%94%E3%83%A9%E3%83%BC%E3%82%B8%E3%83%A3%E3%83%91%E3%83%B3",
+ FlagsSafe | FlagRemoveDotSegments,
+ "http://ja.wikipedia.org/wiki/\xe3\x82\xad\xe3\x83\xa3\xe3\x82\xbf\xe3\x83\x94\xe3\x83\xa9\xe3\x83\xbc\xe3\x82\xb8\xe3\x83\xa3\xe3\x83\x91\xe3\x83\xb3",
+ false,
+ },
+ &testCase{
+ "UrlNorm-1",
+ "http://test.example/?a=%e3%82%82%26",
+ FlagsAllGreedy,
+ "http://test.example/?a=\xe3\x82\x82%26",
+ false,
+ },*/
+ }
+)
+
+func TestRunner(t *testing.T) {
+ for _, tc := range cases {
+ runCase(tc, t)
+ }
+}
+
+func runCase(tc *testCase, t *testing.T) {
+ t.Logf("running %s...", tc.nm)
+ if tc.parsed {
+ u, e := url.Parse(tc.src)
+ if e != nil {
+ t.Errorf("%s - FAIL : %s", tc.nm, e)
+ return
+ } else {
+ NormalizeURL(u, tc.flgs)
+ if s := u.String(); s != tc.res {
+ t.Errorf("%s - FAIL expected '%s', got '%s'", tc.nm, tc.res, s)
+ }
+ }
+ } else {
+ if s, e := NormalizeURLString(tc.src, tc.flgs); e != nil {
+ t.Errorf("%s - FAIL : %s", tc.nm, e)
+ } else if s != tc.res {
+ t.Errorf("%s - FAIL expected '%s', got '%s'", tc.nm, tc.res, s)
+ }
+ }
+}
+
+func TestDecodeUnnecessaryEscapesAll(t *testing.T) {
+ var url = "http://host/"
+
+ for i := 0; i < 256; i++ {
+ url += fmt.Sprintf("%%%02x", i)
+ }
+ s, err := NormalizeURLString(url, FlagDecodeUnnecessaryEscapes)
+ if err != nil {
+ t.Fatalf("parse error: %s", err)
+ }
+
+ const want = "http://host/%00%01%02%03%04%05%06%07%08%09%0A%0B%0C%0D%0E%0F%10%11%12%13%14%15%16%17%18%19%1A%1B%1C%1D%1E%1F%20!%22%23$%25&'()*+,-./0123456789:;%3C=%3E%3F@ABCDEFGHIJKLMNOPQRSTUVWXYZ[%5C]%5E_%60abcdefghijklmnopqrstuvwxyz%7B%7C%7D~%7F%80%81%82%83%84%85%86%87%88%89%8A%8B%8C%8D%8E%8F%90%91%92%93%94%95%96%97%98%99%9A%9B%9C%9D%9E%9F%A0%A1%A2%A3%A4%A5%A6%A7%A8%A9%AA%AB%AC%AD%AE%AF%B0%B1%B2%B3%B4%B5%B6%B7%B8%B9%BA%BB%BC%BD%BE%BF%C0%C1%C2%C3%C4%C5%C6%C7%C8%C9%CA%CB%CC%CD%CE%CF%D0%D1%D2%D3%D4%D5%D6%D7%D8%D9%DA%DB%DC%DD%DE%DF%E0%E1%E2%E3%E4%E5%E6%E7%E8%E9%EA%EB%EC%ED%EE%EF%F0%F1%F2%F3%F4%F5%F6%F7%F8%F9%FA%FB%FC%FD%FE%FF"
+ if s != want {
+ t.Errorf("DecodeUnnecessaryEscapesAll:\nwant\n%s\ngot\n%s", want, s)
+ }
+}
+
+func TestEncodeNecessaryEscapesAll(t *testing.T) {
+ const base = "http://host/"
+ var path []byte
+
+ for i := 0; i < 256; i++ {
+ // Since go1.12, url.Parse fails if the raw URL contains ASCII control characters,
+ // meaning anything < 0x20 and 0x7f (DEL), so do not add those bytes to the constructed url.
+ // See https://github.com/PuerkitoBio/purell/issues/28
+ if i != 0x25 && !unicode.IsControl(rune(i)) {
+ path = append(path, byte(i))
+ }
+ }
+ s, err := NormalizeURLString(base+string(path), FlagEncodeNecessaryEscapes)
+ if err != nil {
+ t.Fatalf("parse error: %s", err)
+ }
+
+ const want = "http://host/%20!%22#$&'()*+,-./0123456789:;%3C=%3E?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[%5C]%5E_%60abcdefghijklmnopqrstuvwxyz%7B%7C%7D~%A0%A1%A2%A3%A4%A5%A6%A7%A8%A9%AA%AB%AC%AD%AE%AF%B0%B1%B2%B3%B4%B5%B6%B7%B8%B9%BA%BB%BC%BD%BE%BF%C0%C1%C2%C3%C4%C5%C6%C7%C8%C9%CA%CB%CC%CD%CE%CF%D0%D1%D2%D3%D4%D5%D6%D7%D8%D9%DA%DB%DC%DD%DE%DF%E0%E1%E2%E3%E4%E5%E6%E7%E8%E9%EA%EB%EC%ED%EE%EF%F0%F1%F2%F3%F4%F5%F6%F7%F8%F9%FA%FB%FC%FD%FE%FF"
+ if s != want {
+ t.Errorf("EncodeNecessaryEscapesAll:\nwant\n%s\ngot\n%s", want, s)
+ }
+}
diff --git a/urlesc/LICENSE b/urlesc/LICENSE
new file mode 100644
index 0000000..7448756
--- /dev/null
+++ b/urlesc/LICENSE
@@ -0,0 +1,27 @@
+Copyright (c) 2012 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/urlesc/urlesc.go b/urlesc/urlesc.go
new file mode 100644
index 0000000..1b84624
--- /dev/null
+++ b/urlesc/urlesc.go
@@ -0,0 +1,180 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package urlesc implements query escaping as per RFC 3986.
+// It contains some parts of the net/url package, modified so as to allow
+// some reserved characters incorrectly escaped by net/url.
+// See https://github.com/golang/go/issues/5684
+package urlesc
+
+import (
+ "bytes"
+ "net/url"
+ "strings"
+)
+
+type encoding int
+
+const (
+ encodePath encoding = 1 + iota
+ encodeUserPassword
+ encodeQueryComponent
+ encodeFragment
+)
+
+// Return true if the specified character should be escaped when
+// appearing in a URL string, according to RFC 3986.
+func shouldEscape(c byte, mode encoding) bool {
+ // §2.3 Unreserved characters (alphanum)
+ if 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' {
+ return false
+ }
+
+ switch c {
+ case '-', '.', '_', '~': // §2.3 Unreserved characters (mark)
+ return false
+
+ // §2.2 Reserved characters (reserved)
+ case ':', '/', '?', '#', '[', ']', '@', // gen-delims
+ '!', '$', '&', '\'', '(', ')', '*', '+', ',', ';', '=': // sub-delims
+ // Different sections of the URL allow a few of
+ // the reserved characters to appear unescaped.
+ switch mode {
+ case encodePath: // §3.3
+ // The RFC allows sub-delims and : @.
+ // '/', '[' and ']' can be used to assign meaning to individual path
+ // segments. This package only manipulates the path as a whole,
+ // so we allow those as well. That leaves only ? and # to escape.
+ return c == '?' || c == '#'
+
+ case encodeUserPassword: // §3.2.1
+ // The RFC allows : and sub-delims in
+ // userinfo. The parsing of userinfo treats ':' as special so we must escape
+ // all the gen-delims.
+ return c == ':' || c == '/' || c == '?' || c == '#' || c == '[' || c == ']' || c == '@'
+
+ case encodeQueryComponent: // §3.4
+ // The RFC allows / and ?.
+ return c != '/' && c != '?'
+
+ case encodeFragment: // §4.1
+ // The RFC text is silent but the grammar allows
+ // everything, so escape nothing but #
+ return c == '#'
+ }
+ }
+
+ // Everything else must be escaped.
+ return true
+}
+
+// QueryEscape escapes the string so it can be safely placed
+// inside a URL query.
+func QueryEscape(s string) string {
+ return escape(s, encodeQueryComponent)
+}
+
+func escape(s string, mode encoding) string {
+ spaceCount, hexCount := 0, 0
+ for i := 0; i < len(s); i++ {
+ c := s[i]
+ if shouldEscape(c, mode) {
+ if c == ' ' && mode == encodeQueryComponent {
+ spaceCount++
+ } else {
+ hexCount++
+ }
+ }
+ }
+
+ if spaceCount == 0 && hexCount == 0 {
+ return s
+ }
+
+ t := make([]byte, len(s)+2*hexCount)
+ j := 0
+ for i := 0; i < len(s); i++ {
+ switch c := s[i]; {
+ case c == ' ' && mode == encodeQueryComponent:
+ t[j] = '+'
+ j++
+ case shouldEscape(c, mode):
+ t[j] = '%'
+ t[j+1] = "0123456789ABCDEF"[c>>4]
+ t[j+2] = "0123456789ABCDEF"[c&15]
+ j += 3
+ default:
+ t[j] = s[i]
+ j++
+ }
+ }
+ return string(t)
+}
+
+var uiReplacer = strings.NewReplacer(
+ "%21", "!",
+ "%27", "'",
+ "%28", "(",
+ "%29", ")",
+ "%2A", "*",
+)
+
+// unescapeUserinfo unescapes some characters that need not to be escaped as per RFC3986.
+func unescapeUserinfo(s string) string {
+ return uiReplacer.Replace(s)
+}
+
+// Escape reassembles the URL into a valid URL string.
+// The general form of the result is one of:
+//
+// scheme:opaque
+// scheme://userinfo@host/path?query#fragment
+//
+// If u.Opaque is non-empty, String uses the first form;
+// otherwise it uses the second form.
+//
+// In the second form, the following rules apply:
+// - if u.Scheme is empty, scheme: is omitted.
+// - if u.User is nil, userinfo@ is omitted.
+// - if u.Host is empty, host/ is omitted.
+// - if u.Scheme and u.Host are empty and u.User is nil,
+// the entire scheme://userinfo@host/ is omitted.
+// - if u.Host is non-empty and u.Path begins with a /,
+// the form host/path does not add its own /.
+// - if u.RawQuery is empty, ?query is omitted.
+// - if u.Fragment is empty, #fragment is omitted.
+func Escape(u *url.URL) string {
+ var buf bytes.Buffer
+ if u.Scheme != "" {
+ buf.WriteString(u.Scheme)
+ buf.WriteByte(':')
+ }
+ if u.Opaque != "" {
+ buf.WriteString(u.Opaque)
+ } else {
+ if u.Scheme != "" || u.Host != "" || u.User != nil {
+ buf.WriteString("//")
+ if ui := u.User; ui != nil {
+ buf.WriteString(unescapeUserinfo(ui.String()))
+ buf.WriteByte('@')
+ }
+ if h := u.Host; h != "" {
+ buf.WriteString(h)
+ }
+ }
+ if u.Path != "" && u.Path[0] != '/' && u.Host != "" {
+ buf.WriteByte('/')
+ }
+ buf.WriteString(escape(u.Path, encodePath))
+ }
+ if u.RawQuery != "" {
+ buf.WriteByte('?')
+ buf.WriteString(u.RawQuery)
+ }
+ if u.Fragment != "" {
+ buf.WriteByte('#')
+ buf.WriteString(escape(u.Fragment, encodeFragment))
+ }
+ return buf.String()
+}
diff --git a/urlesc/urlesc_test.go b/urlesc/urlesc_test.go
new file mode 100644
index 0000000..45202e1
--- /dev/null
+++ b/urlesc/urlesc_test.go
@@ -0,0 +1,641 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package urlesc
+
+import (
+ "net/url"
+ "testing"
+)
+
+type URLTest struct {
+ in string
+ out *url.URL
+ roundtrip string // expected result of reserializing the URL; empty means same as "in".
+}
+
+var urltests = []URLTest{
+ // no path
+ {
+ "http://www.google.com",
+ &url.URL{
+ Scheme: "http",
+ Host: "www.google.com",
+ },
+ "",
+ },
+ // path
+ {
+ "http://www.google.com/",
+ &url.URL{
+ Scheme: "http",
+ Host: "www.google.com",
+ Path: "/",
+ },
+ "",
+ },
+ // path with hex escaping
+ {
+ "http://www.google.com/file%20one%26two",
+ &url.URL{
+ Scheme: "http",
+ Host: "www.google.com",
+ Path: "/file one&two",
+ },
+ "http://www.google.com/file%20one&two",
+ },
+ // user
+ {
+ "ftp://webmaster@www.google.com/",
+ &url.URL{
+ Scheme: "ftp",
+ User: url.User("webmaster"),
+ Host: "www.google.com",
+ Path: "/",
+ },
+ "",
+ },
+ // escape sequence in username
+ {
+ "ftp://john%20doe@www.google.com/",
+ &url.URL{
+ Scheme: "ftp",
+ User: url.User("john doe"),
+ Host: "www.google.com",
+ Path: "/",
+ },
+ "ftp://john%20doe@www.google.com/",
+ },
+ // query
+ {
+ "http://www.google.com/?q=go+language",
+ &url.URL{
+ Scheme: "http",
+ Host: "www.google.com",
+ Path: "/",
+ RawQuery: "q=go+language",
+ },
+ "",
+ },
+ // query with hex escaping: NOT parsed
+ {
+ "http://www.google.com/?q=go%20language",
+ &url.URL{
+ Scheme: "http",
+ Host: "www.google.com",
+ Path: "/",
+ RawQuery: "q=go%20language",
+ },
+ "",
+ },
+ // %20 outside query
+ {
+ "http://www.google.com/a%20b?q=c+d",
+ &url.URL{
+ Scheme: "http",
+ Host: "www.google.com",
+ Path: "/a b",
+ RawQuery: "q=c+d",
+ },
+ "",
+ },
+ // path without leading /, so no parsing
+ {
+ "http:www.google.com/?q=go+language",
+ &url.URL{
+ Scheme: "http",
+ Opaque: "www.google.com/",
+ RawQuery: "q=go+language",
+ },
+ "http:www.google.com/?q=go+language",
+ },
+ // path without leading /, so no parsing
+ {
+ "http:%2f%2fwww.google.com/?q=go+language",
+ &url.URL{
+ Scheme: "http",
+ Opaque: "%2f%2fwww.google.com/",
+ RawQuery: "q=go+language",
+ },
+ "http:%2f%2fwww.google.com/?q=go+language",
+ },
+ // non-authority with path
+ {
+ "mailto:/webmaster@golang.org",
+ &url.URL{
+ Scheme: "mailto",
+ Path: "/webmaster@golang.org",
+ },
+ "mailto:///webmaster@golang.org", // unfortunate compromise
+ },
+ // non-authority
+ {
+ "mailto:webmaster@golang.org",
+ &url.URL{
+ Scheme: "mailto",
+ Opaque: "webmaster@golang.org",
+ },
+ "",
+ },
+ // unescaped :// in query should not create a scheme
+ {
+ "/foo?query=http://bad",
+ &url.URL{
+ Path: "/foo",
+ RawQuery: "query=http://bad",
+ },
+ "",
+ },
+ // leading // without scheme should create an authority
+ {
+ "//foo",
+ &url.URL{
+ Host: "foo",
+ },
+ "",
+ },
+ // leading // without scheme, with userinfo, path, and query
+ {
+ "//user@foo/path?a=b",
+ &url.URL{
+ User: url.User("user"),
+ Host: "foo",
+ Path: "/path",
+ RawQuery: "a=b",
+ },
+ "",
+ },
+ // Three leading slashes isn't an authority, but doesn't return an error.
+ // (We can't return an error, as this code is also used via
+ // ServeHTTP -> ReadRequest -> Parse, which is arguably a
+ // different URL parsing context, but currently shares the
+ // same codepath)
+ {
+ "///threeslashes",
+ &url.URL{
+ Path: "///threeslashes",
+ },
+ "",
+ },
+ {
+ "http://user:password@google.com",
+ &url.URL{
+ Scheme: "http",
+ User: url.UserPassword("user", "password"),
+ Host: "google.com",
+ },
+ "http://user:password@google.com",
+ },
+ // unescaped @ in username should not confuse host
+ {
+ "http://j@ne:password@google.com",
+ &url.URL{
+ Scheme: "http",
+ User: url.UserPassword("j@ne", "password"),
+ Host: "google.com",
+ },
+ "http://j%40ne:password@google.com",
+ },
+ // unescaped @ in password should not confuse host
+ {
+ "http://jane:p@ssword@google.com",
+ &url.URL{
+ Scheme: "http",
+ User: url.UserPassword("jane", "p@ssword"),
+ Host: "google.com",
+ },
+ "http://jane:p%40ssword@google.com",
+ },
+ {
+ "http://j@ne:password@google.com/p@th?q=@go",
+ &url.URL{
+ Scheme: "http",
+ User: url.UserPassword("j@ne", "password"),
+ Host: "google.com",
+ Path: "/p@th",
+ RawQuery: "q=@go",
+ },
+ "http://j%40ne:password@google.com/p@th?q=@go",
+ },
+ {
+ "http://www.google.com/?q=go+language#foo",
+ &url.URL{
+ Scheme: "http",
+ Host: "www.google.com",
+ Path: "/",
+ RawQuery: "q=go+language",
+ Fragment: "foo",
+ },
+ "",
+ },
+ {
+ "http://www.google.com/?q=go+language#foo%26bar",
+ &url.URL{
+ Scheme: "http",
+ Host: "www.google.com",
+ Path: "/",
+ RawQuery: "q=go+language",
+ Fragment: "foo&bar",
+ },
+ "http://www.google.com/?q=go+language#foo&bar",
+ },
+ {
+ "file:///home/adg/rabbits",
+ &url.URL{
+ Scheme: "file",
+ Host: "",
+ Path: "/home/adg/rabbits",
+ },
+ "file:///home/adg/rabbits",
+ },
+ // "Windows" paths are no exception to the rule.
+ // See golang.org/issue/6027, especially comment #9.
+ {
+ "file:///C:/FooBar/Baz.txt",
+ &url.URL{
+ Scheme: "file",
+ Host: "",
+ Path: "/C:/FooBar/Baz.txt",
+ },
+ "file:///C:/FooBar/Baz.txt",
+ },
+ // case-insensitive scheme
+ {
+ "MaIlTo:webmaster@golang.org",
+ &url.URL{
+ Scheme: "mailto",
+ Opaque: "webmaster@golang.org",
+ },
+ "mailto:webmaster@golang.org",
+ },
+ // Relative path
+ {
+ "a/b/c",
+ &url.URL{
+ Path: "a/b/c",
+ },
+ "a/b/c",
+ },
+ // escaped '?' in username and password
+ {
+ "http://%3Fam:pa%3Fsword@google.com",
+ &url.URL{
+ Scheme: "http",
+ User: url.UserPassword("?am", "pa?sword"),
+ Host: "google.com",
+ },
+ "",
+ },
+ // escaped '?' and '#' in path
+ {
+ "http://example.com/%3F%23",
+ &url.URL{
+ Scheme: "http",
+ Host: "example.com",
+ Path: "?#",
+ },
+ "",
+ },
+ // unescaped [ ] ! ' ( ) * in path
+ {
+ "http://example.com/[]!'()*",
+ &url.URL{
+ Scheme: "http",
+ Host: "example.com",
+ Path: "[]!'()*",
+ },
+ "http://example.com/[]!'()*",
+ },
+ // escaped : / ? # [ ] @ in username and password
+ {
+ "http://%3A%2F%3F:%23%5B%5D%40@example.com",
+ &url.URL{
+ Scheme: "http",
+ User: url.UserPassword(":/?", "#[]@"),
+ Host: "example.com",
+ },
+ "",
+ },
+ // unescaped ! $ & ' ( ) * + , ; = in username and password
+ {
+ "http://!$&'():*+,;=@example.com",
+ &url.URL{
+ Scheme: "http",
+ User: url.UserPassword("!$&'()", "*+,;="),
+ Host: "example.com",
+ },
+ "",
+ },
+ // unescaped = : / . ? = in query component
+ {
+ "http://example.com/?q=http://google.com/?q=",
+ &url.URL{
+ Scheme: "http",
+ Host: "example.com",
+ Path: "/",
+ RawQuery: "q=http://google.com/?q=",
+ },
+ "",
+ },
+ // unescaped : / ? [ ] @ ! $ & ' ( ) * + , ; = in fragment
+ {
+ "http://example.com/#:/?%23[]@!$&'()*+,;=",
+ &url.URL{
+ Scheme: "http",
+ Host: "example.com",
+ Path: "/",
+ Fragment: ":/?#[]@!$&'()*+,;=",
+ },
+ "",
+ },
+}
+
+func DoTestString(t *testing.T, parse func(string) (*url.URL, error), name string, tests []URLTest) {
+ for _, tt := range tests {
+ u, err := parse(tt.in)
+ if err != nil {
+ t.Errorf("%s(%q) returned error %s", name, tt.in, err)
+ continue
+ }
+ expected := tt.in
+ if len(tt.roundtrip) > 0 {
+ expected = tt.roundtrip
+ }
+ s := Escape(u)
+ if s != expected {
+ t.Errorf("Escape(%s(%q)) == %q (expected %q)", name, tt.in, s, expected)
+ }
+ }
+}
+
+func TestURLString(t *testing.T) {
+ DoTestString(t, url.Parse, "Parse", urltests)
+
+ // no leading slash on path should prepend
+ // slash on String() call
+ noslash := URLTest{
+ "http://www.google.com/search",
+ &url.URL{
+ Scheme: "http",
+ Host: "www.google.com",
+ Path: "search",
+ },
+ "",
+ }
+ s := Escape(noslash.out)
+ if s != noslash.in {
+ t.Errorf("Expected %s; go %s", noslash.in, s)
+ }
+}
+
+type EscapeTest struct {
+ in string
+ out string
+ err error
+}
+
+var escapeTests = []EscapeTest{
+ {
+ "",
+ "",
+ nil,
+ },
+ {
+ "abc",
+ "abc",
+ nil,
+ },
+ {
+ "one two",
+ "one+two",
+ nil,
+ },
+ {
+ "10%",
+ "10%25",
+ nil,
+ },
+ {
+ " ?&=#+%!<>#\"{}|\\^[]`☺\t:/@$'()*,;",
+ "+?%26%3D%23%2B%25%21%3C%3E%23%22%7B%7D%7C%5C%5E%5B%5D%60%E2%98%BA%09%3A/%40%24%27%28%29%2A%2C%3B",
+ nil,
+ },
+}
+
+func TestEscape(t *testing.T) {
+ for _, tt := range escapeTests {
+ actual := QueryEscape(tt.in)
+ if tt.out != actual {
+ t.Errorf("QueryEscape(%q) = %q, want %q", tt.in, actual, tt.out)
+ }
+
+ // for bonus points, verify that escape:unescape is an identity.
+ roundtrip, err := url.QueryUnescape(actual)
+ if roundtrip != tt.in || err != nil {
+ t.Errorf("QueryUnescape(%q) = %q, %s; want %q, %s", actual, roundtrip, err, tt.in, "[no error]")
+ }
+ }
+}
+
+var resolveReferenceTests = []struct {
+ base, rel, expected string
+}{
+ // Absolute URL references
+ {"http://foo.com?a=b", "https://bar.com/", "https://bar.com/"},
+ {"http://foo.com/", "https://bar.com/?a=b", "https://bar.com/?a=b"},
+ {"http://foo.com/bar", "mailto:foo@example.com", "mailto:foo@example.com"},
+
+ // Path-absolute references
+ {"http://foo.com/bar", "/baz", "http://foo.com/baz"},
+ {"http://foo.com/bar?a=b#f", "/baz", "http://foo.com/baz"},
+ {"http://foo.com/bar?a=b", "/baz?c=d", "http://foo.com/baz?c=d"},
+
+ // Scheme-relative
+ {"https://foo.com/bar?a=b", "//bar.com/quux", "https://bar.com/quux"},
+
+ // Path-relative references:
+
+ // ... current directory
+ {"http://foo.com", ".", "http://foo.com/"},
+ {"http://foo.com/bar", ".", "http://foo.com/"},
+ {"http://foo.com/bar/", ".", "http://foo.com/bar/"},
+
+ // ... going down
+ {"http://foo.com", "bar", "http://foo.com/bar"},
+ {"http://foo.com/", "bar", "http://foo.com/bar"},
+ {"http://foo.com/bar/baz", "quux", "http://foo.com/bar/quux"},
+
+ // ... going up
+ {"http://foo.com/bar/baz", "../quux", "http://foo.com/quux"},
+ {"http://foo.com/bar/baz", "../../../../../quux", "http://foo.com/quux"},
+ {"http://foo.com/bar", "..", "http://foo.com/"},
+ {"http://foo.com/bar/baz", "./..", "http://foo.com/"},
+ // ".." in the middle (issue 3560)
+ {"http://foo.com/bar/baz", "quux/dotdot/../tail", "http://foo.com/bar/quux/tail"},
+ {"http://foo.com/bar/baz", "quux/./dotdot/../tail", "http://foo.com/bar/quux/tail"},
+ {"http://foo.com/bar/baz", "quux/./dotdot/.././tail", "http://foo.com/bar/quux/tail"},
+ {"http://foo.com/bar/baz", "quux/./dotdot/./../tail", "http://foo.com/bar/quux/tail"},
+ {"http://foo.com/bar/baz", "quux/./dotdot/dotdot/././../../tail", "http://foo.com/bar/quux/tail"},
+ {"http://foo.com/bar/baz", "quux/./dotdot/dotdot/./.././../tail", "http://foo.com/bar/quux/tail"},
+ {"http://foo.com/bar/baz", "quux/./dotdot/dotdot/dotdot/./../../.././././tail", "http://foo.com/bar/quux/tail"},
+ {"http://foo.com/bar/baz", "quux/./dotdot/../dotdot/../dot/./tail/..", "http://foo.com/bar/quux/dot/"},
+
+ // Remove any dot-segments prior to forming the target URI.
+ // http://tools.ietf.org/html/rfc3986#section-5.2.4
+ {"http://foo.com/dot/./dotdot/../foo/bar", "../baz", "http://foo.com/dot/baz"},
+
+ // Triple dot isn't special
+ {"http://foo.com/bar", "...", "http://foo.com/..."},
+
+ // Fragment
+ {"http://foo.com/bar", ".#frag", "http://foo.com/#frag"},
+
+ // RFC 3986: Normal Examples
+ // http://tools.ietf.org/html/rfc3986#section-5.4.1
+ {"http://a/b/c/d;p?q", "g:h", "g:h"},
+ {"http://a/b/c/d;p?q", "g", "http://a/b/c/g"},
+ {"http://a/b/c/d;p?q", "./g", "http://a/b/c/g"},
+ {"http://a/b/c/d;p?q", "g/", "http://a/b/c/g/"},
+ {"http://a/b/c/d;p?q", "/g", "http://a/g"},
+ {"http://a/b/c/d;p?q", "//g", "http://g"},
+ {"http://a/b/c/d;p?q", "?y", "http://a/b/c/d;p?y"},
+ {"http://a/b/c/d;p?q", "g?y", "http://a/b/c/g?y"},
+ {"http://a/b/c/d;p?q", "#s", "http://a/b/c/d;p?q#s"},
+ {"http://a/b/c/d;p?q", "g#s", "http://a/b/c/g#s"},
+ {"http://a/b/c/d;p?q", "g?y#s", "http://a/b/c/g?y#s"},
+ {"http://a/b/c/d;p?q", ";x", "http://a/b/c/;x"},
+ {"http://a/b/c/d;p?q", "g;x", "http://a/b/c/g;x"},
+ {"http://a/b/c/d;p?q", "g;x?y#s", "http://a/b/c/g;x?y#s"},
+ {"http://a/b/c/d;p?q", "", "http://a/b/c/d;p?q"},
+ {"http://a/b/c/d;p?q", ".", "http://a/b/c/"},
+ {"http://a/b/c/d;p?q", "./", "http://a/b/c/"},
+ {"http://a/b/c/d;p?q", "..", "http://a/b/"},
+ {"http://a/b/c/d;p?q", "../", "http://a/b/"},
+ {"http://a/b/c/d;p?q", "../g", "http://a/b/g"},
+ {"http://a/b/c/d;p?q", "../..", "http://a/"},
+ {"http://a/b/c/d;p?q", "../../", "http://a/"},
+ {"http://a/b/c/d;p?q", "../../g", "http://a/g"},
+
+ // RFC 3986: Abnormal Examples
+ // http://tools.ietf.org/html/rfc3986#section-5.4.2
+ {"http://a/b/c/d;p?q", "../../../g", "http://a/g"},
+ {"http://a/b/c/d;p?q", "../../../../g", "http://a/g"},
+ {"http://a/b/c/d;p?q", "/./g", "http://a/g"},
+ {"http://a/b/c/d;p?q", "/../g", "http://a/g"},
+ {"http://a/b/c/d;p?q", "g.", "http://a/b/c/g."},
+ {"http://a/b/c/d;p?q", ".g", "http://a/b/c/.g"},
+ {"http://a/b/c/d;p?q", "g..", "http://a/b/c/g.."},
+ {"http://a/b/c/d;p?q", "..g", "http://a/b/c/..g"},
+ {"http://a/b/c/d;p?q", "./../g", "http://a/b/g"},
+ {"http://a/b/c/d;p?q", "./g/.", "http://a/b/c/g/"},
+ {"http://a/b/c/d;p?q", "g/./h", "http://a/b/c/g/h"},
+ {"http://a/b/c/d;p?q", "g/../h", "http://a/b/c/h"},
+ {"http://a/b/c/d;p?q", "g;x=1/./y", "http://a/b/c/g;x=1/y"},
+ {"http://a/b/c/d;p?q", "g;x=1/../y", "http://a/b/c/y"},
+ {"http://a/b/c/d;p?q", "g?y/./x", "http://a/b/c/g?y/./x"},
+ {"http://a/b/c/d;p?q", "g?y/../x", "http://a/b/c/g?y/../x"},
+ {"http://a/b/c/d;p?q", "g#s/./x", "http://a/b/c/g#s/./x"},
+ {"http://a/b/c/d;p?q", "g#s/../x", "http://a/b/c/g#s/../x"},
+
+ // Extras.
+ {"https://a/b/c/d;p?q", "//g?q", "https://g?q"},
+ {"https://a/b/c/d;p?q", "//g#s", "https://g#s"},
+ {"https://a/b/c/d;p?q", "//g/d/e/f?y#s", "https://g/d/e/f?y#s"},
+ {"https://a/b/c/d;p#s", "?y", "https://a/b/c/d;p?y"},
+ {"https://a/b/c/d;p?q#s", "?y", "https://a/b/c/d;p?y"},
+}
+
+func TestResolveReference(t *testing.T) {
+ mustParse := func(url_ string) *url.URL {
+ u, err := url.Parse(url_)
+ if err != nil {
+ t.Fatalf("Expected URL to parse: %q, got error: %v", url_, err)
+ }
+ return u
+ }
+ opaque := &url.URL{Scheme: "scheme", Opaque: "opaque"}
+ for _, test := range resolveReferenceTests {
+ base := mustParse(test.base)
+ rel := mustParse(test.rel)
+ url := base.ResolveReference(rel)
+ if Escape(url) != test.expected {
+ t.Errorf("URL(%q).ResolveReference(%q) == %q, got %q", test.base, test.rel, test.expected, Escape(url))
+ }
+ // Ensure that new instances are returned.
+ if base == url {
+ t.Errorf("Expected URL.ResolveReference to return new URL instance.")
+ }
+ // Test the convenience wrapper too.
+ url, err := base.Parse(test.rel)
+ if err != nil {
+ t.Errorf("URL(%q).Parse(%q) failed: %v", test.base, test.rel, err)
+ } else if Escape(url) != test.expected {
+ t.Errorf("URL(%q).Parse(%q) == %q, got %q", test.base, test.rel, test.expected, Escape(url))
+ } else if base == url {
+ // Ensure that new instances are returned for the wrapper too.
+ t.Errorf("Expected URL.Parse to return new URL instance.")
+ }
+ // Ensure Opaque resets the URL.
+ url = base.ResolveReference(opaque)
+ if *url != *opaque {
+ t.Errorf("ResolveReference failed to resolve opaque URL: want %#v, got %#v", url, opaque)
+ }
+ // Test the convenience wrapper with an opaque URL too.
+ url, err = base.Parse("scheme:opaque")
+ if err != nil {
+ t.Errorf(`URL(%q).Parse("scheme:opaque") failed: %v`, test.base, err)
+ } else if *url != *opaque {
+ t.Errorf("Parse failed to resolve opaque URL: want %#v, got %#v", url, opaque)
+ } else if base == url {
+ // Ensure that new instances are returned, again.
+ t.Errorf("Expected URL.Parse to return new URL instance.")
+ }
+ }
+}
+
+type shouldEscapeTest struct {
+ in byte
+ mode encoding
+ escape bool
+}
+
+var shouldEscapeTests = []shouldEscapeTest{
+ // Unreserved characters (§2.3)
+ {'a', encodePath, false},
+ {'a', encodeUserPassword, false},
+ {'a', encodeQueryComponent, false},
+ {'a', encodeFragment, false},
+ {'z', encodePath, false},
+ {'A', encodePath, false},
+ {'Z', encodePath, false},
+ {'0', encodePath, false},
+ {'9', encodePath, false},
+ {'-', encodePath, false},
+ {'-', encodeUserPassword, false},
+ {'-', encodeQueryComponent, false},
+ {'-', encodeFragment, false},
+ {'.', encodePath, false},
+ {'_', encodePath, false},
+ {'~', encodePath, false},
+
+ // User information (§3.2.1)
+ {':', encodeUserPassword, true},
+ {'/', encodeUserPassword, true},
+ {'?', encodeUserPassword, true},
+ {'@', encodeUserPassword, true},
+ {'$', encodeUserPassword, false},
+ {'&', encodeUserPassword, false},
+ {'+', encodeUserPassword, false},
+ {',', encodeUserPassword, false},
+ {';', encodeUserPassword, false},
+ {'=', encodeUserPassword, false},
+}
+
+func TestShouldEscape(t *testing.T) {
+ for _, tt := range shouldEscapeTests {
+ if shouldEscape(tt.in, tt.mode) != tt.escape {
+ t.Errorf("shouldEscape(%q, %v) returned %v; expected %v", tt.in, tt.mode, !tt.escape, tt.escape)
+ }
+ }
+}