From bc759de26e502e1797dc8507d2261272a317eca7 Mon Sep 17 00:00:00 2001 From: Jordan Date: Sun, 20 Nov 2022 09:59:11 -0700 Subject: misc: use updated URL normalization/escape libraries --- go.mod | 3 +- go.sum | 4 + keep.go | 13 +- normalize/LICENSE | 12 - normalize/normalize.go | 379 ------------ normalize/normalize_test.go | 782 ------------------------ urlesc/LICENSE | 27 - urlesc/urlesc.go | 180 ------ urlesc/urlesc_test.go | 641 ------------------- vendor/github.com/PuerkitoBio/purell/.gitignore | 5 + vendor/github.com/PuerkitoBio/purell/LICENSE | 13 + vendor/github.com/PuerkitoBio/purell/README.md | 188 ++++++ vendor/github.com/PuerkitoBio/purell/purell.go | 378 ++++++++++++ vendor/github.com/PuerkitoBio/purell/urlesc.go | 174 ++++++ vendor/modules.txt | 3 + 15 files changed, 774 insertions(+), 2028 deletions(-) delete mode 100644 normalize/LICENSE delete mode 100644 normalize/normalize.go delete mode 100644 normalize/normalize_test.go delete mode 100644 urlesc/LICENSE delete mode 100644 urlesc/urlesc.go delete mode 100644 urlesc/urlesc_test.go create mode 100644 vendor/github.com/PuerkitoBio/purell/.gitignore create mode 100644 vendor/github.com/PuerkitoBio/purell/LICENSE create mode 100644 vendor/github.com/PuerkitoBio/purell/README.md create mode 100644 vendor/github.com/PuerkitoBio/purell/purell.go create mode 100644 vendor/github.com/PuerkitoBio/purell/urlesc.go diff --git a/go.mod b/go.mod index e1881e7..5b84532 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,15 @@ module keep go 1.17 require ( + github.com/PuerkitoBio/purell v1.2.0 github.com/bwmarrin/discordgo v0.26.1 github.com/mattn/go-sqlite3 v1.14.15 golang.org/x/net v0.0.0-20220927171203-f486391704dc - golang.org/x/text v0.3.7 ) require ( github.com/gorilla/websocket v1.5.0 // indirect golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be // indirect golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec // indirect + golang.org/x/text v0.3.7 // indirect ) diff --git a/go.sum b/go.sum index ee52aee..08a0885 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/PuerkitoBio/purell v1.2.0 h1:/Jdm5QfyM8zdlqT6WVZU4cfP23sot6CEHA4CS49Ezig= +github.com/PuerkitoBio/purell v1.2.0/go.mod h1:OhLRTaaIzhvIyofkJfB24gokC7tM42Px5UhoT32THBk= github.com/bwmarrin/discordgo v0.26.1 h1:AIrM+g3cl+iYBr4yBxCBp9tD9jR3K7upEjl0d89FRkE= github.com/bwmarrin/discordgo v0.26.1/go.mod h1:NJZpH+1AfhIcyQsPeuBKsUtYrRnjkyu0kIVMCHkZtRY= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -10,11 +12,13 @@ golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be h1:fmw3UbQh+nxngCAHrDCCzt golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220927171203-f486391704dc h1:FxpXZdoBqT8RjqTy6i1E8nXHhW21wK7ptQ/EPIGxzPQ= golang.org/x/net v0.0.0-20220927171203-f486391704dc/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI= golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/keep.go b/keep.go index 96fba29..5751bad 100644 --- a/keep.go +++ b/keep.go @@ -19,7 +19,7 @@ import ( "github.com/bwmarrin/discordgo" "golang.org/x/net/publicsuffix" - "keep/normalize" + "github.com/PuerkitoBio/purell" ) type Config struct { @@ -188,11 +188,12 @@ func messageCreate(s *discordgo.Session, m *discordgo.MessageCreate) { } // Normalize URL (RFC 3986) - uStr := normalize.NormalizeURL(u, - normalize.FlagsSafe|normalize.FlagRemoveDotSegments| - normalize.FlagRemoveDuplicateSlashes| - normalize.FlagRemoveFragment| - normalize.FlagSortQuery) + uStr := purell.NormalizeURL(u, + purell.FlagsSafe| + purell.FlagRemoveDotSegments| + purell.FlagRemoveDuplicateSlashes| + purell.FlagRemoveFragment| + purell.FlagSortQuery) // Ensure host is not present in ignoreList set if isIgnored(config.Ignore, uStr) { diff --git a/normalize/LICENSE b/normalize/LICENSE deleted file mode 100644 index 4b9986d..0000000 --- a/normalize/LICENSE +++ /dev/null @@ -1,12 +0,0 @@ -Copyright (c) 2012, Martin Angers -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - -* Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/normalize/normalize.go b/normalize/normalize.go deleted file mode 100644 index 0220f65..0000000 --- a/normalize/normalize.go +++ /dev/null @@ -1,379 +0,0 @@ -/* -Package normalize offers URL normalization as described on the wikipedia page: -http://en.wikipedia.org/wiki/URL_normalization -*/ -package normalize - -import ( - "bytes" - "fmt" - "net/url" - "regexp" - "sort" - "strconv" - "strings" - - "golang.org/x/net/idna" - "golang.org/x/text/unicode/norm" - "golang.org/x/text/width" - "keep/urlesc" -) - -// A set of normalization flags determines how a URL will -// be normalized. -type NormalizationFlags uint - -const ( - // Safe normalizations - FlagLowercaseScheme NormalizationFlags = 1 << iota // HTTP://host -> http://host, applied by default in Go1.1 - FlagLowercaseHost // http://HOST -> http://host - FlagUppercaseEscapes // http://host/t%ef -> http://host/t%EF - FlagDecodeUnnecessaryEscapes // http://host/t%41 -> http://host/tA - FlagEncodeNecessaryEscapes // http://host/!"#$ -> http://host/%21%22#$ - FlagRemoveDefaultPort // http://host:80 -> http://host - FlagRemoveEmptyQuerySeparator // http://host/path? -> http://host/path - - // Usually safe normalizations - FlagRemoveTrailingSlash // http://host/path/ -> http://host/path - FlagAddTrailingSlash // http://host/path -> http://host/path/ (should choose only one of these add/remove trailing slash flags) - FlagRemoveDotSegments // http://host/path/./a/b/../c -> http://host/path/a/c - - // Unsafe normalizations - FlagRemoveDirectoryIndex // http://host/path/index.html -> http://host/path/ - FlagRemoveFragment // http://host/path#fragment -> http://host/path - FlagForceHTTP // https://host -> http://host - FlagRemoveDuplicateSlashes // http://host/path//a///b -> http://host/path/a/b - FlagRemoveWWW // http://www.host/ -> http://host/ - FlagAddWWW // http://host/ -> http://www.host/ (should choose only one of these add/remove WWW flags) - FlagSortQuery // http://host/path?c=3&b=2&a=1&b=1 -> http://host/path?a=1&b=1&b=2&c=3 - - // Normalizations not in the wikipedia article, required to cover tests cases - // submitted by jehiah - FlagDecodeDWORDHost // http://1113982867 -> http://66.102.7.147 - FlagDecodeOctalHost // http://0102.0146.07.0223 -> http://66.102.7.147 - FlagDecodeHexHost // http://0x42660793 -> http://66.102.7.147 - FlagRemoveUnnecessaryHostDots // http://.host../path -> http://host/path - FlagRemoveEmptyPortSeparator // http://host:/path -> http://host/path - - // Convenience set of safe normalizations - FlagsSafe NormalizationFlags = FlagLowercaseHost | FlagLowercaseScheme | FlagUppercaseEscapes | FlagDecodeUnnecessaryEscapes | FlagEncodeNecessaryEscapes | FlagRemoveDefaultPort | FlagRemoveEmptyQuerySeparator - - // For convenience sets, "greedy" uses the "remove trailing slash" and "remove www. prefix" flags, - // while "non-greedy" uses the "add (or keep) the trailing slash" and "add www. prefix". - - // Convenience set of usually safe normalizations (includes FlagsSafe) - FlagsUsuallySafeGreedy NormalizationFlags = FlagsSafe | FlagRemoveTrailingSlash | FlagRemoveDotSegments - FlagsUsuallySafeNonGreedy NormalizationFlags = FlagsSafe | FlagAddTrailingSlash | FlagRemoveDotSegments - - // Convenience set of unsafe normalizations (includes FlagsUsuallySafe) - FlagsUnsafeGreedy NormalizationFlags = FlagsUsuallySafeGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagRemoveWWW | FlagSortQuery - FlagsUnsafeNonGreedy NormalizationFlags = FlagsUsuallySafeNonGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagAddWWW | FlagSortQuery - - // Convenience set of all available flags - FlagsAllGreedy = FlagsUnsafeGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator - FlagsAllNonGreedy = FlagsUnsafeNonGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator -) - -const ( - defaultHttpPort = ":80" - defaultHttpsPort = ":443" -) - -// Regular expressions used by the normalizations -var rxPort = regexp.MustCompile(`(:\d+)/?$`) -var rxDirIndex = regexp.MustCompile(`(^|/)((?:default|index)\.\w{1,4})$`) -var rxDupSlashes = regexp.MustCompile(`/{2,}`) -var rxDWORDHost = regexp.MustCompile(`^(\d+)((?:\.+)?(?:\:\d*)?)$`) -var rxOctalHost = regexp.MustCompile(`^(0\d*)\.(0\d*)\.(0\d*)\.(0\d*)((?:\.+)?(?:\:\d*)?)$`) -var rxHexHost = regexp.MustCompile(`^0x([0-9A-Fa-f]+)((?:\.+)?(?:\:\d*)?)$`) -var rxHostDots = regexp.MustCompile(`^(.+?)(:\d+)?$`) -var rxEmptyPort = regexp.MustCompile(`:+$`) - -// Map of flags to implementation function. -// FlagDecodeUnnecessaryEscapes has no action, since it is done automatically -// by parsing the string as an URL. Same for FlagUppercaseEscapes and FlagRemoveEmptyQuerySeparator. - -// Since maps have undefined traversing order, make a slice of ordered keys -var flagsOrder = []NormalizationFlags{ - FlagLowercaseScheme, - FlagLowercaseHost, - FlagRemoveDefaultPort, - FlagRemoveDirectoryIndex, - FlagRemoveDotSegments, - FlagRemoveFragment, - FlagForceHTTP, // Must be after remove default port (because https=443/http=80) - FlagRemoveDuplicateSlashes, - FlagRemoveWWW, - FlagAddWWW, - FlagSortQuery, - FlagDecodeDWORDHost, - FlagDecodeOctalHost, - FlagDecodeHexHost, - FlagRemoveUnnecessaryHostDots, - FlagRemoveEmptyPortSeparator, - FlagRemoveTrailingSlash, // These two (add/remove trailing slash) must be last - FlagAddTrailingSlash, -} - -// ... and then the map, where order is unimportant -var flags = map[NormalizationFlags]func(*url.URL){ - FlagLowercaseScheme: lowercaseScheme, - FlagLowercaseHost: lowercaseHost, - FlagRemoveDefaultPort: removeDefaultPort, - FlagRemoveDirectoryIndex: removeDirectoryIndex, - FlagRemoveDotSegments: removeDotSegments, - FlagRemoveFragment: removeFragment, - FlagForceHTTP: forceHTTP, - FlagRemoveDuplicateSlashes: removeDuplicateSlashes, - FlagRemoveWWW: removeWWW, - FlagAddWWW: addWWW, - FlagSortQuery: sortQuery, - FlagDecodeDWORDHost: decodeDWORDHost, - FlagDecodeOctalHost: decodeOctalHost, - FlagDecodeHexHost: decodeHexHost, - FlagRemoveUnnecessaryHostDots: removeUnncessaryHostDots, - FlagRemoveEmptyPortSeparator: removeEmptyPortSeparator, - FlagRemoveTrailingSlash: removeTrailingSlash, - FlagAddTrailingSlash: addTrailingSlash, -} - -// MustNormalizeURLString returns the normalized string, and panics if an error occurs. -// It takes an URL string as input, as well as the normalization flags. -func MustNormalizeURLString(u string, f NormalizationFlags) string { - result, e := NormalizeURLString(u, f) - if e != nil { - panic(e) - } - return result -} - -// NormalizeURLString returns the normalized string, or an error if it can't be parsed into an URL object. -// It takes an URL string as input, as well as the normalization flags. -func NormalizeURLString(u string, f NormalizationFlags) (string, error) { - parsed, err := url.Parse(u) - if err != nil { - return "", err - } - - if f&FlagLowercaseHost == FlagLowercaseHost { - parsed.Host = strings.ToLower(parsed.Host) - } - - // The idna package doesn't fully conform to RFC 5895 - // (https://tools.ietf.org/html/rfc5895), so we do it here. - // Taken from Go 1.8 cycle source, courtesy of bradfitz. - // TODO: Remove when (if?) idna package conforms to RFC 5895. - parsed.Host = width.Fold.String(parsed.Host) - parsed.Host = norm.NFC.String(parsed.Host) - if parsed.Host, err = idna.ToASCII(parsed.Host); err != nil { - return "", err - } - - return NormalizeURL(parsed, f), nil -} - -// NormalizeURL returns the normalized string. -// It takes a parsed URL object as input, as well as the normalization flags. -func NormalizeURL(u *url.URL, f NormalizationFlags) string { - for _, k := range flagsOrder { - if f&k == k { - flags[k](u) - } - } - return urlesc.Escape(u) -} - -func lowercaseScheme(u *url.URL) { - if len(u.Scheme) > 0 { - u.Scheme = strings.ToLower(u.Scheme) - } -} - -func lowercaseHost(u *url.URL) { - if len(u.Host) > 0 { - u.Host = strings.ToLower(u.Host) - } -} - -func removeDefaultPort(u *url.URL) { - if len(u.Host) > 0 { - scheme := strings.ToLower(u.Scheme) - u.Host = rxPort.ReplaceAllStringFunc(u.Host, func(val string) string { - if (scheme == "http" && val == defaultHttpPort) || (scheme == "https" && val == defaultHttpsPort) { - return "" - } - return val - }) - } -} - -func removeTrailingSlash(u *url.URL) { - if l := len(u.Path); l > 0 { - if strings.HasSuffix(u.Path, "/") { - u.Path = u.Path[:l-1] - } - } else if l = len(u.Host); l > 0 { - if strings.HasSuffix(u.Host, "/") { - u.Host = u.Host[:l-1] - } - } -} - -func addTrailingSlash(u *url.URL) { - if l := len(u.Path); l > 0 { - if !strings.HasSuffix(u.Path, "/") { - u.Path += "/" - } - } else if l = len(u.Host); l > 0 { - if !strings.HasSuffix(u.Host, "/") { - u.Host += "/" - } - } -} - -func removeDotSegments(u *url.URL) { - if len(u.Path) > 0 { - var dotFree []string - var lastIsDot bool - - sections := strings.Split(u.Path, "/") - for _, s := range sections { - if s == ".." { - if len(dotFree) > 0 { - dotFree = dotFree[:len(dotFree)-1] - } - } else if s != "." { - dotFree = append(dotFree, s) - } - lastIsDot = (s == "." || s == "..") - } - // Special case if host does not end with / and new path does not begin with / - u.Path = strings.Join(dotFree, "/") - if u.Host != "" && !strings.HasSuffix(u.Host, "/") && !strings.HasPrefix(u.Path, "/") { - u.Path = "/" + u.Path - } - // Special case if the last segment was a dot, make sure the path ends with a slash - if lastIsDot && !strings.HasSuffix(u.Path, "/") { - u.Path += "/" - } - } -} - -func removeDirectoryIndex(u *url.URL) { - if len(u.Path) > 0 { - u.Path = rxDirIndex.ReplaceAllString(u.Path, "$1") - } -} - -func removeFragment(u *url.URL) { - u.Fragment = "" -} - -func forceHTTP(u *url.URL) { - if strings.ToLower(u.Scheme) == "https" { - u.Scheme = "http" - } -} - -func removeDuplicateSlashes(u *url.URL) { - if len(u.Path) > 0 { - u.Path = rxDupSlashes.ReplaceAllString(u.Path, "/") - } -} - -func removeWWW(u *url.URL) { - if len(u.Host) > 0 && strings.HasPrefix(strings.ToLower(u.Host), "www.") { - u.Host = u.Host[4:] - } -} - -func addWWW(u *url.URL) { - if len(u.Host) > 0 && !strings.HasPrefix(strings.ToLower(u.Host), "www.") { - u.Host = "www." + u.Host - } -} - -func sortQuery(u *url.URL) { - q := u.Query() - - if len(q) > 0 { - arKeys := make([]string, len(q)) - i := 0 - for k := range q { - arKeys[i] = k - i++ - } - sort.Strings(arKeys) - buf := new(bytes.Buffer) - for _, k := range arKeys { - sort.Strings(q[k]) - for _, v := range q[k] { - if buf.Len() > 0 { - buf.WriteRune('&') - } - buf.WriteString(fmt.Sprintf("%s=%s", k, urlesc.QueryEscape(v))) - } - } - - // Rebuild the raw query string - u.RawQuery = buf.String() - } -} - -func decodeDWORDHost(u *url.URL) { - if len(u.Host) > 0 { - if matches := rxDWORDHost.FindStringSubmatch(u.Host); len(matches) > 2 { - var parts [4]int64 - - dword, _ := strconv.ParseInt(matches[1], 10, 0) - for i, shift := range []uint{24, 16, 8, 0} { - parts[i] = dword >> shift & 0xFF - } - u.Host = fmt.Sprintf("%d.%d.%d.%d%s", parts[0], parts[1], parts[2], parts[3], matches[2]) - } - } -} - -func decodeOctalHost(u *url.URL) { - if len(u.Host) > 0 { - if matches := rxOctalHost.FindStringSubmatch(u.Host); len(matches) > 5 { - var parts [4]int64 - - for i := 1; i <= 4; i++ { - parts[i-1], _ = strconv.ParseInt(matches[i], 8, 0) - } - u.Host = fmt.Sprintf("%d.%d.%d.%d%s", parts[0], parts[1], parts[2], parts[3], matches[5]) - } - } -} - -func decodeHexHost(u *url.URL) { - if len(u.Host) > 0 { - if matches := rxHexHost.FindStringSubmatch(u.Host); len(matches) > 2 { - // Conversion is safe because of regex validation - parsed, _ := strconv.ParseInt(matches[1], 16, 0) - // Set host as DWORD (base 10) encoded host - u.Host = fmt.Sprintf("%d%s", parsed, matches[2]) - // The rest is the same as decoding a DWORD host - decodeDWORDHost(u) - } - } -} - -func removeUnncessaryHostDots(u *url.URL) { - if len(u.Host) > 0 { - if matches := rxHostDots.FindStringSubmatch(u.Host); len(matches) > 1 { - // Trim the leading and trailing dots - u.Host = strings.Trim(matches[1], ".") - if len(matches) > 2 { - u.Host += matches[2] - } - } - } -} - -func removeEmptyPortSeparator(u *url.URL) { - if len(u.Host) > 0 { - u.Host = rxEmptyPort.ReplaceAllString(u.Host, "") - } -} diff --git a/normalize/normalize_test.go b/normalize/normalize_test.go deleted file mode 100644 index 177ae01..0000000 --- a/normalize/normalize_test.go +++ /dev/null @@ -1,782 +0,0 @@ -package normalize - -import ( - "fmt" - "net/url" - "testing" - "unicode" -) - -type testCase struct { - nm string - src string - flgs NormalizationFlags - res string - parsed bool -} - -var ( - cases = [...]*testCase{ - { - "LowerScheme", - "HTTP://www.SRC.ca", - FlagLowercaseScheme, - "http://www.SRC.ca", - false, - }, - { - "LowerScheme2", - "http://www.SRC.ca", - FlagLowercaseScheme, - "http://www.SRC.ca", - false, - }, - { - "LowerHost", - "HTTP://www.SRC.ca/", - FlagLowercaseHost, - "http://www.src.ca/", // Since Go1.1, scheme is automatically lowercased - false, - }, - { - "UpperEscapes", - `http://www.whatever.com/Some%aa%20Special%8Ecases/`, - FlagUppercaseEscapes, - "http://www.whatever.com/Some%AA%20Special%8Ecases/", - false, - }, - { - "UnnecessaryEscapes", - `http://www.toto.com/%41%42%2E%44/%32%33%52%2D/%5f%7E`, - FlagDecodeUnnecessaryEscapes, - "http://www.toto.com/AB.D/23R-/_~", - false, - }, - { - "RemoveDefaultPort", - "HTTP://www.SRC.ca:80/", - FlagRemoveDefaultPort, - "http://www.SRC.ca/", // Since Go1.1, scheme is automatically lowercased - false, - }, - { - "RemoveDefaultPort2", - "HTTP://www.SRC.ca:80", - FlagRemoveDefaultPort, - "http://www.SRC.ca", // Since Go1.1, scheme is automatically lowercased - false, - }, - { - "RemoveDefaultPort3", - "HTTP://www.SRC.ca:8080", - FlagRemoveDefaultPort, - "http://www.SRC.ca:8080", // Since Go1.1, scheme is automatically lowercased - false, - }, - { - "Safe", - "HTTP://www.SRC.ca:80/to%1ato%8b%ee/OKnow%41%42%43%7e", - FlagsSafe, - "http://www.src.ca/to%1Ato%8B%EE/OKnowABC~", - false, - }, - { - "BothLower", - "HTTP://www.SRC.ca:80/to%1ato%8b%ee/OKnow%41%42%43%7e", - FlagLowercaseHost | FlagLowercaseScheme, - "http://www.src.ca:80/to%1Ato%8B%EE/OKnowABC~", - false, - }, - { - "RemoveTrailingSlash", - "HTTP://www.SRC.ca:80/", - FlagRemoveTrailingSlash, - "http://www.SRC.ca:80", // Since Go1.1, scheme is automatically lowercased - false, - }, - { - "RemoveTrailingSlash2", - "HTTP://www.SRC.ca:80/toto/titi/", - FlagRemoveTrailingSlash, - "http://www.SRC.ca:80/toto/titi", // Since Go1.1, scheme is automatically lowercased - false, - }, - { - "RemoveTrailingSlash3", - "HTTP://www.SRC.ca:80/toto/titi/fin/?a=1", - FlagRemoveTrailingSlash, - "http://www.SRC.ca:80/toto/titi/fin?a=1", // Since Go1.1, scheme is automatically lowercased - false, - }, - { - "AddTrailingSlash", - "HTTP://www.SRC.ca:80", - FlagAddTrailingSlash, - "http://www.SRC.ca:80/", // Since Go1.1, scheme is automatically lowercased - false, - }, - { - "AddTrailingSlash2", - "HTTP://www.SRC.ca:80/toto/titi.html", - FlagAddTrailingSlash, - "http://www.SRC.ca:80/toto/titi.html/", // Since Go1.1, scheme is automatically lowercased - false, - }, - { - "AddTrailingSlash3", - "HTTP://www.SRC.ca:80/toto/titi/fin?a=1", - FlagAddTrailingSlash, - "http://www.SRC.ca:80/toto/titi/fin/?a=1", // Since Go1.1, scheme is automatically lowercased - false, - }, - { - "RemoveDotSegments", - "HTTP://root/a/b/./../../c/", - FlagRemoveDotSegments, - "http://root/c/", // Since Go1.1, scheme is automatically lowercased - false, - }, - { - "RemoveDotSegments2", - "HTTP://root/../a/b/./../c/../d", - FlagRemoveDotSegments, - "http://root/a/d", // Since Go1.1, scheme is automatically lowercased - false, - }, - { - "UsuallySafe", - "HTTP://www.SRC.ca:80/to%1ato%8b%ee/./c/d/../OKnow%41%42%43%7e/?a=b#test", - FlagsUsuallySafeGreedy, - "http://www.src.ca/to%1Ato%8B%EE/c/OKnowABC~?a=b#test", - false, - }, - { - "RemoveDirectoryIndex", - "HTTP://root/a/b/c/default.aspx", - FlagRemoveDirectoryIndex, - "http://root/a/b/c/", // Since Go1.1, scheme is automatically lowercased - false, - }, - { - "RemoveDirectoryIndex2", - "HTTP://root/a/b/c/default#a=b", - FlagRemoveDirectoryIndex, - "http://root/a/b/c/default#a=b", // Since Go1.1, scheme is automatically lowercased - false, - }, - { - "RemoveFragment", - "HTTP://root/a/b/c/default#toto=tata", - FlagRemoveFragment, - "http://root/a/b/c/default", // Since Go1.1, scheme is automatically lowercased - false, - }, - { - "ForceHTTP", - "https://root/a/b/c/default#toto=tata", - FlagForceHTTP, - "http://root/a/b/c/default#toto=tata", - false, - }, - { - "RemoveDuplicateSlashes", - "https://root/a//b///c////default#toto=tata", - FlagRemoveDuplicateSlashes, - "https://root/a/b/c/default#toto=tata", - false, - }, - { - "RemoveDuplicateSlashes2", - "https://root//a//b///c////default#toto=tata", - FlagRemoveDuplicateSlashes, - "https://root/a/b/c/default#toto=tata", - false, - }, - { - "RemoveWWW", - "https://www.root/a/b/c/", - FlagRemoveWWW, - "https://root/a/b/c/", - false, - }, - { - "RemoveWWW2", - "https://WwW.Root/a/b/c/", - FlagRemoveWWW, - "https://Root/a/b/c/", - false, - }, - { - "AddWWW", - "https://Root/a/b/c/", - FlagAddWWW, - "https://www.Root/a/b/c/", - false, - }, - { - "SortQuery", - "http://root/toto/?b=4&a=1&c=3&b=2&a=5", - FlagSortQuery, - "http://root/toto/?a=1&a=5&b=2&b=4&c=3", - false, - }, - { - "RemoveEmptyQuerySeparator", - "http://root/toto/?", - FlagRemoveEmptyQuerySeparator, - "http://root/toto/", - false, - }, - { - "Unsafe", - "HTTPS://www.RooT.com/toto/t%45%1f///a/./b/../c/?z=3&w=2&a=4&w=1#invalid", - FlagsUnsafeGreedy, - "http://root.com/toto/tE%1F/a/c?a=4&w=1&w=2&z=3", - false, - }, - { - "Safe2", - "HTTPS://www.RooT.com/toto/t%45%1f///a/./b/../c/?z=3&w=2&a=4&w=1#invalid", - FlagsSafe, - "https://www.root.com/toto/tE%1F///a/./b/../c/?z=3&w=2&a=4&w=1#invalid", - false, - }, - { - "UsuallySafe2", - "HTTPS://www.RooT.com/toto/t%45%1f///a/./b/../c/?z=3&w=2&a=4&w=1#invalid", - FlagsUsuallySafeGreedy, - "https://www.root.com/toto/tE%1F///a/c?z=3&w=2&a=4&w=1#invalid", - false, - }, - { - "AddTrailingSlashBug", - "http://src.ca/", - FlagsAllNonGreedy, - "http://www.src.ca/", - false, - }, - { - "SourceModified", - "HTTPS://www.RooT.com/toto/t%45%1f///a/./b/../c/?z=3&w=2&a=4&w=1#invalid", - FlagsUnsafeGreedy, - "http://root.com/toto/tE%1F/a/c?a=4&w=1&w=2&z=3", - true, - }, - { - "IPv6-1", - "http://[2001:db8:1f70::999:de8:7648:6e8]/test", - FlagsSafe | FlagRemoveDotSegments, - "http://[2001:db8:1f70::999:de8:7648:6e8]/test", - false, - }, - { - "IPv6-2", - "http://[::ffff:192.168.1.1]/test", - FlagsSafe | FlagRemoveDotSegments, - "http://[::ffff:192.168.1.1]/test", - false, - }, - { - "IPv6-3", - "http://[::ffff:192.168.1.1]:80/test", - FlagsSafe | FlagRemoveDotSegments, - "http://[::ffff:192.168.1.1]/test", - false, - }, - { - "IPv6-4", - "htTps://[::fFff:192.168.1.1]:443/test", - FlagsSafe | FlagRemoveDotSegments, - "https://[::ffff:192.168.1.1]/test", - false, - }, - { - "FTP", - "ftp://user:pass@ftp.foo.net/foo/bar", - FlagsSafe | FlagRemoveDotSegments, - "ftp://user:pass@ftp.foo.net/foo/bar", - false, - }, - { - "Standard-1", - "http://www.foo.com:80/foo", - FlagsSafe | FlagRemoveDotSegments, - "http://www.foo.com/foo", - false, - }, - { - "Standard-2", - "http://www.foo.com:8000/foo", - FlagsSafe | FlagRemoveDotSegments, - "http://www.foo.com:8000/foo", - false, - }, - { - "Standard-3", - "http://www.foo.com/%7ebar", - FlagsSafe | FlagRemoveDotSegments, - "http://www.foo.com/~bar", - false, - }, - { - "Standard-4", - "http://www.foo.com/%7Ebar", - FlagsSafe | FlagRemoveDotSegments, - "http://www.foo.com/~bar", - false, - }, - { - "Standard-5", - "http://USER:pass@www.Example.COM/foo/bar", - FlagsSafe | FlagRemoveDotSegments, - "http://USER:pass@www.example.com/foo/bar", - false, - }, - { - "Standard-6", - "http://test.example/?a=%26&b=1", - FlagsSafe | FlagRemoveDotSegments, - "http://test.example/?a=%26&b=1", - false, - }, - { - "Standard-7", - "http://test.example/%25/?p=%20val%20%25", - FlagsSafe | FlagRemoveDotSegments, - "http://test.example/%25/?p=%20val%20%25", - false, - }, - { - "Standard-8", - "http://test.example/path/with a%20space+/", - FlagsSafe | FlagRemoveDotSegments, - "http://test.example/path/with%20a%20space+/", - false, - }, - { - "Standard-9", - "http://test.example/?", - FlagsSafe | FlagRemoveDotSegments, - "http://test.example/", - false, - }, - { - "Standard-10", - "http://a.COM/path/?b&a", - FlagsSafe | FlagRemoveDotSegments, - "http://a.com/path/?b&a", - false, - }, - { - "StandardCasesAddTrailingSlash", - "http://test.example?", - FlagsSafe | FlagAddTrailingSlash, - "http://test.example/", - false, - }, - { - "OctalIP-1", - "http://0123.011.0.4/", - FlagsSafe | FlagDecodeOctalHost, - "http://0123.011.0.4/", - false, - }, - { - "OctalIP-2", - "http://0102.0146.07.0223/", - FlagsSafe | FlagDecodeOctalHost, - "http://66.102.7.147/", - false, - }, - { - "OctalIP-3", - "http://0102.0146.07.0223.:23/", - FlagsSafe | FlagDecodeOctalHost, - "http://66.102.7.147.:23/", - false, - }, - { - "OctalIP-4", - "http://USER:pass@0102.0146.07.0223../", - FlagsSafe | FlagDecodeOctalHost, - "http://USER:pass@66.102.7.147../", - false, - }, - { - "DWORDIP-1", - "http://123.1113982867/", - FlagsSafe | FlagDecodeDWORDHost, - "http://123.1113982867/", - false, - }, - { - "DWORDIP-2", - "http://1113982867/", - FlagsSafe | FlagDecodeDWORDHost, - "http://66.102.7.147/", - false, - }, - { - "DWORDIP-3", - "http://1113982867.:23/", - FlagsSafe | FlagDecodeDWORDHost, - "http://66.102.7.147.:23/", - false, - }, - { - "DWORDIP-4", - "http://USER:pass@1113982867../", - FlagsSafe | FlagDecodeDWORDHost, - "http://USER:pass@66.102.7.147../", - false, - }, - { - "HexIP-1", - "http://0x123.1113982867/", - FlagsSafe | FlagDecodeHexHost, - "http://0x123.1113982867/", - false, - }, - { - "HexIP-2", - "http://0x42660793/", - FlagsSafe | FlagDecodeHexHost, - "http://66.102.7.147/", - false, - }, - { - "HexIP-3", - "http://0x42660793.:23/", - FlagsSafe | FlagDecodeHexHost, - "http://66.102.7.147.:23/", - false, - }, - { - "HexIP-4", - "http://USER:pass@0x42660793../", - FlagsSafe | FlagDecodeHexHost, - "http://USER:pass@66.102.7.147../", - false, - }, - { - "UnnecessaryHostDots-1", - "http://.www.foo.com../foo/bar.html", - FlagsSafe | FlagRemoveUnnecessaryHostDots, - "http://www.foo.com/foo/bar.html", - false, - }, - { - "UnnecessaryHostDots-2", - "http://www.foo.com./foo/bar.html", - FlagsSafe | FlagRemoveUnnecessaryHostDots, - "http://www.foo.com/foo/bar.html", - false, - }, - { - "UnnecessaryHostDots-3", - "http://www.foo.com.:81/foo", - FlagsSafe | FlagRemoveUnnecessaryHostDots, - "http://www.foo.com:81/foo", - false, - }, - { - "UnnecessaryHostDots-4", - "http://www.example.com./", - FlagsSafe | FlagRemoveUnnecessaryHostDots, - "http://www.example.com/", - false, - }, - { - "EmptyPort-1", - "http://www.thedraymin.co.uk:/main/?p=308", - FlagsSafe | FlagRemoveEmptyPortSeparator, - "http://www.thedraymin.co.uk/main/?p=308", - false, - }, - { - "EmptyPort-2", - "http://www.src.ca:", - FlagsSafe | FlagRemoveEmptyPortSeparator, - "http://www.src.ca", - false, - }, - { - "Slashes-1", - "http://test.example/foo/bar/.", - FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, - "http://test.example/foo/bar/", - false, - }, - { - "Slashes-2", - "http://test.example/foo/bar/./", - FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, - "http://test.example/foo/bar/", - false, - }, - { - "Slashes-3", - "http://test.example/foo/bar/..", - FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, - "http://test.example/foo/", - false, - }, - { - "Slashes-4", - "http://test.example/foo/bar/../", - FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, - "http://test.example/foo/", - false, - }, - { - "Slashes-5", - "http://test.example/foo/bar/../baz", - FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, - "http://test.example/foo/baz", - false, - }, - { - "Slashes-6", - "http://test.example/foo/bar/../..", - FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, - "http://test.example/", - false, - }, - { - "Slashes-7", - "http://test.example/foo/bar/../../", - FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, - "http://test.example/", - false, - }, - { - "Slashes-8", - "http://test.example/foo/bar/../../baz", - FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, - "http://test.example/baz", - false, - }, - { - "Slashes-9", - "http://test.example/foo/bar/../../../baz", - FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, - "http://test.example/baz", - false, - }, - { - "Slashes-10", - "http://test.example/foo/bar/../../../../baz", - FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, - "http://test.example/baz", - false, - }, - { - "Slashes-11", - "http://test.example/./foo", - FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, - "http://test.example/foo", - false, - }, - { - "Slashes-12", - "http://test.example/../foo", - FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, - "http://test.example/foo", - false, - }, - { - "Slashes-13", - "http://test.example/foo.", - FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, - "http://test.example/foo.", - false, - }, - { - "Slashes-14", - "http://test.example/.foo", - FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, - "http://test.example/.foo", - false, - }, - { - "Slashes-15", - "http://test.example/foo..", - FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, - "http://test.example/foo..", - false, - }, - { - "Slashes-16", - "http://test.example/..foo", - FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, - "http://test.example/..foo", - false, - }, - { - "Slashes-17", - "http://test.example/./../foo", - FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, - "http://test.example/foo", - false, - }, - { - "Slashes-18", - "http://test.example/./foo/.", - FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, - "http://test.example/foo/", - false, - }, - { - "Slashes-19", - "http://test.example/foo/./bar", - FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, - "http://test.example/foo/bar", - false, - }, - { - "Slashes-20", - "http://test.example/foo/../bar", - FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, - "http://test.example/bar", - false, - }, - { - "Slashes-21", - "http://test.example/foo//", - FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, - "http://test.example/foo/", - false, - }, - { - "Slashes-22", - "http://test.example/foo///bar//", - FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, - "http://test.example/foo/bar/", - false, - }, - { - "Relative", - "foo/bar", - FlagsAllGreedy, - "foo/bar", - false, - }, - { - "Relative-1", - "./../foo", - FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, - "foo", - false, - }, - { - "Relative-2", - "./foo/bar/../baz/../bang/..", - FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, - "foo/", - false, - }, - { - "Relative-3", - "foo///bar//", - FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, - "foo/bar/", - false, - }, - { - "Relative-4", - "www.youtube.com", - FlagsUsuallySafeGreedy, - "www.youtube.com", - false, - }, - { - "Issue-#24", - "///foo///bar///", - FlagRemoveDuplicateSlashes | FlagRemoveTrailingSlash, - "/foo/bar", - false, - }, - /*&testCase{ - "UrlNorm-5", - "http://ja.wikipedia.org/wiki/%E3%82%AD%E3%83%A3%E3%82%BF%E3%83%94%E3%83%A9%E3%83%BC%E3%82%B8%E3%83%A3%E3%83%91%E3%83%B3", - FlagsSafe | FlagRemoveDotSegments, - "http://ja.wikipedia.org/wiki/\xe3\x82\xad\xe3\x83\xa3\xe3\x82\xbf\xe3\x83\x94\xe3\x83\xa9\xe3\x83\xbc\xe3\x82\xb8\xe3\x83\xa3\xe3\x83\x91\xe3\x83\xb3", - false, - }, - &testCase{ - "UrlNorm-1", - "http://test.example/?a=%e3%82%82%26", - FlagsAllGreedy, - "http://test.example/?a=\xe3\x82\x82%26", - false, - },*/ - } -) - -func TestRunner(t *testing.T) { - for _, tc := range cases { - runCase(tc, t) - } -} - -func runCase(tc *testCase, t *testing.T) { - t.Logf("running %s...", tc.nm) - if tc.parsed { - u, e := url.Parse(tc.src) - if e != nil { - t.Errorf("%s - FAIL : %s", tc.nm, e) - return - } else { - NormalizeURL(u, tc.flgs) - if s := u.String(); s != tc.res { - t.Errorf("%s - FAIL expected '%s', got '%s'", tc.nm, tc.res, s) - } - } - } else { - if s, e := NormalizeURLString(tc.src, tc.flgs); e != nil { - t.Errorf("%s - FAIL : %s", tc.nm, e) - } else if s != tc.res { - t.Errorf("%s - FAIL expected '%s', got '%s'", tc.nm, tc.res, s) - } - } -} - -func TestDecodeUnnecessaryEscapesAll(t *testing.T) { - var url = "http://host/" - - for i := 0; i < 256; i++ { - url += fmt.Sprintf("%%%02x", i) - } - s, err := NormalizeURLString(url, FlagDecodeUnnecessaryEscapes) - if err != nil { - t.Fatalf("parse error: %s", err) - } - - const want = "http://host/%00%01%02%03%04%05%06%07%08%09%0A%0B%0C%0D%0E%0F%10%11%12%13%14%15%16%17%18%19%1A%1B%1C%1D%1E%1F%20!%22%23$%25&'()*+,-./0123456789:;%3C=%3E%3F@ABCDEFGHIJKLMNOPQRSTUVWXYZ[%5C]%5E_%60abcdefghijklmnopqrstuvwxyz%7B%7C%7D~%7F%80%81%82%83%84%85%86%87%88%89%8A%8B%8C%8D%8E%8F%90%91%92%93%94%95%96%97%98%99%9A%9B%9C%9D%9E%9F%A0%A1%A2%A3%A4%A5%A6%A7%A8%A9%AA%AB%AC%AD%AE%AF%B0%B1%B2%B3%B4%B5%B6%B7%B8%B9%BA%BB%BC%BD%BE%BF%C0%C1%C2%C3%C4%C5%C6%C7%C8%C9%CA%CB%CC%CD%CE%CF%D0%D1%D2%D3%D4%D5%D6%D7%D8%D9%DA%DB%DC%DD%DE%DF%E0%E1%E2%E3%E4%E5%E6%E7%E8%E9%EA%EB%EC%ED%EE%EF%F0%F1%F2%F3%F4%F5%F6%F7%F8%F9%FA%FB%FC%FD%FE%FF" - if s != want { - t.Errorf("DecodeUnnecessaryEscapesAll:\nwant\n%s\ngot\n%s", want, s) - } -} - -func TestEncodeNecessaryEscapesAll(t *testing.T) { - const base = "http://host/" - var path []byte - - for i := 0; i < 256; i++ { - // Since go1.12, url.Parse fails if the raw URL contains ASCII control characters, - // meaning anything < 0x20 and 0x7f (DEL), so do not add those bytes to the constructed url. - // See https://github.com/PuerkitoBio/purell/issues/28 - if i != 0x25 && !unicode.IsControl(rune(i)) { - path = append(path, byte(i)) - } - } - s, err := NormalizeURLString(base+string(path), FlagEncodeNecessaryEscapes) - if err != nil { - t.Fatalf("parse error: %s", err) - } - - const want = "http://host/%20!%22#$&'()*+,-./0123456789:;%3C=%3E?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[%5C]%5E_%60abcdefghijklmnopqrstuvwxyz%7B%7C%7D~%A0%A1%A2%A3%A4%A5%A6%A7%A8%A9%AA%AB%AC%AD%AE%AF%B0%B1%B2%B3%B4%B5%B6%B7%B8%B9%BA%BB%BC%BD%BE%BF%C0%C1%C2%C3%C4%C5%C6%C7%C8%C9%CA%CB%CC%CD%CE%CF%D0%D1%D2%D3%D4%D5%D6%D7%D8%D9%DA%DB%DC%DD%DE%DF%E0%E1%E2%E3%E4%E5%E6%E7%E8%E9%EA%EB%EC%ED%EE%EF%F0%F1%F2%F3%F4%F5%F6%F7%F8%F9%FA%FB%FC%FD%FE%FF" - if s != want { - t.Errorf("EncodeNecessaryEscapesAll:\nwant\n%s\ngot\n%s", want, s) - } -} diff --git a/urlesc/LICENSE b/urlesc/LICENSE deleted file mode 100644 index 7448756..0000000 --- a/urlesc/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2012 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/urlesc/urlesc.go b/urlesc/urlesc.go deleted file mode 100644 index 1b84624..0000000 --- a/urlesc/urlesc.go +++ /dev/null @@ -1,180 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package urlesc implements query escaping as per RFC 3986. -// It contains some parts of the net/url package, modified so as to allow -// some reserved characters incorrectly escaped by net/url. -// See https://github.com/golang/go/issues/5684 -package urlesc - -import ( - "bytes" - "net/url" - "strings" -) - -type encoding int - -const ( - encodePath encoding = 1 + iota - encodeUserPassword - encodeQueryComponent - encodeFragment -) - -// Return true if the specified character should be escaped when -// appearing in a URL string, according to RFC 3986. -func shouldEscape(c byte, mode encoding) bool { - // §2.3 Unreserved characters (alphanum) - if 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' { - return false - } - - switch c { - case '-', '.', '_', '~': // §2.3 Unreserved characters (mark) - return false - - // §2.2 Reserved characters (reserved) - case ':', '/', '?', '#', '[', ']', '@', // gen-delims - '!', '$', '&', '\'', '(', ')', '*', '+', ',', ';', '=': // sub-delims - // Different sections of the URL allow a few of - // the reserved characters to appear unescaped. - switch mode { - case encodePath: // §3.3 - // The RFC allows sub-delims and : @. - // '/', '[' and ']' can be used to assign meaning to individual path - // segments. This package only manipulates the path as a whole, - // so we allow those as well. That leaves only ? and # to escape. - return c == '?' || c == '#' - - case encodeUserPassword: // §3.2.1 - // The RFC allows : and sub-delims in - // userinfo. The parsing of userinfo treats ':' as special so we must escape - // all the gen-delims. - return c == ':' || c == '/' || c == '?' || c == '#' || c == '[' || c == ']' || c == '@' - - case encodeQueryComponent: // §3.4 - // The RFC allows / and ?. - return c != '/' && c != '?' - - case encodeFragment: // §4.1 - // The RFC text is silent but the grammar allows - // everything, so escape nothing but # - return c == '#' - } - } - - // Everything else must be escaped. - return true -} - -// QueryEscape escapes the string so it can be safely placed -// inside a URL query. -func QueryEscape(s string) string { - return escape(s, encodeQueryComponent) -} - -func escape(s string, mode encoding) string { - spaceCount, hexCount := 0, 0 - for i := 0; i < len(s); i++ { - c := s[i] - if shouldEscape(c, mode) { - if c == ' ' && mode == encodeQueryComponent { - spaceCount++ - } else { - hexCount++ - } - } - } - - if spaceCount == 0 && hexCount == 0 { - return s - } - - t := make([]byte, len(s)+2*hexCount) - j := 0 - for i := 0; i < len(s); i++ { - switch c := s[i]; { - case c == ' ' && mode == encodeQueryComponent: - t[j] = '+' - j++ - case shouldEscape(c, mode): - t[j] = '%' - t[j+1] = "0123456789ABCDEF"[c>>4] - t[j+2] = "0123456789ABCDEF"[c&15] - j += 3 - default: - t[j] = s[i] - j++ - } - } - return string(t) -} - -var uiReplacer = strings.NewReplacer( - "%21", "!", - "%27", "'", - "%28", "(", - "%29", ")", - "%2A", "*", -) - -// unescapeUserinfo unescapes some characters that need not to be escaped as per RFC3986. -func unescapeUserinfo(s string) string { - return uiReplacer.Replace(s) -} - -// Escape reassembles the URL into a valid URL string. -// The general form of the result is one of: -// -// scheme:opaque -// scheme://userinfo@host/path?query#fragment -// -// If u.Opaque is non-empty, String uses the first form; -// otherwise it uses the second form. -// -// In the second form, the following rules apply: -// - if u.Scheme is empty, scheme: is omitted. -// - if u.User is nil, userinfo@ is omitted. -// - if u.Host is empty, host/ is omitted. -// - if u.Scheme and u.Host are empty and u.User is nil, -// the entire scheme://userinfo@host/ is omitted. -// - if u.Host is non-empty and u.Path begins with a /, -// the form host/path does not add its own /. -// - if u.RawQuery is empty, ?query is omitted. -// - if u.Fragment is empty, #fragment is omitted. -func Escape(u *url.URL) string { - var buf bytes.Buffer - if u.Scheme != "" { - buf.WriteString(u.Scheme) - buf.WriteByte(':') - } - if u.Opaque != "" { - buf.WriteString(u.Opaque) - } else { - if u.Scheme != "" || u.Host != "" || u.User != nil { - buf.WriteString("//") - if ui := u.User; ui != nil { - buf.WriteString(unescapeUserinfo(ui.String())) - buf.WriteByte('@') - } - if h := u.Host; h != "" { - buf.WriteString(h) - } - } - if u.Path != "" && u.Path[0] != '/' && u.Host != "" { - buf.WriteByte('/') - } - buf.WriteString(escape(u.Path, encodePath)) - } - if u.RawQuery != "" { - buf.WriteByte('?') - buf.WriteString(u.RawQuery) - } - if u.Fragment != "" { - buf.WriteByte('#') - buf.WriteString(escape(u.Fragment, encodeFragment)) - } - return buf.String() -} diff --git a/urlesc/urlesc_test.go b/urlesc/urlesc_test.go deleted file mode 100644 index 45202e1..0000000 --- a/urlesc/urlesc_test.go +++ /dev/null @@ -1,641 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package urlesc - -import ( - "net/url" - "testing" -) - -type URLTest struct { - in string - out *url.URL - roundtrip string // expected result of reserializing the URL; empty means same as "in". -} - -var urltests = []URLTest{ - // no path - { - "http://www.google.com", - &url.URL{ - Scheme: "http", - Host: "www.google.com", - }, - "", - }, - // path - { - "http://www.google.com/", - &url.URL{ - Scheme: "http", - Host: "www.google.com", - Path: "/", - }, - "", - }, - // path with hex escaping - { - "http://www.google.com/file%20one%26two", - &url.URL{ - Scheme: "http", - Host: "www.google.com", - Path: "/file one&two", - }, - "http://www.google.com/file%20one&two", - }, - // user - { - "ftp://webmaster@www.google.com/", - &url.URL{ - Scheme: "ftp", - User: url.User("webmaster"), - Host: "www.google.com", - Path: "/", - }, - "", - }, - // escape sequence in username - { - "ftp://john%20doe@www.google.com/", - &url.URL{ - Scheme: "ftp", - User: url.User("john doe"), - Host: "www.google.com", - Path: "/", - }, - "ftp://john%20doe@www.google.com/", - }, - // query - { - "http://www.google.com/?q=go+language", - &url.URL{ - Scheme: "http", - Host: "www.google.com", - Path: "/", - RawQuery: "q=go+language", - }, - "", - }, - // query with hex escaping: NOT parsed - { - "http://www.google.com/?q=go%20language", - &url.URL{ - Scheme: "http", - Host: "www.google.com", - Path: "/", - RawQuery: "q=go%20language", - }, - "", - }, - // %20 outside query - { - "http://www.google.com/a%20b?q=c+d", - &url.URL{ - Scheme: "http", - Host: "www.google.com", - Path: "/a b", - RawQuery: "q=c+d", - }, - "", - }, - // path without leading /, so no parsing - { - "http:www.google.com/?q=go+language", - &url.URL{ - Scheme: "http", - Opaque: "www.google.com/", - RawQuery: "q=go+language", - }, - "http:www.google.com/?q=go+language", - }, - // path without leading /, so no parsing - { - "http:%2f%2fwww.google.com/?q=go+language", - &url.URL{ - Scheme: "http", - Opaque: "%2f%2fwww.google.com/", - RawQuery: "q=go+language", - }, - "http:%2f%2fwww.google.com/?q=go+language", - }, - // non-authority with path - { - "mailto:/webmaster@golang.org", - &url.URL{ - Scheme: "mailto", - Path: "/webmaster@golang.org", - }, - "mailto:///webmaster@golang.org", // unfortunate compromise - }, - // non-authority - { - "mailto:webmaster@golang.org", - &url.URL{ - Scheme: "mailto", - Opaque: "webmaster@golang.org", - }, - "", - }, - // unescaped :// in query should not create a scheme - { - "/foo?query=http://bad", - &url.URL{ - Path: "/foo", - RawQuery: "query=http://bad", - }, - "", - }, - // leading // without scheme should create an authority - { - "//foo", - &url.URL{ - Host: "foo", - }, - "", - }, - // leading // without scheme, with userinfo, path, and query - { - "//user@foo/path?a=b", - &url.URL{ - User: url.User("user"), - Host: "foo", - Path: "/path", - RawQuery: "a=b", - }, - "", - }, - // Three leading slashes isn't an authority, but doesn't return an error. - // (We can't return an error, as this code is also used via - // ServeHTTP -> ReadRequest -> Parse, which is arguably a - // different URL parsing context, but currently shares the - // same codepath) - { - "///threeslashes", - &url.URL{ - Path: "///threeslashes", - }, - "", - }, - { - "http://user:password@google.com", - &url.URL{ - Scheme: "http", - User: url.UserPassword("user", "password"), - Host: "google.com", - }, - "http://user:password@google.com", - }, - // unescaped @ in username should not confuse host - { - "http://j@ne:password@google.com", - &url.URL{ - Scheme: "http", - User: url.UserPassword("j@ne", "password"), - Host: "google.com", - }, - "http://j%40ne:password@google.com", - }, - // unescaped @ in password should not confuse host - { - "http://jane:p@ssword@google.com", - &url.URL{ - Scheme: "http", - User: url.UserPassword("jane", "p@ssword"), - Host: "google.com", - }, - "http://jane:p%40ssword@google.com", - }, - { - "http://j@ne:password@google.com/p@th?q=@go", - &url.URL{ - Scheme: "http", - User: url.UserPassword("j@ne", "password"), - Host: "google.com", - Path: "/p@th", - RawQuery: "q=@go", - }, - "http://j%40ne:password@google.com/p@th?q=@go", - }, - { - "http://www.google.com/?q=go+language#foo", - &url.URL{ - Scheme: "http", - Host: "www.google.com", - Path: "/", - RawQuery: "q=go+language", - Fragment: "foo", - }, - "", - }, - { - "http://www.google.com/?q=go+language#foo%26bar", - &url.URL{ - Scheme: "http", - Host: "www.google.com", - Path: "/", - RawQuery: "q=go+language", - Fragment: "foo&bar", - }, - "http://www.google.com/?q=go+language#foo&bar", - }, - { - "file:///home/adg/rabbits", - &url.URL{ - Scheme: "file", - Host: "", - Path: "/home/adg/rabbits", - }, - "file:///home/adg/rabbits", - }, - // "Windows" paths are no exception to the rule. - // See golang.org/issue/6027, especially comment #9. - { - "file:///C:/FooBar/Baz.txt", - &url.URL{ - Scheme: "file", - Host: "", - Path: "/C:/FooBar/Baz.txt", - }, - "file:///C:/FooBar/Baz.txt", - }, - // case-insensitive scheme - { - "MaIlTo:webmaster@golang.org", - &url.URL{ - Scheme: "mailto", - Opaque: "webmaster@golang.org", - }, - "mailto:webmaster@golang.org", - }, - // Relative path - { - "a/b/c", - &url.URL{ - Path: "a/b/c", - }, - "a/b/c", - }, - // escaped '?' in username and password - { - "http://%3Fam:pa%3Fsword@google.com", - &url.URL{ - Scheme: "http", - User: url.UserPassword("?am", "pa?sword"), - Host: "google.com", - }, - "", - }, - // escaped '?' and '#' in path - { - "http://example.com/%3F%23", - &url.URL{ - Scheme: "http", - Host: "example.com", - Path: "?#", - }, - "", - }, - // unescaped [ ] ! ' ( ) * in path - { - "http://example.com/[]!'()*", - &url.URL{ - Scheme: "http", - Host: "example.com", - Path: "[]!'()*", - }, - "http://example.com/[]!'()*", - }, - // escaped : / ? # [ ] @ in username and password - { - "http://%3A%2F%3F:%23%5B%5D%40@example.com", - &url.URL{ - Scheme: "http", - User: url.UserPassword(":/?", "#[]@"), - Host: "example.com", - }, - "", - }, - // unescaped ! $ & ' ( ) * + , ; = in username and password - { - "http://!$&'():*+,;=@example.com", - &url.URL{ - Scheme: "http", - User: url.UserPassword("!$&'()", "*+,;="), - Host: "example.com", - }, - "", - }, - // unescaped = : / . ? = in query component - { - "http://example.com/?q=http://google.com/?q=", - &url.URL{ - Scheme: "http", - Host: "example.com", - Path: "/", - RawQuery: "q=http://google.com/?q=", - }, - "", - }, - // unescaped : / ? [ ] @ ! $ & ' ( ) * + , ; = in fragment - { - "http://example.com/#:/?%23[]@!$&'()*+,;=", - &url.URL{ - Scheme: "http", - Host: "example.com", - Path: "/", - Fragment: ":/?#[]@!$&'()*+,;=", - }, - "", - }, -} - -func DoTestString(t *testing.T, parse func(string) (*url.URL, error), name string, tests []URLTest) { - for _, tt := range tests { - u, err := parse(tt.in) - if err != nil { - t.Errorf("%s(%q) returned error %s", name, tt.in, err) - continue - } - expected := tt.in - if len(tt.roundtrip) > 0 { - expected = tt.roundtrip - } - s := Escape(u) - if s != expected { - t.Errorf("Escape(%s(%q)) == %q (expected %q)", name, tt.in, s, expected) - } - } -} - -func TestURLString(t *testing.T) { - DoTestString(t, url.Parse, "Parse", urltests) - - // no leading slash on path should prepend - // slash on String() call - noslash := URLTest{ - "http://www.google.com/search", - &url.URL{ - Scheme: "http", - Host: "www.google.com", - Path: "search", - }, - "", - } - s := Escape(noslash.out) - if s != noslash.in { - t.Errorf("Expected %s; go %s", noslash.in, s) - } -} - -type EscapeTest struct { - in string - out string - err error -} - -var escapeTests = []EscapeTest{ - { - "", - "", - nil, - }, - { - "abc", - "abc", - nil, - }, - { - "one two", - "one+two", - nil, - }, - { - "10%", - "10%25", - nil, - }, - { - " ?&=#+%!<>#\"{}|\\^[]`☺\t:/@$'()*,;", - "+?%26%3D%23%2B%25%21%3C%3E%23%22%7B%7D%7C%5C%5E%5B%5D%60%E2%98%BA%09%3A/%40%24%27%28%29%2A%2C%3B", - nil, - }, -} - -func TestEscape(t *testing.T) { - for _, tt := range escapeTests { - actual := QueryEscape(tt.in) - if tt.out != actual { - t.Errorf("QueryEscape(%q) = %q, want %q", tt.in, actual, tt.out) - } - - // for bonus points, verify that escape:unescape is an identity. - roundtrip, err := url.QueryUnescape(actual) - if roundtrip != tt.in || err != nil { - t.Errorf("QueryUnescape(%q) = %q, %s; want %q, %s", actual, roundtrip, err, tt.in, "[no error]") - } - } -} - -var resolveReferenceTests = []struct { - base, rel, expected string -}{ - // Absolute URL references - {"http://foo.com?a=b", "https://bar.com/", "https://bar.com/"}, - {"http://foo.com/", "https://bar.com/?a=b", "https://bar.com/?a=b"}, - {"http://foo.com/bar", "mailto:foo@example.com", "mailto:foo@example.com"}, - - // Path-absolute references - {"http://foo.com/bar", "/baz", "http://foo.com/baz"}, - {"http://foo.com/bar?a=b#f", "/baz", "http://foo.com/baz"}, - {"http://foo.com/bar?a=b", "/baz?c=d", "http://foo.com/baz?c=d"}, - - // Scheme-relative - {"https://foo.com/bar?a=b", "//bar.com/quux", "https://bar.com/quux"}, - - // Path-relative references: - - // ... current directory - {"http://foo.com", ".", "http://foo.com/"}, - {"http://foo.com/bar", ".", "http://foo.com/"}, - {"http://foo.com/bar/", ".", "http://foo.com/bar/"}, - - // ... going down - {"http://foo.com", "bar", "http://foo.com/bar"}, - {"http://foo.com/", "bar", "http://foo.com/bar"}, - {"http://foo.com/bar/baz", "quux", "http://foo.com/bar/quux"}, - - // ... going up - {"http://foo.com/bar/baz", "../quux", "http://foo.com/quux"}, - {"http://foo.com/bar/baz", "../../../../../quux", "http://foo.com/quux"}, - {"http://foo.com/bar", "..", "http://foo.com/"}, - {"http://foo.com/bar/baz", "./..", "http://foo.com/"}, - // ".." in the middle (issue 3560) - {"http://foo.com/bar/baz", "quux/dotdot/../tail", "http://foo.com/bar/quux/tail"}, - {"http://foo.com/bar/baz", "quux/./dotdot/../tail", "http://foo.com/bar/quux/tail"}, - {"http://foo.com/bar/baz", "quux/./dotdot/.././tail", "http://foo.com/bar/quux/tail"}, - {"http://foo.com/bar/baz", "quux/./dotdot/./../tail", "http://foo.com/bar/quux/tail"}, - {"http://foo.com/bar/baz", "quux/./dotdot/dotdot/././../../tail", "http://foo.com/bar/quux/tail"}, - {"http://foo.com/bar/baz", "quux/./dotdot/dotdot/./.././../tail", "http://foo.com/bar/quux/tail"}, - {"http://foo.com/bar/baz", "quux/./dotdot/dotdot/dotdot/./../../.././././tail", "http://foo.com/bar/quux/tail"}, - {"http://foo.com/bar/baz", "quux/./dotdot/../dotdot/../dot/./tail/..", "http://foo.com/bar/quux/dot/"}, - - // Remove any dot-segments prior to forming the target URI. - // http://tools.ietf.org/html/rfc3986#section-5.2.4 - {"http://foo.com/dot/./dotdot/../foo/bar", "../baz", "http://foo.com/dot/baz"}, - - // Triple dot isn't special - {"http://foo.com/bar", "...", "http://foo.com/..."}, - - // Fragment - {"http://foo.com/bar", ".#frag", "http://foo.com/#frag"}, - - // RFC 3986: Normal Examples - // http://tools.ietf.org/html/rfc3986#section-5.4.1 - {"http://a/b/c/d;p?q", "g:h", "g:h"}, - {"http://a/b/c/d;p?q", "g", "http://a/b/c/g"}, - {"http://a/b/c/d;p?q", "./g", "http://a/b/c/g"}, - {"http://a/b/c/d;p?q", "g/", "http://a/b/c/g/"}, - {"http://a/b/c/d;p?q", "/g", "http://a/g"}, - {"http://a/b/c/d;p?q", "//g", "http://g"}, - {"http://a/b/c/d;p?q", "?y", "http://a/b/c/d;p?y"}, - {"http://a/b/c/d;p?q", "g?y", "http://a/b/c/g?y"}, - {"http://a/b/c/d;p?q", "#s", "http://a/b/c/d;p?q#s"}, - {"http://a/b/c/d;p?q", "g#s", "http://a/b/c/g#s"}, - {"http://a/b/c/d;p?q", "g?y#s", "http://a/b/c/g?y#s"}, - {"http://a/b/c/d;p?q", ";x", "http://a/b/c/;x"}, - {"http://a/b/c/d;p?q", "g;x", "http://a/b/c/g;x"}, - {"http://a/b/c/d;p?q", "g;x?y#s", "http://a/b/c/g;x?y#s"}, - {"http://a/b/c/d;p?q", "", "http://a/b/c/d;p?q"}, - {"http://a/b/c/d;p?q", ".", "http://a/b/c/"}, - {"http://a/b/c/d;p?q", "./", "http://a/b/c/"}, - {"http://a/b/c/d;p?q", "..", "http://a/b/"}, - {"http://a/b/c/d;p?q", "../", "http://a/b/"}, - {"http://a/b/c/d;p?q", "../g", "http://a/b/g"}, - {"http://a/b/c/d;p?q", "../..", "http://a/"}, - {"http://a/b/c/d;p?q", "../../", "http://a/"}, - {"http://a/b/c/d;p?q", "../../g", "http://a/g"}, - - // RFC 3986: Abnormal Examples - // http://tools.ietf.org/html/rfc3986#section-5.4.2 - {"http://a/b/c/d;p?q", "../../../g", "http://a/g"}, - {"http://a/b/c/d;p?q", "../../../../g", "http://a/g"}, - {"http://a/b/c/d;p?q", "/./g", "http://a/g"}, - {"http://a/b/c/d;p?q", "/../g", "http://a/g"}, - {"http://a/b/c/d;p?q", "g.", "http://a/b/c/g."}, - {"http://a/b/c/d;p?q", ".g", "http://a/b/c/.g"}, - {"http://a/b/c/d;p?q", "g..", "http://a/b/c/g.."}, - {"http://a/b/c/d;p?q", "..g", "http://a/b/c/..g"}, - {"http://a/b/c/d;p?q", "./../g", "http://a/b/g"}, - {"http://a/b/c/d;p?q", "./g/.", "http://a/b/c/g/"}, - {"http://a/b/c/d;p?q", "g/./h", "http://a/b/c/g/h"}, - {"http://a/b/c/d;p?q", "g/../h", "http://a/b/c/h"}, - {"http://a/b/c/d;p?q", "g;x=1/./y", "http://a/b/c/g;x=1/y"}, - {"http://a/b/c/d;p?q", "g;x=1/../y", "http://a/b/c/y"}, - {"http://a/b/c/d;p?q", "g?y/./x", "http://a/b/c/g?y/./x"}, - {"http://a/b/c/d;p?q", "g?y/../x", "http://a/b/c/g?y/../x"}, - {"http://a/b/c/d;p?q", "g#s/./x", "http://a/b/c/g#s/./x"}, - {"http://a/b/c/d;p?q", "g#s/../x", "http://a/b/c/g#s/../x"}, - - // Extras. - {"https://a/b/c/d;p?q", "//g?q", "https://g?q"}, - {"https://a/b/c/d;p?q", "//g#s", "https://g#s"}, - {"https://a/b/c/d;p?q", "//g/d/e/f?y#s", "https://g/d/e/f?y#s"}, - {"https://a/b/c/d;p#s", "?y", "https://a/b/c/d;p?y"}, - {"https://a/b/c/d;p?q#s", "?y", "https://a/b/c/d;p?y"}, -} - -func TestResolveReference(t *testing.T) { - mustParse := func(url_ string) *url.URL { - u, err := url.Parse(url_) - if err != nil { - t.Fatalf("Expected URL to parse: %q, got error: %v", url_, err) - } - return u - } - opaque := &url.URL{Scheme: "scheme", Opaque: "opaque"} - for _, test := range resolveReferenceTests { - base := mustParse(test.base) - rel := mustParse(test.rel) - url := base.ResolveReference(rel) - if Escape(url) != test.expected { - t.Errorf("URL(%q).ResolveReference(%q) == %q, got %q", test.base, test.rel, test.expected, Escape(url)) - } - // Ensure that new instances are returned. - if base == url { - t.Errorf("Expected URL.ResolveReference to return new URL instance.") - } - // Test the convenience wrapper too. - url, err := base.Parse(test.rel) - if err != nil { - t.Errorf("URL(%q).Parse(%q) failed: %v", test.base, test.rel, err) - } else if Escape(url) != test.expected { - t.Errorf("URL(%q).Parse(%q) == %q, got %q", test.base, test.rel, test.expected, Escape(url)) - } else if base == url { - // Ensure that new instances are returned for the wrapper too. - t.Errorf("Expected URL.Parse to return new URL instance.") - } - // Ensure Opaque resets the URL. - url = base.ResolveReference(opaque) - if *url != *opaque { - t.Errorf("ResolveReference failed to resolve opaque URL: want %#v, got %#v", url, opaque) - } - // Test the convenience wrapper with an opaque URL too. - url, err = base.Parse("scheme:opaque") - if err != nil { - t.Errorf(`URL(%q).Parse("scheme:opaque") failed: %v`, test.base, err) - } else if *url != *opaque { - t.Errorf("Parse failed to resolve opaque URL: want %#v, got %#v", url, opaque) - } else if base == url { - // Ensure that new instances are returned, again. - t.Errorf("Expected URL.Parse to return new URL instance.") - } - } -} - -type shouldEscapeTest struct { - in byte - mode encoding - escape bool -} - -var shouldEscapeTests = []shouldEscapeTest{ - // Unreserved characters (§2.3) - {'a', encodePath, false}, - {'a', encodeUserPassword, false}, - {'a', encodeQueryComponent, false}, - {'a', encodeFragment, false}, - {'z', encodePath, false}, - {'A', encodePath, false}, - {'Z', encodePath, false}, - {'0', encodePath, false}, - {'9', encodePath, false}, - {'-', encodePath, false}, - {'-', encodeUserPassword, false}, - {'-', encodeQueryComponent, false}, - {'-', encodeFragment, false}, - {'.', encodePath, false}, - {'_', encodePath, false}, - {'~', encodePath, false}, - - // User information (§3.2.1) - {':', encodeUserPassword, true}, - {'/', encodeUserPassword, true}, - {'?', encodeUserPassword, true}, - {'@', encodeUserPassword, true}, - {'$', encodeUserPassword, false}, - {'&', encodeUserPassword, false}, - {'+', encodeUserPassword, false}, - {',', encodeUserPassword, false}, - {';', encodeUserPassword, false}, - {'=', encodeUserPassword, false}, -} - -func TestShouldEscape(t *testing.T) { - for _, tt := range shouldEscapeTests { - if shouldEscape(tt.in, tt.mode) != tt.escape { - t.Errorf("shouldEscape(%q, %v) returned %v; expected %v", tt.in, tt.mode, !tt.escape, tt.escape) - } - } -} diff --git a/vendor/github.com/PuerkitoBio/purell/.gitignore b/vendor/github.com/PuerkitoBio/purell/.gitignore new file mode 100644 index 0000000..748e4c8 --- /dev/null +++ b/vendor/github.com/PuerkitoBio/purell/.gitignore @@ -0,0 +1,5 @@ +*.sublime-* +.DS_Store +*.swp +*.swo +tags diff --git a/vendor/github.com/PuerkitoBio/purell/LICENSE b/vendor/github.com/PuerkitoBio/purell/LICENSE new file mode 100644 index 0000000..640c3ac --- /dev/null +++ b/vendor/github.com/PuerkitoBio/purell/LICENSE @@ -0,0 +1,13 @@ +Copyright (c) 2012-2022, The Go Authors +Copyright (c) 2012-2022, Martin Angers, Yuki Okushi & Contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +* Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/PuerkitoBio/purell/README.md b/vendor/github.com/PuerkitoBio/purell/README.md new file mode 100644 index 0000000..8461279 --- /dev/null +++ b/vendor/github.com/PuerkitoBio/purell/README.md @@ -0,0 +1,188 @@ +# Purell + +Purell is a tiny Go library to normalize URLs. It returns a pure URL. Pure-ell. Sanitizer and all. Yeah, I know... + +Based on the [wikipedia paper][wiki] and the [RFC 3986 document][rfc]. + +[![CI](https://github.com/PuerkitoBio/purell/actions/workflows/ci.yml/badge.svg)](https://github.com/PuerkitoBio/purell/actions/workflows/ci.yml) + +## Install + +`go get github.com/PuerkitoBio/purell` + +## Changelog + +* **v1.1.1** : Fix failing test due to Go1.12 changes (thanks to @ianlancetaylor). +* **2016-11-14 (v1.1.0)** : IDN: Conform to RFC 5895: Fold character width (thanks to @beeker1121). +* **2016-07-27 (v1.0.0)** : Normalize IDN to ASCII (thanks to @zenovich). +* **2015-02-08** : Add fix for relative paths issue ([PR #5][pr5]) and add fix for unnecessary encoding of reserved characters ([see issue #7][iss7]). +* **v0.2.0** : Add benchmarks, Attempt IDN support. +* **v0.1.0** : Initial release. + +## Examples + +From `example_test.go` (note that in your code, you would import "github.com/PuerkitoBio/purell", and would prefix references to its methods and constants with "purell."): + +```go +package purell + +import ( + "fmt" + "net/url" +) + +func ExampleNormalizeURLString() { + if normalized, err := NormalizeURLString("hTTp://someWEBsite.com:80/Amazing%3f/url/", + FlagLowercaseScheme|FlagLowercaseHost|FlagUppercaseEscapes); err != nil { + panic(err) + } else { + fmt.Print(normalized) + } + // Output: http://somewebsite.com:80/Amazing%3F/url/ +} + +func ExampleMustNormalizeURLString() { + normalized := MustNormalizeURLString("hTTpS://someWEBsite.com:443/Amazing%fa/url/", + FlagsUnsafeGreedy) + fmt.Print(normalized) + + // Output: http://somewebsite.com/Amazing%FA/url +} + +func ExampleNormalizeURL() { + if u, err := url.Parse("Http://SomeUrl.com:8080/a/b/.././c///g?c=3&a=1&b=9&c=0#target"); err != nil { + panic(err) + } else { + normalized := NormalizeURL(u, FlagsUsuallySafeGreedy|FlagRemoveDuplicateSlashes|FlagRemoveFragment) + fmt.Print(normalized) + } + + // Output: http://someurl.com:8080/a/c/g?c=3&a=1&b=9&c=0 +} +``` + +## API + +As seen in the examples above, purell offers three methods, `NormalizeURLString(string, NormalizationFlags) (string, error)`, `MustNormalizeURLString(string, NormalizationFlags) (string)` and `NormalizeURL(*url.URL, NormalizationFlags) (string)`. They all normalize the provided URL based on the specified flags. Here are the available flags: + +```go +const ( + // Safe normalizations + FlagLowercaseScheme NormalizationFlags = 1 << iota // HTTP://host -> http://host, applied by default in Go1.1 + FlagLowercaseHost // http://HOST -> http://host + FlagUppercaseEscapes // http://host/t%ef -> http://host/t%EF + FlagDecodeUnnecessaryEscapes // http://host/t%41 -> http://host/tA + FlagEncodeNecessaryEscapes // http://host/!"#$ -> http://host/%21%22#$ + FlagRemoveDefaultPort // http://host:80 -> http://host + FlagRemoveEmptyQuerySeparator // http://host/path? -> http://host/path + + // Usually safe normalizations + FlagRemoveTrailingSlash // http://host/path/ -> http://host/path + FlagAddTrailingSlash // http://host/path -> http://host/path/ (should choose only one of these add/remove trailing slash flags) + FlagRemoveDotSegments // http://host/path/./a/b/../c -> http://host/path/a/c + + // Unsafe normalizations + FlagRemoveDirectoryIndex // http://host/path/index.html -> http://host/path/ + FlagRemoveFragment // http://host/path#fragment -> http://host/path + FlagForceHTTP // https://host -> http://host + FlagRemoveDuplicateSlashes // http://host/path//a///b -> http://host/path/a/b + FlagRemoveWWW // http://www.host/ -> http://host/ + FlagAddWWW // http://host/ -> http://www.host/ (should choose only one of these add/remove WWW flags) + FlagSortQuery // http://host/path?c=3&b=2&a=1&b=1 -> http://host/path?a=1&b=1&b=2&c=3 + + // Normalizations not in the wikipedia article, required to cover tests cases + // submitted by jehiah + FlagDecodeDWORDHost // http://1113982867 -> http://66.102.7.147 + FlagDecodeOctalHost // http://0102.0146.07.0223 -> http://66.102.7.147 + FlagDecodeHexHost // http://0x42660793 -> http://66.102.7.147 + FlagRemoveUnnecessaryHostDots // http://.host../path -> http://host/path + FlagRemoveEmptyPortSeparator // http://host:/path -> http://host/path + + // Convenience set of safe normalizations + FlagsSafe NormalizationFlags = FlagLowercaseHost | FlagLowercaseScheme | FlagUppercaseEscapes | FlagDecodeUnnecessaryEscapes | FlagEncodeNecessaryEscapes | FlagRemoveDefaultPort | FlagRemoveEmptyQuerySeparator + + // For convenience sets, "greedy" uses the "remove trailing slash" and "remove www. prefix" flags, + // while "non-greedy" uses the "add (or keep) the trailing slash" and "add www. prefix". + + // Convenience set of usually safe normalizations (includes FlagsSafe) + FlagsUsuallySafeGreedy NormalizationFlags = FlagsSafe | FlagRemoveTrailingSlash | FlagRemoveDotSegments + FlagsUsuallySafeNonGreedy NormalizationFlags = FlagsSafe | FlagAddTrailingSlash | FlagRemoveDotSegments + + // Convenience set of unsafe normalizations (includes FlagsUsuallySafe) + FlagsUnsafeGreedy NormalizationFlags = FlagsUsuallySafeGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagRemoveWWW | FlagSortQuery + FlagsUnsafeNonGreedy NormalizationFlags = FlagsUsuallySafeNonGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagAddWWW | FlagSortQuery + + // Convenience set of all available flags + FlagsAllGreedy = FlagsUnsafeGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator + FlagsAllNonGreedy = FlagsUnsafeNonGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator +) +``` + +For convenience, the set of flags `FlagsSafe`, `FlagsUsuallySafe[Greedy|NonGreedy]`, `FlagsUnsafe[Greedy|NonGreedy]` and `FlagsAll[Greedy|NonGreedy]` are provided for the similarly grouped normalizations on [wikipedia's URL normalization page][wiki]. You can add (using the bitwise OR `|` operator) or remove (using the bitwise AND NOT `&^` operator) individual flags from the sets if required, to build your own custom set. + +The [full godoc reference is available on gopkgdoc][godoc]. + +Some things to note: + +* `FlagDecodeUnnecessaryEscapes`, `FlagEncodeNecessaryEscapes`, `FlagUppercaseEscapes` and `FlagRemoveEmptyQuerySeparator` are always implicitly set, because internally, the URL string is parsed as an URL object, which automatically decodes unnecessary escapes, uppercases and encodes necessary ones, and removes empty query separators (an unnecessary `?` at the end of the url). So this operation cannot **not** be done. For this reason, `FlagRemoveEmptyQuerySeparator` (as well as the other three) has been included in the `FlagsSafe` convenience set, instead of `FlagsUnsafe`, where Wikipedia puts it. + +* The `FlagDecodeUnnecessaryEscapes` decodes the following escapes (*from -> to*): + - %24 -> $ + - %26 -> & + - %2B-%3B -> +,-./0123456789:; + - %3D -> = + - %40-%5A -> @ABCDEFGHIJKLMNOPQRSTUVWXYZ + - %5F -> _ + - %61-%7A -> abcdefghijklmnopqrstuvwxyz + - %7E -> ~ + + +* When the `NormalizeURL` function is used (passing an URL object), this source URL object is modified (that is, after the call, the URL object will be modified to reflect the normalization). + +* The *replace IP with domain name* normalization (`http://208.77.188.166/ → http://www.example.com/`) is obviously not possible for a library without making some network requests. This is not implemented in purell. + +* The *remove unused query string parameters* and *remove default query parameters* are also not implemented, since this is a very case-specific normalization, and it is quite trivial to do with an URL object. + +### Safe vs Usually Safe vs Unsafe + +Purell allows you to control the level of risk you take while normalizing an URL. You can aggressively normalize, play it totally safe, or anything in between. + +Consider the following URL: + +`HTTPS://www.RooT.com/toto/t%45%1f///a/./b/../c/?z=3&w=2&a=4&w=1#invalid` + +Normalizing with the `FlagsSafe` gives: + +`https://www.root.com/toto/tE%1F///a/./b/../c/?z=3&w=2&a=4&w=1#invalid` + +With the `FlagsUsuallySafeGreedy`: + +`https://www.root.com/toto/tE%1F///a/c?z=3&w=2&a=4&w=1#invalid` + +And with `FlagsUnsafeGreedy`: + +`http://root.com/toto/tE%1F/a/c?a=4&w=1&w=2&z=3` + +## TODOs + +* Add a class/default instance to allow specifying custom directory index names? At the moment, removing directory index removes `(^|/)((?:default|index)\.\w{1,4})$`. + +## Thanks / Contributions + +@rogpeppe +@jehiah +@opennota +@pchristopher1275 +@zenovich +@beeker1121 + +## License + +The [BSD 3-Clause license][bsd]. + +[bsd]: http://opensource.org/licenses/BSD-3-Clause +[wiki]: http://en.wikipedia.org/wiki/URL_normalization +[rfc]: http://tools.ietf.org/html/rfc3986#section-6 +[godoc]: http://go.pkgdoc.org/github.com/PuerkitoBio/purell +[pr5]: https://github.com/PuerkitoBio/purell/pull/5 +[iss7]: https://github.com/PuerkitoBio/purell/issues/7 diff --git a/vendor/github.com/PuerkitoBio/purell/purell.go b/vendor/github.com/PuerkitoBio/purell/purell.go new file mode 100644 index 0000000..74c8272 --- /dev/null +++ b/vendor/github.com/PuerkitoBio/purell/purell.go @@ -0,0 +1,378 @@ +/* +Package purell offers URL normalization as described on the wikipedia page: +http://en.wikipedia.org/wiki/URL_normalization +*/ +package purell + +import ( + "bytes" + "fmt" + "net/url" + "regexp" + "sort" + "strconv" + "strings" + + "golang.org/x/net/idna" + "golang.org/x/text/unicode/norm" + "golang.org/x/text/width" +) + +// A set of normalization flags determines how a URL will +// be normalized. +type NormalizationFlags uint + +const ( + // Safe normalizations + FlagLowercaseScheme NormalizationFlags = 1 << iota // HTTP://host -> http://host, applied by default in Go1.1 + FlagLowercaseHost // http://HOST -> http://host + FlagUppercaseEscapes // http://host/t%ef -> http://host/t%EF + FlagDecodeUnnecessaryEscapes // http://host/t%41 -> http://host/tA + FlagEncodeNecessaryEscapes // http://host/!"#$ -> http://host/%21%22#$ + FlagRemoveDefaultPort // http://host:80 -> http://host + FlagRemoveEmptyQuerySeparator // http://host/path? -> http://host/path + + // Usually safe normalizations + FlagRemoveTrailingSlash // http://host/path/ -> http://host/path + FlagAddTrailingSlash // http://host/path -> http://host/path/ (should choose only one of these add/remove trailing slash flags) + FlagRemoveDotSegments // http://host/path/./a/b/../c -> http://host/path/a/c + + // Unsafe normalizations + FlagRemoveDirectoryIndex // http://host/path/index.html -> http://host/path/ + FlagRemoveFragment // http://host/path#fragment -> http://host/path + FlagForceHTTP // https://host -> http://host + FlagRemoveDuplicateSlashes // http://host/path//a///b -> http://host/path/a/b + FlagRemoveWWW // http://www.host/ -> http://host/ + FlagAddWWW // http://host/ -> http://www.host/ (should choose only one of these add/remove WWW flags) + FlagSortQuery // http://host/path?c=3&b=2&a=1&b=1 -> http://host/path?a=1&b=1&b=2&c=3 + + // Normalizations not in the wikipedia article, required to cover tests cases + // submitted by jehiah + FlagDecodeDWORDHost // http://1113982867 -> http://66.102.7.147 + FlagDecodeOctalHost // http://0102.0146.07.0223 -> http://66.102.7.147 + FlagDecodeHexHost // http://0x42660793 -> http://66.102.7.147 + FlagRemoveUnnecessaryHostDots // http://.host../path -> http://host/path + FlagRemoveEmptyPortSeparator // http://host:/path -> http://host/path + + // Convenience set of safe normalizations + FlagsSafe NormalizationFlags = FlagLowercaseHost | FlagLowercaseScheme | FlagUppercaseEscapes | FlagDecodeUnnecessaryEscapes | FlagEncodeNecessaryEscapes | FlagRemoveDefaultPort | FlagRemoveEmptyQuerySeparator + + // For convenience sets, "greedy" uses the "remove trailing slash" and "remove www. prefix" flags, + // while "non-greedy" uses the "add (or keep) the trailing slash" and "add www. prefix". + + // Convenience set of usually safe normalizations (includes FlagsSafe) + FlagsUsuallySafeGreedy NormalizationFlags = FlagsSafe | FlagRemoveTrailingSlash | FlagRemoveDotSegments + FlagsUsuallySafeNonGreedy NormalizationFlags = FlagsSafe | FlagAddTrailingSlash | FlagRemoveDotSegments + + // Convenience set of unsafe normalizations (includes FlagsUsuallySafe) + FlagsUnsafeGreedy NormalizationFlags = FlagsUsuallySafeGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagRemoveWWW | FlagSortQuery + FlagsUnsafeNonGreedy NormalizationFlags = FlagsUsuallySafeNonGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagAddWWW | FlagSortQuery + + // Convenience set of all available flags + FlagsAllGreedy = FlagsUnsafeGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator + FlagsAllNonGreedy = FlagsUnsafeNonGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator +) + +const ( + defaultHttpPort = ":80" + defaultHttpsPort = ":443" +) + +// Regular expressions used by the normalizations +var rxPort = regexp.MustCompile(`(:\d+)/?$`) +var rxDirIndex = regexp.MustCompile(`(^|/)((?:default|index)\.\w{1,4})$`) +var rxDupSlashes = regexp.MustCompile(`/{2,}`) +var rxDWORDHost = regexp.MustCompile(`^(\d+)((?:\.+)?(?:\:\d*)?)$`) +var rxOctalHost = regexp.MustCompile(`^(0\d*)\.(0\d*)\.(0\d*)\.(0\d*)((?:\.+)?(?:\:\d*)?)$`) +var rxHexHost = regexp.MustCompile(`^0x([0-9A-Fa-f]+)((?:\.+)?(?:\:\d*)?)$`) +var rxHostDots = regexp.MustCompile(`^(.+?)(:\d+)?$`) +var rxEmptyPort = regexp.MustCompile(`:+$`) + +// Map of flags to implementation function. +// FlagDecodeUnnecessaryEscapes has no action, since it is done automatically +// by parsing the string as an URL. Same for FlagUppercaseEscapes and FlagRemoveEmptyQuerySeparator. + +// Since maps have undefined traversing order, make a slice of ordered keys +var flagsOrder = []NormalizationFlags{ + FlagLowercaseScheme, + FlagLowercaseHost, + FlagRemoveDefaultPort, + FlagRemoveDirectoryIndex, + FlagRemoveDotSegments, + FlagRemoveFragment, + FlagForceHTTP, // Must be after remove default port (because https=443/http=80) + FlagRemoveDuplicateSlashes, + FlagRemoveWWW, + FlagAddWWW, + FlagSortQuery, + FlagDecodeDWORDHost, + FlagDecodeOctalHost, + FlagDecodeHexHost, + FlagRemoveUnnecessaryHostDots, + FlagRemoveEmptyPortSeparator, + FlagRemoveTrailingSlash, // These two (add/remove trailing slash) must be last + FlagAddTrailingSlash, +} + +// ... and then the map, where order is unimportant +var flags = map[NormalizationFlags]func(*url.URL){ + FlagLowercaseScheme: lowercaseScheme, + FlagLowercaseHost: lowercaseHost, + FlagRemoveDefaultPort: removeDefaultPort, + FlagRemoveDirectoryIndex: removeDirectoryIndex, + FlagRemoveDotSegments: removeDotSegments, + FlagRemoveFragment: removeFragment, + FlagForceHTTP: forceHTTP, + FlagRemoveDuplicateSlashes: removeDuplicateSlashes, + FlagRemoveWWW: removeWWW, + FlagAddWWW: addWWW, + FlagSortQuery: sortQuery, + FlagDecodeDWORDHost: decodeDWORDHost, + FlagDecodeOctalHost: decodeOctalHost, + FlagDecodeHexHost: decodeHexHost, + FlagRemoveUnnecessaryHostDots: removeUnncessaryHostDots, + FlagRemoveEmptyPortSeparator: removeEmptyPortSeparator, + FlagRemoveTrailingSlash: removeTrailingSlash, + FlagAddTrailingSlash: addTrailingSlash, +} + +// MustNormalizeURLString returns the normalized string, and panics if an error occurs. +// It takes an URL string as input, as well as the normalization flags. +func MustNormalizeURLString(u string, f NormalizationFlags) string { + result, e := NormalizeURLString(u, f) + if e != nil { + panic(e) + } + return result +} + +// NormalizeURLString returns the normalized string, or an error if it can't be parsed into an URL object. +// It takes an URL string as input, as well as the normalization flags. +func NormalizeURLString(u string, f NormalizationFlags) (string, error) { + parsed, err := url.Parse(u) + if err != nil { + return "", err + } + + if f&FlagLowercaseHost == FlagLowercaseHost { + parsed.Host = strings.ToLower(parsed.Host) + } + + // The idna package doesn't fully conform to RFC 5895 + // (https://tools.ietf.org/html/rfc5895), so we do it here. + // Taken from Go 1.8 cycle source, courtesy of bradfitz. + // TODO: Remove when (if?) idna package conforms to RFC 5895. + parsed.Host = width.Fold.String(parsed.Host) + parsed.Host = norm.NFC.String(parsed.Host) + if parsed.Host, err = idna.ToASCII(parsed.Host); err != nil { + return "", err + } + + return NormalizeURL(parsed, f), nil +} + +// NormalizeURL returns the normalized string. +// It takes a parsed URL object as input, as well as the normalization flags. +func NormalizeURL(u *url.URL, f NormalizationFlags) string { + for _, k := range flagsOrder { + if f&k == k { + flags[k](u) + } + } + return escapeURL(u) +} + +func lowercaseScheme(u *url.URL) { + if len(u.Scheme) > 0 { + u.Scheme = strings.ToLower(u.Scheme) + } +} + +func lowercaseHost(u *url.URL) { + if len(u.Host) > 0 { + u.Host = strings.ToLower(u.Host) + } +} + +func removeDefaultPort(u *url.URL) { + if len(u.Host) > 0 { + scheme := strings.ToLower(u.Scheme) + u.Host = rxPort.ReplaceAllStringFunc(u.Host, func(val string) string { + if (scheme == "http" && val == defaultHttpPort) || (scheme == "https" && val == defaultHttpsPort) { + return "" + } + return val + }) + } +} + +func removeTrailingSlash(u *url.URL) { + if l := len(u.Path); l > 0 { + if strings.HasSuffix(u.Path, "/") { + u.Path = u.Path[:l-1] + } + } else if l = len(u.Host); l > 0 { + if strings.HasSuffix(u.Host, "/") { + u.Host = u.Host[:l-1] + } + } +} + +func addTrailingSlash(u *url.URL) { + if l := len(u.Path); l > 0 { + if !strings.HasSuffix(u.Path, "/") { + u.Path += "/" + } + } else if l = len(u.Host); l > 0 { + if !strings.HasSuffix(u.Host, "/") { + u.Host += "/" + } + } +} + +func removeDotSegments(u *url.URL) { + if len(u.Path) > 0 { + var dotFree []string + var lastIsDot bool + + sections := strings.Split(u.Path, "/") + for _, s := range sections { + if s == ".." { + if len(dotFree) > 0 { + dotFree = dotFree[:len(dotFree)-1] + } + } else if s != "." { + dotFree = append(dotFree, s) + } + lastIsDot = (s == "." || s == "..") + } + // Special case if host does not end with / and new path does not begin with / + u.Path = strings.Join(dotFree, "/") + if u.Host != "" && !strings.HasSuffix(u.Host, "/") && !strings.HasPrefix(u.Path, "/") { + u.Path = "/" + u.Path + } + // Special case if the last segment was a dot, make sure the path ends with a slash + if lastIsDot && !strings.HasSuffix(u.Path, "/") { + u.Path += "/" + } + } +} + +func removeDirectoryIndex(u *url.URL) { + if len(u.Path) > 0 { + u.Path = rxDirIndex.ReplaceAllString(u.Path, "$1") + } +} + +func removeFragment(u *url.URL) { + u.Fragment = "" +} + +func forceHTTP(u *url.URL) { + if strings.ToLower(u.Scheme) == "https" { + u.Scheme = "http" + } +} + +func removeDuplicateSlashes(u *url.URL) { + if len(u.Path) > 0 { + u.Path = rxDupSlashes.ReplaceAllString(u.Path, "/") + } +} + +func removeWWW(u *url.URL) { + if len(u.Host) > 0 && strings.HasPrefix(strings.ToLower(u.Host), "www.") { + u.Host = u.Host[4:] + } +} + +func addWWW(u *url.URL) { + if len(u.Host) > 0 && !strings.HasPrefix(strings.ToLower(u.Host), "www.") { + u.Host = "www." + u.Host + } +} + +func sortQuery(u *url.URL) { + q := u.Query() + + if len(q) > 0 { + arKeys := make([]string, len(q)) + i := 0 + for k := range q { + arKeys[i] = k + i++ + } + sort.Strings(arKeys) + buf := new(bytes.Buffer) + for _, k := range arKeys { + sort.Strings(q[k]) + for _, v := range q[k] { + if buf.Len() > 0 { + buf.WriteRune('&') + } + buf.WriteString(fmt.Sprintf("%s=%s", k, url.QueryEscape(v))) + } + } + + // Rebuild the raw query string + u.RawQuery = buf.String() + } +} + +func decodeDWORDHost(u *url.URL) { + if len(u.Host) > 0 { + if matches := rxDWORDHost.FindStringSubmatch(u.Host); len(matches) > 2 { + var parts [4]int64 + + dword, _ := strconv.ParseInt(matches[1], 10, 0) + for i, shift := range []uint{24, 16, 8, 0} { + parts[i] = dword >> shift & 0xFF + } + u.Host = fmt.Sprintf("%d.%d.%d.%d%s", parts[0], parts[1], parts[2], parts[3], matches[2]) + } + } +} + +func decodeOctalHost(u *url.URL) { + if len(u.Host) > 0 { + if matches := rxOctalHost.FindStringSubmatch(u.Host); len(matches) > 5 { + var parts [4]int64 + + for i := 1; i <= 4; i++ { + parts[i-1], _ = strconv.ParseInt(matches[i], 8, 0) + } + u.Host = fmt.Sprintf("%d.%d.%d.%d%s", parts[0], parts[1], parts[2], parts[3], matches[5]) + } + } +} + +func decodeHexHost(u *url.URL) { + if len(u.Host) > 0 { + if matches := rxHexHost.FindStringSubmatch(u.Host); len(matches) > 2 { + // Conversion is safe because of regex validation + parsed, _ := strconv.ParseInt(matches[1], 16, 0) + // Set host as DWORD (base 10) encoded host + u.Host = fmt.Sprintf("%d%s", parsed, matches[2]) + // The rest is the same as decoding a DWORD host + decodeDWORDHost(u) + } + } +} + +func removeUnncessaryHostDots(u *url.URL) { + if len(u.Host) > 0 { + if matches := rxHostDots.FindStringSubmatch(u.Host); len(matches) > 1 { + // Trim the leading and trailing dots + u.Host = strings.Trim(matches[1], ".") + if len(matches) > 2 { + u.Host += matches[2] + } + } + } +} + +func removeEmptyPortSeparator(u *url.URL) { + if len(u.Host) > 0 { + u.Host = rxEmptyPort.ReplaceAllString(u.Host, "") + } +} diff --git a/vendor/github.com/PuerkitoBio/purell/urlesc.go b/vendor/github.com/PuerkitoBio/purell/urlesc.go new file mode 100644 index 0000000..53d815f --- /dev/null +++ b/vendor/github.com/PuerkitoBio/purell/urlesc.go @@ -0,0 +1,174 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements query escaping as per RFC 3986. +// It contains some parts of the net/url package, modified so as to allow +// some reserved characters incorrectly escaped by net/url. +// See https://github.com/golang/go/issues/5684 +package purell + +import ( + "bytes" + "net/url" + "strings" +) + +type encoding int + +const ( + encodePath encoding = 1 + iota + encodeUserPassword + encodeQueryComponent + encodeFragment +) + +// Return true if the specified character should be escaped when +// appearing in a URL string, according to RFC 3986. +func shouldEscape(c byte, mode encoding) bool { + // §2.3 Unreserved characters (alphanum) + if 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' { + return false + } + + switch c { + case '-', '.', '_', '~': // §2.3 Unreserved characters (mark) + return false + + // §2.2 Reserved characters (reserved) + case ':', '/', '?', '#', '[', ']', '@', // gen-delims + '!', '$', '&', '\'', '(', ')', '*', '+', ',', ';', '=': // sub-delims + // Different sections of the URL allow a few of + // the reserved characters to appear unescaped. + switch mode { + case encodePath: // §3.3 + // The RFC allows sub-delims and : @. + // '/', '[' and ']' can be used to assign meaning to individual path + // segments. This package only manipulates the path as a whole, + // so we allow those as well. That leaves only ? and # to escape. + return c == '?' || c == '#' + + case encodeUserPassword: // §3.2.1 + // The RFC allows : and sub-delims in + // userinfo. The parsing of userinfo treats ':' as special so we must escape + // all the gen-delims. + return c == ':' || c == '/' || c == '?' || c == '#' || c == '[' || c == ']' || c == '@' + + case encodeQueryComponent: // §3.4 + // The RFC allows / and ?. + return c != '/' && c != '?' + + case encodeFragment: // §4.1 + // The RFC text is silent but the grammar allows + // everything, so escape nothing but # + return c == '#' + } + } + + // Everything else must be escaped. + return true +} + +func escape(s string, mode encoding) string { + spaceCount, hexCount := 0, 0 + for i := 0; i < len(s); i++ { + c := s[i] + if shouldEscape(c, mode) { + if c == ' ' && mode == encodeQueryComponent { + spaceCount++ + } else { + hexCount++ + } + } + } + + if spaceCount == 0 && hexCount == 0 { + return s + } + + t := make([]byte, len(s)+2*hexCount) + j := 0 + for i := 0; i < len(s); i++ { + switch c := s[i]; { + case c == ' ' && mode == encodeQueryComponent: + t[j] = '+' + j++ + case shouldEscape(c, mode): + t[j] = '%' + t[j+1] = "0123456789ABCDEF"[c>>4] + t[j+2] = "0123456789ABCDEF"[c&15] + j += 3 + default: + t[j] = s[i] + j++ + } + } + return string(t) +} + +var uiReplacer = strings.NewReplacer( + "%21", "!", + "%27", "'", + "%28", "(", + "%29", ")", + "%2A", "*", +) + +// unescapeUserinfo unescapes some characters that need not to be escaped as per RFC3986. +func unescapeUserinfo(s string) string { + return uiReplacer.Replace(s) +} + +// Escape reassembles the URL into a valid URL string. +// The general form of the result is one of: +// +// scheme:opaque +// scheme://userinfo@host/path?query#fragment +// +// If u.Opaque is non-empty, String uses the first form; +// otherwise it uses the second form. +// +// In the second form, the following rules apply: +// - if u.Scheme is empty, scheme: is omitted. +// - if u.User is nil, userinfo@ is omitted. +// - if u.Host is empty, host/ is omitted. +// - if u.Scheme and u.Host are empty and u.User is nil, +// the entire scheme://userinfo@host/ is omitted. +// - if u.Host is non-empty and u.Path begins with a /, +// the form host/path does not add its own /. +// - if u.RawQuery is empty, ?query is omitted. +// - if u.Fragment is empty, #fragment is omitted. +func escapeURL(u *url.URL) string { + var buf bytes.Buffer + if u.Scheme != "" { + buf.WriteString(u.Scheme) + buf.WriteByte(':') + } + if u.Opaque != "" { + buf.WriteString(u.Opaque) + } else { + if u.Scheme != "" || u.Host != "" || u.User != nil { + buf.WriteString("//") + if ui := u.User; ui != nil { + buf.WriteString(unescapeUserinfo(ui.String())) + buf.WriteByte('@') + } + if h := u.Host; h != "" { + buf.WriteString(h) + } + } + if u.Path != "" && u.Path[0] != '/' && u.Host != "" { + buf.WriteByte('/') + } + buf.WriteString(escape(u.Path, encodePath)) + } + if u.RawQuery != "" { + buf.WriteByte('?') + buf.WriteString(u.RawQuery) + } + if u.Fragment != "" { + buf.WriteByte('#') + buf.WriteString(escape(u.Fragment, encodeFragment)) + } + return buf.String() +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 612467a..e2e285a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,3 +1,6 @@ +# github.com/PuerkitoBio/purell v1.2.0 +## explicit; go 1.19 +github.com/PuerkitoBio/purell # github.com/bwmarrin/discordgo v0.26.1 ## explicit; go 1.13 github.com/bwmarrin/discordgo -- cgit v1.2.3-54-g00ecf