diff options
Diffstat (limited to 'normalize')
| -rw-r--r-- | normalize/LICENSE | 12 | ||||
| -rw-r--r-- | normalize/normalize.go | 379 | ||||
| -rw-r--r-- | normalize/normalize_test.go | 782 |
3 files changed, 1173 insertions, 0 deletions
diff --git a/normalize/LICENSE b/normalize/LICENSE new file mode 100644 index 0000000..4b9986d --- /dev/null +++ b/normalize/LICENSE @@ -0,0 +1,12 @@ +Copyright (c) 2012, Martin Angers +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +* Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/normalize/normalize.go b/normalize/normalize.go new file mode 100644 index 0000000..0220f65 --- /dev/null +++ b/normalize/normalize.go @@ -0,0 +1,379 @@ +/* +Package normalize offers URL normalization as described on the wikipedia page: +http://en.wikipedia.org/wiki/URL_normalization +*/ +package normalize + +import ( + "bytes" + "fmt" + "net/url" + "regexp" + "sort" + "strconv" + "strings" + + "golang.org/x/net/idna" + "golang.org/x/text/unicode/norm" + "golang.org/x/text/width" + "keep/urlesc" +) + +// A set of normalization flags determines how a URL will +// be normalized. +type NormalizationFlags uint + +const ( + // Safe normalizations + FlagLowercaseScheme NormalizationFlags = 1 << iota // HTTP://host -> http://host, applied by default in Go1.1 + FlagLowercaseHost // http://HOST -> http://host + FlagUppercaseEscapes // http://host/t%ef -> http://host/t%EF + FlagDecodeUnnecessaryEscapes // http://host/t%41 -> http://host/tA + FlagEncodeNecessaryEscapes // http://host/!"#$ -> http://host/%21%22#$ + FlagRemoveDefaultPort // http://host:80 -> http://host + FlagRemoveEmptyQuerySeparator // http://host/path? -> http://host/path + + // Usually safe normalizations + FlagRemoveTrailingSlash // http://host/path/ -> http://host/path + FlagAddTrailingSlash // http://host/path -> http://host/path/ (should choose only one of these add/remove trailing slash flags) + FlagRemoveDotSegments // http://host/path/./a/b/../c -> http://host/path/a/c + + // Unsafe normalizations + FlagRemoveDirectoryIndex // http://host/path/index.html -> http://host/path/ + FlagRemoveFragment // http://host/path#fragment -> http://host/path + FlagForceHTTP // https://host -> http://host + FlagRemoveDuplicateSlashes // http://host/path//a///b -> http://host/path/a/b + FlagRemoveWWW // http://www.host/ -> http://host/ + FlagAddWWW // http://host/ -> http://www.host/ (should choose only one of these add/remove WWW flags) + FlagSortQuery // http://host/path?c=3&b=2&a=1&b=1 -> http://host/path?a=1&b=1&b=2&c=3 + + // Normalizations not in the wikipedia article, required to cover tests cases + // submitted by jehiah + FlagDecodeDWORDHost // http://1113982867 -> http://66.102.7.147 + FlagDecodeOctalHost // http://0102.0146.07.0223 -> http://66.102.7.147 + FlagDecodeHexHost // http://0x42660793 -> http://66.102.7.147 + FlagRemoveUnnecessaryHostDots // http://.host../path -> http://host/path + FlagRemoveEmptyPortSeparator // http://host:/path -> http://host/path + + // Convenience set of safe normalizations + FlagsSafe NormalizationFlags = FlagLowercaseHost | FlagLowercaseScheme | FlagUppercaseEscapes | FlagDecodeUnnecessaryEscapes | FlagEncodeNecessaryEscapes | FlagRemoveDefaultPort | FlagRemoveEmptyQuerySeparator + + // For convenience sets, "greedy" uses the "remove trailing slash" and "remove www. prefix" flags, + // while "non-greedy" uses the "add (or keep) the trailing slash" and "add www. prefix". + + // Convenience set of usually safe normalizations (includes FlagsSafe) + FlagsUsuallySafeGreedy NormalizationFlags = FlagsSafe | FlagRemoveTrailingSlash | FlagRemoveDotSegments + FlagsUsuallySafeNonGreedy NormalizationFlags = FlagsSafe | FlagAddTrailingSlash | FlagRemoveDotSegments + + // Convenience set of unsafe normalizations (includes FlagsUsuallySafe) + FlagsUnsafeGreedy NormalizationFlags = FlagsUsuallySafeGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagRemoveWWW | FlagSortQuery + FlagsUnsafeNonGreedy NormalizationFlags = FlagsUsuallySafeNonGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagAddWWW | FlagSortQuery + + // Convenience set of all available flags + FlagsAllGreedy = FlagsUnsafeGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator + FlagsAllNonGreedy = FlagsUnsafeNonGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator +) + +const ( + defaultHttpPort = ":80" + defaultHttpsPort = ":443" +) + +// Regular expressions used by the normalizations +var rxPort = regexp.MustCompile(`(:\d+)/?$`) +var rxDirIndex = regexp.MustCompile(`(^|/)((?:default|index)\.\w{1,4})$`) +var rxDupSlashes = regexp.MustCompile(`/{2,}`) +var rxDWORDHost = regexp.MustCompile(`^(\d+)((?:\.+)?(?:\:\d*)?)$`) +var rxOctalHost = regexp.MustCompile(`^(0\d*)\.(0\d*)\.(0\d*)\.(0\d*)((?:\.+)?(?:\:\d*)?)$`) +var rxHexHost = regexp.MustCompile(`^0x([0-9A-Fa-f]+)((?:\.+)?(?:\:\d*)?)$`) +var rxHostDots = regexp.MustCompile(`^(.+?)(:\d+)?$`) +var rxEmptyPort = regexp.MustCompile(`:+$`) + +// Map of flags to implementation function. +// FlagDecodeUnnecessaryEscapes has no action, since it is done automatically +// by parsing the string as an URL. Same for FlagUppercaseEscapes and FlagRemoveEmptyQuerySeparator. + +// Since maps have undefined traversing order, make a slice of ordered keys +var flagsOrder = []NormalizationFlags{ + FlagLowercaseScheme, + FlagLowercaseHost, + FlagRemoveDefaultPort, + FlagRemoveDirectoryIndex, + FlagRemoveDotSegments, + FlagRemoveFragment, + FlagForceHTTP, // Must be after remove default port (because https=443/http=80) + FlagRemoveDuplicateSlashes, + FlagRemoveWWW, + FlagAddWWW, + FlagSortQuery, + FlagDecodeDWORDHost, + FlagDecodeOctalHost, + FlagDecodeHexHost, + FlagRemoveUnnecessaryHostDots, + FlagRemoveEmptyPortSeparator, + FlagRemoveTrailingSlash, // These two (add/remove trailing slash) must be last + FlagAddTrailingSlash, +} + +// ... and then the map, where order is unimportant +var flags = map[NormalizationFlags]func(*url.URL){ + FlagLowercaseScheme: lowercaseScheme, + FlagLowercaseHost: lowercaseHost, + FlagRemoveDefaultPort: removeDefaultPort, + FlagRemoveDirectoryIndex: removeDirectoryIndex, + FlagRemoveDotSegments: removeDotSegments, + FlagRemoveFragment: removeFragment, + FlagForceHTTP: forceHTTP, + FlagRemoveDuplicateSlashes: removeDuplicateSlashes, + FlagRemoveWWW: removeWWW, + FlagAddWWW: addWWW, + FlagSortQuery: sortQuery, + FlagDecodeDWORDHost: decodeDWORDHost, + FlagDecodeOctalHost: decodeOctalHost, + FlagDecodeHexHost: decodeHexHost, + FlagRemoveUnnecessaryHostDots: removeUnncessaryHostDots, + FlagRemoveEmptyPortSeparator: removeEmptyPortSeparator, + FlagRemoveTrailingSlash: removeTrailingSlash, + FlagAddTrailingSlash: addTrailingSlash, +} + +// MustNormalizeURLString returns the normalized string, and panics if an error occurs. +// It takes an URL string as input, as well as the normalization flags. +func MustNormalizeURLString(u string, f NormalizationFlags) string { + result, e := NormalizeURLString(u, f) + if e != nil { + panic(e) + } + return result +} + +// NormalizeURLString returns the normalized string, or an error if it can't be parsed into an URL object. +// It takes an URL string as input, as well as the normalization flags. +func NormalizeURLString(u string, f NormalizationFlags) (string, error) { + parsed, err := url.Parse(u) + if err != nil { + return "", err + } + + if f&FlagLowercaseHost == FlagLowercaseHost { + parsed.Host = strings.ToLower(parsed.Host) + } + + // The idna package doesn't fully conform to RFC 5895 + // (https://tools.ietf.org/html/rfc5895), so we do it here. + // Taken from Go 1.8 cycle source, courtesy of bradfitz. + // TODO: Remove when (if?) idna package conforms to RFC 5895. + parsed.Host = width.Fold.String(parsed.Host) + parsed.Host = norm.NFC.String(parsed.Host) + if parsed.Host, err = idna.ToASCII(parsed.Host); err != nil { + return "", err + } + + return NormalizeURL(parsed, f), nil +} + +// NormalizeURL returns the normalized string. +// It takes a parsed URL object as input, as well as the normalization flags. +func NormalizeURL(u *url.URL, f NormalizationFlags) string { + for _, k := range flagsOrder { + if f&k == k { + flags[k](u) + } + } + return urlesc.Escape(u) +} + +func lowercaseScheme(u *url.URL) { + if len(u.Scheme) > 0 { + u.Scheme = strings.ToLower(u.Scheme) + } +} + +func lowercaseHost(u *url.URL) { + if len(u.Host) > 0 { + u.Host = strings.ToLower(u.Host) + } +} + +func removeDefaultPort(u *url.URL) { + if len(u.Host) > 0 { + scheme := strings.ToLower(u.Scheme) + u.Host = rxPort.ReplaceAllStringFunc(u.Host, func(val string) string { + if (scheme == "http" && val == defaultHttpPort) || (scheme == "https" && val == defaultHttpsPort) { + return "" + } + return val + }) + } +} + +func removeTrailingSlash(u *url.URL) { + if l := len(u.Path); l > 0 { + if strings.HasSuffix(u.Path, "/") { + u.Path = u.Path[:l-1] + } + } else if l = len(u.Host); l > 0 { + if strings.HasSuffix(u.Host, "/") { + u.Host = u.Host[:l-1] + } + } +} + +func addTrailingSlash(u *url.URL) { + if l := len(u.Path); l > 0 { + if !strings.HasSuffix(u.Path, "/") { + u.Path += "/" + } + } else if l = len(u.Host); l > 0 { + if !strings.HasSuffix(u.Host, "/") { + u.Host += "/" + } + } +} + +func removeDotSegments(u *url.URL) { + if len(u.Path) > 0 { + var dotFree []string + var lastIsDot bool + + sections := strings.Split(u.Path, "/") + for _, s := range sections { + if s == ".." { + if len(dotFree) > 0 { + dotFree = dotFree[:len(dotFree)-1] + } + } else if s != "." { + dotFree = append(dotFree, s) + } + lastIsDot = (s == "." || s == "..") + } + // Special case if host does not end with / and new path does not begin with / + u.Path = strings.Join(dotFree, "/") + if u.Host != "" && !strings.HasSuffix(u.Host, "/") && !strings.HasPrefix(u.Path, "/") { + u.Path = "/" + u.Path + } + // Special case if the last segment was a dot, make sure the path ends with a slash + if lastIsDot && !strings.HasSuffix(u.Path, "/") { + u.Path += "/" + } + } +} + +func removeDirectoryIndex(u *url.URL) { + if len(u.Path) > 0 { + u.Path = rxDirIndex.ReplaceAllString(u.Path, "$1") + } +} + +func removeFragment(u *url.URL) { + u.Fragment = "" +} + +func forceHTTP(u *url.URL) { + if strings.ToLower(u.Scheme) == "https" { + u.Scheme = "http" + } +} + +func removeDuplicateSlashes(u *url.URL) { + if len(u.Path) > 0 { + u.Path = rxDupSlashes.ReplaceAllString(u.Path, "/") + } +} + +func removeWWW(u *url.URL) { + if len(u.Host) > 0 && strings.HasPrefix(strings.ToLower(u.Host), "www.") { + u.Host = u.Host[4:] + } +} + +func addWWW(u *url.URL) { + if len(u.Host) > 0 && !strings.HasPrefix(strings.ToLower(u.Host), "www.") { + u.Host = "www." + u.Host + } +} + +func sortQuery(u *url.URL) { + q := u.Query() + + if len(q) > 0 { + arKeys := make([]string, len(q)) + i := 0 + for k := range q { + arKeys[i] = k + i++ + } + sort.Strings(arKeys) + buf := new(bytes.Buffer) + for _, k := range arKeys { + sort.Strings(q[k]) + for _, v := range q[k] { + if buf.Len() > 0 { + buf.WriteRune('&') + } + buf.WriteString(fmt.Sprintf("%s=%s", k, urlesc.QueryEscape(v))) + } + } + + // Rebuild the raw query string + u.RawQuery = buf.String() + } +} + +func decodeDWORDHost(u *url.URL) { + if len(u.Host) > 0 { + if matches := rxDWORDHost.FindStringSubmatch(u.Host); len(matches) > 2 { + var parts [4]int64 + + dword, _ := strconv.ParseInt(matches[1], 10, 0) + for i, shift := range []uint{24, 16, 8, 0} { + parts[i] = dword >> shift & 0xFF + } + u.Host = fmt.Sprintf("%d.%d.%d.%d%s", parts[0], parts[1], parts[2], parts[3], matches[2]) + } + } +} + +func decodeOctalHost(u *url.URL) { + if len(u.Host) > 0 { + if matches := rxOctalHost.FindStringSubmatch(u.Host); len(matches) > 5 { + var parts [4]int64 + + for i := 1; i <= 4; i++ { + parts[i-1], _ = strconv.ParseInt(matches[i], 8, 0) + } + u.Host = fmt.Sprintf("%d.%d.%d.%d%s", parts[0], parts[1], parts[2], parts[3], matches[5]) + } + } +} + +func decodeHexHost(u *url.URL) { + if len(u.Host) > 0 { + if matches := rxHexHost.FindStringSubmatch(u.Host); len(matches) > 2 { + // Conversion is safe because of regex validation + parsed, _ := strconv.ParseInt(matches[1], 16, 0) + // Set host as DWORD (base 10) encoded host + u.Host = fmt.Sprintf("%d%s", parsed, matches[2]) + // The rest is the same as decoding a DWORD host + decodeDWORDHost(u) + } + } +} + +func removeUnncessaryHostDots(u *url.URL) { + if len(u.Host) > 0 { + if matches := rxHostDots.FindStringSubmatch(u.Host); len(matches) > 1 { + // Trim the leading and trailing dots + u.Host = strings.Trim(matches[1], ".") + if len(matches) > 2 { + u.Host += matches[2] + } + } + } +} + +func removeEmptyPortSeparator(u *url.URL) { + if len(u.Host) > 0 { + u.Host = rxEmptyPort.ReplaceAllString(u.Host, "") + } +} diff --git a/normalize/normalize_test.go b/normalize/normalize_test.go new file mode 100644 index 0000000..177ae01 --- /dev/null +++ b/normalize/normalize_test.go @@ -0,0 +1,782 @@ +package normalize + +import ( + "fmt" + "net/url" + "testing" + "unicode" +) + +type testCase struct { + nm string + src string + flgs NormalizationFlags + res string + parsed bool +} + +var ( + cases = [...]*testCase{ + { + "LowerScheme", + "HTTP://www.SRC.ca", + FlagLowercaseScheme, + "http://www.SRC.ca", + false, + }, + { + "LowerScheme2", + "http://www.SRC.ca", + FlagLowercaseScheme, + "http://www.SRC.ca", + false, + }, + { + "LowerHost", + "HTTP://www.SRC.ca/", + FlagLowercaseHost, + "http://www.src.ca/", // Since Go1.1, scheme is automatically lowercased + false, + }, + { + "UpperEscapes", + `http://www.whatever.com/Some%aa%20Special%8Ecases/`, + FlagUppercaseEscapes, + "http://www.whatever.com/Some%AA%20Special%8Ecases/", + false, + }, + { + "UnnecessaryEscapes", + `http://www.toto.com/%41%42%2E%44/%32%33%52%2D/%5f%7E`, + FlagDecodeUnnecessaryEscapes, + "http://www.toto.com/AB.D/23R-/_~", + false, + }, + { + "RemoveDefaultPort", + "HTTP://www.SRC.ca:80/", + FlagRemoveDefaultPort, + "http://www.SRC.ca/", // Since Go1.1, scheme is automatically lowercased + false, + }, + { + "RemoveDefaultPort2", + "HTTP://www.SRC.ca:80", + FlagRemoveDefaultPort, + "http://www.SRC.ca", // Since Go1.1, scheme is automatically lowercased + false, + }, + { + "RemoveDefaultPort3", + "HTTP://www.SRC.ca:8080", + FlagRemoveDefaultPort, + "http://www.SRC.ca:8080", // Since Go1.1, scheme is automatically lowercased + false, + }, + { + "Safe", + "HTTP://www.SRC.ca:80/to%1ato%8b%ee/OKnow%41%42%43%7e", + FlagsSafe, + "http://www.src.ca/to%1Ato%8B%EE/OKnowABC~", + false, + }, + { + "BothLower", + "HTTP://www.SRC.ca:80/to%1ato%8b%ee/OKnow%41%42%43%7e", + FlagLowercaseHost | FlagLowercaseScheme, + "http://www.src.ca:80/to%1Ato%8B%EE/OKnowABC~", + false, + }, + { + "RemoveTrailingSlash", + "HTTP://www.SRC.ca:80/", + FlagRemoveTrailingSlash, + "http://www.SRC.ca:80", // Since Go1.1, scheme is automatically lowercased + false, + }, + { + "RemoveTrailingSlash2", + "HTTP://www.SRC.ca:80/toto/titi/", + FlagRemoveTrailingSlash, + "http://www.SRC.ca:80/toto/titi", // Since Go1.1, scheme is automatically lowercased + false, + }, + { + "RemoveTrailingSlash3", + "HTTP://www.SRC.ca:80/toto/titi/fin/?a=1", + FlagRemoveTrailingSlash, + "http://www.SRC.ca:80/toto/titi/fin?a=1", // Since Go1.1, scheme is automatically lowercased + false, + }, + { + "AddTrailingSlash", + "HTTP://www.SRC.ca:80", + FlagAddTrailingSlash, + "http://www.SRC.ca:80/", // Since Go1.1, scheme is automatically lowercased + false, + }, + { + "AddTrailingSlash2", + "HTTP://www.SRC.ca:80/toto/titi.html", + FlagAddTrailingSlash, + "http://www.SRC.ca:80/toto/titi.html/", // Since Go1.1, scheme is automatically lowercased + false, + }, + { + "AddTrailingSlash3", + "HTTP://www.SRC.ca:80/toto/titi/fin?a=1", + FlagAddTrailingSlash, + "http://www.SRC.ca:80/toto/titi/fin/?a=1", // Since Go1.1, scheme is automatically lowercased + false, + }, + { + "RemoveDotSegments", + "HTTP://root/a/b/./../../c/", + FlagRemoveDotSegments, + "http://root/c/", // Since Go1.1, scheme is automatically lowercased + false, + }, + { + "RemoveDotSegments2", + "HTTP://root/../a/b/./../c/../d", + FlagRemoveDotSegments, + "http://root/a/d", // Since Go1.1, scheme is automatically lowercased + false, + }, + { + "UsuallySafe", + "HTTP://www.SRC.ca:80/to%1ato%8b%ee/./c/d/../OKnow%41%42%43%7e/?a=b#test", + FlagsUsuallySafeGreedy, + "http://www.src.ca/to%1Ato%8B%EE/c/OKnowABC~?a=b#test", + false, + }, + { + "RemoveDirectoryIndex", + "HTTP://root/a/b/c/default.aspx", + FlagRemoveDirectoryIndex, + "http://root/a/b/c/", // Since Go1.1, scheme is automatically lowercased + false, + }, + { + "RemoveDirectoryIndex2", + "HTTP://root/a/b/c/default#a=b", + FlagRemoveDirectoryIndex, + "http://root/a/b/c/default#a=b", // Since Go1.1, scheme is automatically lowercased + false, + }, + { + "RemoveFragment", + "HTTP://root/a/b/c/default#toto=tata", + FlagRemoveFragment, + "http://root/a/b/c/default", // Since Go1.1, scheme is automatically lowercased + false, + }, + { + "ForceHTTP", + "https://root/a/b/c/default#toto=tata", + FlagForceHTTP, + "http://root/a/b/c/default#toto=tata", + false, + }, + { + "RemoveDuplicateSlashes", + "https://root/a//b///c////default#toto=tata", + FlagRemoveDuplicateSlashes, + "https://root/a/b/c/default#toto=tata", + false, + }, + { + "RemoveDuplicateSlashes2", + "https://root//a//b///c////default#toto=tata", + FlagRemoveDuplicateSlashes, + "https://root/a/b/c/default#toto=tata", + false, + }, + { + "RemoveWWW", + "https://www.root/a/b/c/", + FlagRemoveWWW, + "https://root/a/b/c/", + false, + }, + { + "RemoveWWW2", + "https://WwW.Root/a/b/c/", + FlagRemoveWWW, + "https://Root/a/b/c/", + false, + }, + { + "AddWWW", + "https://Root/a/b/c/", + FlagAddWWW, + "https://www.Root/a/b/c/", + false, + }, + { + "SortQuery", + "http://root/toto/?b=4&a=1&c=3&b=2&a=5", + FlagSortQuery, + "http://root/toto/?a=1&a=5&b=2&b=4&c=3", + false, + }, + { + "RemoveEmptyQuerySeparator", + "http://root/toto/?", + FlagRemoveEmptyQuerySeparator, + "http://root/toto/", + false, + }, + { + "Unsafe", + "HTTPS://www.RooT.com/toto/t%45%1f///a/./b/../c/?z=3&w=2&a=4&w=1#invalid", + FlagsUnsafeGreedy, + "http://root.com/toto/tE%1F/a/c?a=4&w=1&w=2&z=3", + false, + }, + { + "Safe2", + "HTTPS://www.RooT.com/toto/t%45%1f///a/./b/../c/?z=3&w=2&a=4&w=1#invalid", + FlagsSafe, + "https://www.root.com/toto/tE%1F///a/./b/../c/?z=3&w=2&a=4&w=1#invalid", + false, + }, + { + "UsuallySafe2", + "HTTPS://www.RooT.com/toto/t%45%1f///a/./b/../c/?z=3&w=2&a=4&w=1#invalid", + FlagsUsuallySafeGreedy, + "https://www.root.com/toto/tE%1F///a/c?z=3&w=2&a=4&w=1#invalid", + false, + }, + { + "AddTrailingSlashBug", + "http://src.ca/", + FlagsAllNonGreedy, + "http://www.src.ca/", + false, + }, + { + "SourceModified", + "HTTPS://www.RooT.com/toto/t%45%1f///a/./b/../c/?z=3&w=2&a=4&w=1#invalid", + FlagsUnsafeGreedy, + "http://root.com/toto/tE%1F/a/c?a=4&w=1&w=2&z=3", + true, + }, + { + "IPv6-1", + "http://[2001:db8:1f70::999:de8:7648:6e8]/test", + FlagsSafe | FlagRemoveDotSegments, + "http://[2001:db8:1f70::999:de8:7648:6e8]/test", + false, + }, + { + "IPv6-2", + "http://[::ffff:192.168.1.1]/test", + FlagsSafe | FlagRemoveDotSegments, + "http://[::ffff:192.168.1.1]/test", + false, + }, + { + "IPv6-3", + "http://[::ffff:192.168.1.1]:80/test", + FlagsSafe | FlagRemoveDotSegments, + "http://[::ffff:192.168.1.1]/test", + false, + }, + { + "IPv6-4", + "htTps://[::fFff:192.168.1.1]:443/test", + FlagsSafe | FlagRemoveDotSegments, + "https://[::ffff:192.168.1.1]/test", + false, + }, + { + "FTP", + "ftp://user:pass@ftp.foo.net/foo/bar", + FlagsSafe | FlagRemoveDotSegments, + "ftp://user:pass@ftp.foo.net/foo/bar", + false, + }, + { + "Standard-1", + "http://www.foo.com:80/foo", + FlagsSafe | FlagRemoveDotSegments, + "http://www.foo.com/foo", + false, + }, + { + "Standard-2", + "http://www.foo.com:8000/foo", + FlagsSafe | FlagRemoveDotSegments, + "http://www.foo.com:8000/foo", + false, + }, + { + "Standard-3", + "http://www.foo.com/%7ebar", + FlagsSafe | FlagRemoveDotSegments, + "http://www.foo.com/~bar", + false, + }, + { + "Standard-4", + "http://www.foo.com/%7Ebar", + FlagsSafe | FlagRemoveDotSegments, + "http://www.foo.com/~bar", + false, + }, + { + "Standard-5", + "http://USER:pass@www.Example.COM/foo/bar", + FlagsSafe | FlagRemoveDotSegments, + "http://USER:pass@www.example.com/foo/bar", + false, + }, + { + "Standard-6", + "http://test.example/?a=%26&b=1", + FlagsSafe | FlagRemoveDotSegments, + "http://test.example/?a=%26&b=1", + false, + }, + { + "Standard-7", + "http://test.example/%25/?p=%20val%20%25", + FlagsSafe | FlagRemoveDotSegments, + "http://test.example/%25/?p=%20val%20%25", + false, + }, + { + "Standard-8", + "http://test.example/path/with a%20space+/", + FlagsSafe | FlagRemoveDotSegments, + "http://test.example/path/with%20a%20space+/", + false, + }, + { + "Standard-9", + "http://test.example/?", + FlagsSafe | FlagRemoveDotSegments, + "http://test.example/", + false, + }, + { + "Standard-10", + "http://a.COM/path/?b&a", + FlagsSafe | FlagRemoveDotSegments, + "http://a.com/path/?b&a", + false, + }, + { + "StandardCasesAddTrailingSlash", + "http://test.example?", + FlagsSafe | FlagAddTrailingSlash, + "http://test.example/", + false, + }, + { + "OctalIP-1", + "http://0123.011.0.4/", + FlagsSafe | FlagDecodeOctalHost, + "http://0123.011.0.4/", + false, + }, + { + "OctalIP-2", + "http://0102.0146.07.0223/", + FlagsSafe | FlagDecodeOctalHost, + "http://66.102.7.147/", + false, + }, + { + "OctalIP-3", + "http://0102.0146.07.0223.:23/", + FlagsSafe | FlagDecodeOctalHost, + "http://66.102.7.147.:23/", + false, + }, + { + "OctalIP-4", + "http://USER:pass@0102.0146.07.0223../", + FlagsSafe | FlagDecodeOctalHost, + "http://USER:pass@66.102.7.147../", + false, + }, + { + "DWORDIP-1", + "http://123.1113982867/", + FlagsSafe | FlagDecodeDWORDHost, + "http://123.1113982867/", + false, + }, + { + "DWORDIP-2", + "http://1113982867/", + FlagsSafe | FlagDecodeDWORDHost, + "http://66.102.7.147/", + false, + }, + { + "DWORDIP-3", + "http://1113982867.:23/", + FlagsSafe | FlagDecodeDWORDHost, + "http://66.102.7.147.:23/", + false, + }, + { + "DWORDIP-4", + "http://USER:pass@1113982867../", + FlagsSafe | FlagDecodeDWORDHost, + "http://USER:pass@66.102.7.147../", + false, + }, + { + "HexIP-1", + "http://0x123.1113982867/", + FlagsSafe | FlagDecodeHexHost, + "http://0x123.1113982867/", + false, + }, + { + "HexIP-2", + "http://0x42660793/", + FlagsSafe | FlagDecodeHexHost, + "http://66.102.7.147/", + false, + }, + { + "HexIP-3", + "http://0x42660793.:23/", + FlagsSafe | FlagDecodeHexHost, + "http://66.102.7.147.:23/", + false, + }, + { + "HexIP-4", + "http://USER:pass@0x42660793../", + FlagsSafe | FlagDecodeHexHost, + "http://USER:pass@66.102.7.147../", + false, + }, + { + "UnnecessaryHostDots-1", + "http://.www.foo.com../foo/bar.html", + FlagsSafe | FlagRemoveUnnecessaryHostDots, + "http://www.foo.com/foo/bar.html", + false, + }, + { + "UnnecessaryHostDots-2", + "http://www.foo.com./foo/bar.html", + FlagsSafe | FlagRemoveUnnecessaryHostDots, + "http://www.foo.com/foo/bar.html", + false, + }, + { + "UnnecessaryHostDots-3", + "http://www.foo.com.:81/foo", + FlagsSafe | FlagRemoveUnnecessaryHostDots, + "http://www.foo.com:81/foo", + false, + }, + { + "UnnecessaryHostDots-4", + "http://www.example.com./", + FlagsSafe | FlagRemoveUnnecessaryHostDots, + "http://www.example.com/", + false, + }, + { + "EmptyPort-1", + "http://www.thedraymin.co.uk:/main/?p=308", + FlagsSafe | FlagRemoveEmptyPortSeparator, + "http://www.thedraymin.co.uk/main/?p=308", + false, + }, + { + "EmptyPort-2", + "http://www.src.ca:", + FlagsSafe | FlagRemoveEmptyPortSeparator, + "http://www.src.ca", + false, + }, + { + "Slashes-1", + "http://test.example/foo/bar/.", + FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, + "http://test.example/foo/bar/", + false, + }, + { + "Slashes-2", + "http://test.example/foo/bar/./", + FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, + "http://test.example/foo/bar/", + false, + }, + { + "Slashes-3", + "http://test.example/foo/bar/..", + FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, + "http://test.example/foo/", + false, + }, + { + "Slashes-4", + "http://test.example/foo/bar/../", + FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, + "http://test.example/foo/", + false, + }, + { + "Slashes-5", + "http://test.example/foo/bar/../baz", + FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, + "http://test.example/foo/baz", + false, + }, + { + "Slashes-6", + "http://test.example/foo/bar/../..", + FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, + "http://test.example/", + false, + }, + { + "Slashes-7", + "http://test.example/foo/bar/../../", + FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, + "http://test.example/", + false, + }, + { + "Slashes-8", + "http://test.example/foo/bar/../../baz", + FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, + "http://test.example/baz", + false, + }, + { + "Slashes-9", + "http://test.example/foo/bar/../../../baz", + FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, + "http://test.example/baz", + false, + }, + { + "Slashes-10", + "http://test.example/foo/bar/../../../../baz", + FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, + "http://test.example/baz", + false, + }, + { + "Slashes-11", + "http://test.example/./foo", + FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, + "http://test.example/foo", + false, + }, + { + "Slashes-12", + "http://test.example/../foo", + FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, + "http://test.example/foo", + false, + }, + { + "Slashes-13", + "http://test.example/foo.", + FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, + "http://test.example/foo.", + false, + }, + { + "Slashes-14", + "http://test.example/.foo", + FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, + "http://test.example/.foo", + false, + }, + { + "Slashes-15", + "http://test.example/foo..", + FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, + "http://test.example/foo..", + false, + }, + { + "Slashes-16", + "http://test.example/..foo", + FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, + "http://test.example/..foo", + false, + }, + { + "Slashes-17", + "http://test.example/./../foo", + FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, + "http://test.example/foo", + false, + }, + { + "Slashes-18", + "http://test.example/./foo/.", + FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, + "http://test.example/foo/", + false, + }, + { + "Slashes-19", + "http://test.example/foo/./bar", + FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, + "http://test.example/foo/bar", + false, + }, + { + "Slashes-20", + "http://test.example/foo/../bar", + FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, + "http://test.example/bar", + false, + }, + { + "Slashes-21", + "http://test.example/foo//", + FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, + "http://test.example/foo/", + false, + }, + { + "Slashes-22", + "http://test.example/foo///bar//", + FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, + "http://test.example/foo/bar/", + false, + }, + { + "Relative", + "foo/bar", + FlagsAllGreedy, + "foo/bar", + false, + }, + { + "Relative-1", + "./../foo", + FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, + "foo", + false, + }, + { + "Relative-2", + "./foo/bar/../baz/../bang/..", + FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, + "foo/", + false, + }, + { + "Relative-3", + "foo///bar//", + FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes, + "foo/bar/", + false, + }, + { + "Relative-4", + "www.youtube.com", + FlagsUsuallySafeGreedy, + "www.youtube.com", + false, + }, + { + "Issue-#24", + "///foo///bar///", + FlagRemoveDuplicateSlashes | FlagRemoveTrailingSlash, + "/foo/bar", + false, + }, + /*&testCase{ + "UrlNorm-5", + "http://ja.wikipedia.org/wiki/%E3%82%AD%E3%83%A3%E3%82%BF%E3%83%94%E3%83%A9%E3%83%BC%E3%82%B8%E3%83%A3%E3%83%91%E3%83%B3", + FlagsSafe | FlagRemoveDotSegments, + "http://ja.wikipedia.org/wiki/\xe3\x82\xad\xe3\x83\xa3\xe3\x82\xbf\xe3\x83\x94\xe3\x83\xa9\xe3\x83\xbc\xe3\x82\xb8\xe3\x83\xa3\xe3\x83\x91\xe3\x83\xb3", + false, + }, + &testCase{ + "UrlNorm-1", + "http://test.example/?a=%e3%82%82%26", + FlagsAllGreedy, + "http://test.example/?a=\xe3\x82\x82%26", + false, + },*/ + } +) + +func TestRunner(t *testing.T) { + for _, tc := range cases { + runCase(tc, t) + } +} + +func runCase(tc *testCase, t *testing.T) { + t.Logf("running %s...", tc.nm) + if tc.parsed { + u, e := url.Parse(tc.src) + if e != nil { + t.Errorf("%s - FAIL : %s", tc.nm, e) + return + } else { + NormalizeURL(u, tc.flgs) + if s := u.String(); s != tc.res { + t.Errorf("%s - FAIL expected '%s', got '%s'", tc.nm, tc.res, s) + } + } + } else { + if s, e := NormalizeURLString(tc.src, tc.flgs); e != nil { + t.Errorf("%s - FAIL : %s", tc.nm, e) + } else if s != tc.res { + t.Errorf("%s - FAIL expected '%s', got '%s'", tc.nm, tc.res, s) + } + } +} + +func TestDecodeUnnecessaryEscapesAll(t *testing.T) { + var url = "http://host/" + + for i := 0; i < 256; i++ { + url += fmt.Sprintf("%%%02x", i) + } + s, err := NormalizeURLString(url, FlagDecodeUnnecessaryEscapes) + if err != nil { + t.Fatalf("parse error: %s", err) + } + + const want = "http://host/%00%01%02%03%04%05%06%07%08%09%0A%0B%0C%0D%0E%0F%10%11%12%13%14%15%16%17%18%19%1A%1B%1C%1D%1E%1F%20!%22%23$%25&'()*+,-./0123456789:;%3C=%3E%3F@ABCDEFGHIJKLMNOPQRSTUVWXYZ[%5C]%5E_%60abcdefghijklmnopqrstuvwxyz%7B%7C%7D~%7F%80%81%82%83%84%85%86%87%88%89%8A%8B%8C%8D%8E%8F%90%91%92%93%94%95%96%97%98%99%9A%9B%9C%9D%9E%9F%A0%A1%A2%A3%A4%A5%A6%A7%A8%A9%AA%AB%AC%AD%AE%AF%B0%B1%B2%B3%B4%B5%B6%B7%B8%B9%BA%BB%BC%BD%BE%BF%C0%C1%C2%C3%C4%C5%C6%C7%C8%C9%CA%CB%CC%CD%CE%CF%D0%D1%D2%D3%D4%D5%D6%D7%D8%D9%DA%DB%DC%DD%DE%DF%E0%E1%E2%E3%E4%E5%E6%E7%E8%E9%EA%EB%EC%ED%EE%EF%F0%F1%F2%F3%F4%F5%F6%F7%F8%F9%FA%FB%FC%FD%FE%FF" + if s != want { + t.Errorf("DecodeUnnecessaryEscapesAll:\nwant\n%s\ngot\n%s", want, s) + } +} + +func TestEncodeNecessaryEscapesAll(t *testing.T) { + const base = "http://host/" + var path []byte + + for i := 0; i < 256; i++ { + // Since go1.12, url.Parse fails if the raw URL contains ASCII control characters, + // meaning anything < 0x20 and 0x7f (DEL), so do not add those bytes to the constructed url. + // See https://github.com/PuerkitoBio/purell/issues/28 + if i != 0x25 && !unicode.IsControl(rune(i)) { + path = append(path, byte(i)) + } + } + s, err := NormalizeURLString(base+string(path), FlagEncodeNecessaryEscapes) + if err != nil { + t.Fatalf("parse error: %s", err) + } + + const want = "http://host/%20!%22#$&'()*+,-./0123456789:;%3C=%3E?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[%5C]%5E_%60abcdefghijklmnopqrstuvwxyz%7B%7C%7D~%A0%A1%A2%A3%A4%A5%A6%A7%A8%A9%AA%AB%AC%AD%AE%AF%B0%B1%B2%B3%B4%B5%B6%B7%B8%B9%BA%BB%BC%BD%BE%BF%C0%C1%C2%C3%C4%C5%C6%C7%C8%C9%CA%CB%CC%CD%CE%CF%D0%D1%D2%D3%D4%D5%D6%D7%D8%D9%DA%DB%DC%DD%DE%DF%E0%E1%E2%E3%E4%E5%E6%E7%E8%E9%EA%EB%EC%ED%EE%EF%F0%F1%F2%F3%F4%F5%F6%F7%F8%F9%FA%FB%FC%FD%FE%FF" + if s != want { + t.Errorf("EncodeNecessaryEscapesAll:\nwant\n%s\ngot\n%s", want, s) + } +} |