Skip to content

Canonicalize URLs for prefix www. #52

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kezhenxu94 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Canonicalize URLs for prefix www. #52

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions scan.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,9 @@ func (s *Scanner) licenseURL(url string) (License, bool) {
// Then we lower-case it.
url = strings.TrimPrefix(url, "http://")
url = strings.TrimPrefix(url, "https://")
url = strings.TrimPrefix(url, "www.")
url = strings.TrimSuffix(url, ".html")
url = strings.TrimSuffix(url, ".htm")
url = strings.TrimSuffix(url, "/")
url = strings.TrimSuffix(url, "/legalcode") // Common for CC licenses.
url = strings.ToLower(url)
Expand Down
2 changes: 2 additions & 0 deletions url_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ type urlTest struct {
}

var urlTests = []urlTest{
{[]string{"Apache-2.0"}, "https://www.opensource.org/licenses/Apache-2.0"},
{[]string{"Apache-2.0"}, "https://www.apache.org/licenses/LICENSE-2.0"},
{[]string{"CC-BY-4.0"},
"This code is licensed by https://creativecommons.org/licenses/BY/4.0/ so have fun"},
{[]string{"CC-BY-NC-4.0"},
Expand Down
160 changes: 82 additions & 78 deletions urls.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ package licensecheck
// All entries are lower case.
// Keep this list sorted for easy checking.
var builtinURLs = []License{
{URL: "apache.org/licenses/license-2.0", ID: "Apache-2.0"},
{URL: "apache.org/licenses/license-2.0.txt", ID: "Apache-2.0"},
{URL: "creativecommons.org/licenses/by-nc-nd/2.0", ID: "CC-BY-NC-ND-2.0"},
{URL: "creativecommons.org/licenses/by-nc-nd/2.5", ID: "CC-BY-NC-ND-2.5"},
{URL: "creativecommons.org/licenses/by-nc-nd/3.0", ID: "CC-BY-NC-ND-3.0"},
Expand Down Expand Up @@ -46,107 +48,109 @@ var builtinURLs = []License{
{URL: "creativecommons.org/licenses/by/3.0", ID: "CC-BY-3.0"},
{URL: "creativecommons.org/licenses/by/4.0", ID: "CC-BY-4.0"},
{URL: "creativecommons.org/publicdomain/zero/1.0", ID: "CC0-1.0"},
{URL: "eclipse.org/legal/epl-2.0", ID: "EPL-2.0"},
{URL: "gnu.org/licenses/agpl.txt", ID: "AGPL-3.0"},
// {URL: "gnu.org/licenses/autoconf-exception-3.0", ID: "GPL-3.0-with-autoconf-exception"},
// {URL: "gnu.org/licenses/ecos-license", ID: "eCos-2.0"},
{URL: "gnu.org/licenses/fdl-1.3.txt", ID: "GFDL-1.3"},
// {URL: "gnu.org/licenses/gcc-exception-3.1", ID: "GPL-3.0-with-GCC-exception"},
{URL: "gnu.org/licenses/gpl-3.0-standalone", ID: "GPL-3.0"},
// {URL: "gnu.org/licenses/gpl-faq#fontexception", ID: "GPL-2.0-with-font-exception"},
{URL: "gnu.org/licenses/lgpl-2.1", ID: "LGPL-2.1"},
{URL: "gnu.org/licenses/lgpl-3.0-standalone", ID: "LGPL-3.0"},
{URL: "gnu.org/licenses/old-licenses/fdl-1.1.txt", ID: "GFDL-1.1"},
{URL: "gnu.org/licenses/old-licenses/gpl-1.0-standalone", ID: "GPL-1.0"},
{URL: "gnu.org/licenses/old-licenses/gpl-2.0-standalone", ID: "GPL-2.0"},
{URL: "gnu.org/licenses/old-licenses/lgpl-2.0-standalone", ID: "LGPL-2.0"},
{URL: "gnu.org/licenses/old-licenses/lgpl-2.1-standalone", ID: "LGPL-2.1"},
{URL: "gnu.org/prep/maintain/html_node/license-notices-for-other-files", ID: "FSFAP"},
// {URL: "gnu.org/software/classpath/license", ID: "GPL-2.0-with-classpath-exception"},
{URL: "mozilla.org/mpl/mpl-1.1", ID: "MPL-1.1"},
{URL: "opensource.org/licenses/afl-3.0", ID: "AFL-3.0"},
{URL: "opensource.org/licenses/agpl-3.0", ID: "AGPL-3.0"},
{URL: "opensource.org/licenses/apache-1.1", ID: "Apache-1.1"},
{URL: "opensource.org/licenses/apache-2.0", ID: "Apache-2.0"},
{URL: "opensource.org/licenses/apl-1.0", ID: "APL-1.0"},
{URL: "opensource.org/licenses/artistic-1.0", ID: "Artistic-1.0"},
{URL: "opensource.org/licenses/artistic-license-2.0", ID: "Artistic-2.0"},
{URL: "opensource.org/licenses/attribution", ID: "AAL"},
{URL: "opensource.org/licenses/bsd-2-clause", ID: "BSD-2-Clause"},
{URL: "opensource.org/licenses/bsd-3-clause", ID: "BSD-3-Clause"},
{URL: "opensource.org/licenses/bsdpluspatent", ID: "BSD-2-Clause-Patent"},
{URL: "opensource.org/licenses/bsl-1.0", ID: "BSL-1.0"},
{URL: "opensource.org/licenses/catosl-1.1", ID: "CATOSL-1.1"},
{URL: "opensource.org/licenses/cddl1", ID: "CDDL-1.0"},
{URL: "opensource.org/licenses/cnri-python", ID: "CNRI-Python"},
{URL: "opensource.org/licenses/cpal-1.0", ID: "CPAL-1.0"},
{URL: "opensource.org/licenses/cpl-1.0", ID: "CPL-1.0"},
{URL: "opensource.org/licenses/cua-opl-1.0", ID: "CUA-OPL-1.0"},
{URL: "opensource.org/licenses/ecl-1.0", ID: "ECL-1.0"},
{URL: "opensource.org/licenses/ecl-2.0", ID: "ECL-2.0"},
{URL: "opensource.org/licenses/efl-1.0", ID: "EFL-1.0"},
{URL: "opensource.org/licenses/efl-2.0", ID: "EFL-2.0"},
{URL: "opensource.org/licenses/entessa", ID: "Entessa"},
{URL: "opensource.org/licenses/epl-1.0", ID: "EPL-1.0"},
{URL: "opensource.org/licenses/epl-2.0", ID: "EPL-2.0"},
{URL: "opensource.org/licenses/eudatagrid", ID: "EUDatagrid"},
{URL: "opensource.org/licenses/eupl-1.1", ID: "EUPL-1.1"},
{URL: "opensource.org/licenses/fair", ID: "Fair"},
{URL: "opensource.org/licenses/frameworx-1.0", ID: "Frameworx-1.0"},
{URL: "opensource.org/licenses/gpl-2.0", ID: "GPL-2.0"},
{URL: "opensource.org/licenses/gpl-3.0", ID: "GPL-3.0"},
{URL: "opensource.org/licenses/hpnd", ID: "HPND"},
{URL: "opensource.org/licenses/intel", ID: "Intel"},
{URL: "opensource.org/licenses/lpl-1.0", ID: "LPL-1.0"},
{URL: "opensource.org/licenses/ipa", ID: "IPA"},
{URL: "opensource.org/licenses/ipl-1.0", ID: "IPL-1.0"},
{URL: "opensource.org/licenses/isc", ID: "ISC"},
{URL: "opensource.org/licenses/lgpl-2.1", ID: "LGPL-2.1"},
{URL: "opensource.org/licenses/lgpl-3.0", ID: "LGPL-3.0"},
{URL: "opensource.org/licenses/liliq-p-1.1", ID: "LiLiQ-P-1.1"},
{URL: "opensource.org/licenses/liliq-r-1.1", ID: "LiLiQ-R-1.1"},
{URL: "opensource.org/licenses/liliq-rplus-1.1", ID: "LiLiQ-Rplus-1.1"},
{URL: "opensource.org/licenses/lpl-1.0", ID: "LPL-1.0"},
{URL: "opensource.org/licenses/lpl-1.02", ID: "LPL-1.02"},
{URL: "opensource.org/licenses/lppl-1.3c", ID: "LPPL-1.3c"},
{URL: "opensource.org/licenses/miros", ID: "MirOS"},
{URL: "opensource.org/licenses/mit", ID: "MIT"},
{URL: "opensource.org/licenses/motosoto", ID: "Motosoto"},
{URL: "opensource.org/licenses/mpl-1.0", ID: "MPL-1.0"},
{URL: "opensource.org/licenses/mpl-1.1", ID: "MPL-1.1"},
{URL: "opensource.org/licenses/mpl-2.0", ID: "MPL-2.0"},
{URL: "opensource.org/licenses/ms-pl", ID: "MS-PL"},
{URL: "opensource.org/licenses/ms-rl", ID: "MS-RL"},
{URL: "opensource.org/licenses/multics", ID: "Multics"},
{URL: "opensource.org/licenses/nasa-1.3", ID: "NASA-1.3"},
{URL: "opensource.org/licenses/naumen", ID: "Naumen"},
{URL: "opensource.org/licenses/ncsa", ID: "NCSA"},
{URL: "opensource.org/licenses/ngpl", ID: "NGPL"},
{URL: "opensource.org/licenses/nokia", ID: "Nokia"},
{URL: "opensource.org/licenses/nosl3.0", ID: "NPOSL-3.0"},
{URL: "opensource.org/licenses/ntp", ID: "NTP"},
{URL: "opensource.org/licenses/oclc-2.0", ID: "OCLC-2.0"},
{URL: "opensource.org/licenses/ofl-1.1", ID: "OFL-1.1"},
{URL: "opensource.org/licenses/ogtsl", ID: "OGTSL"},
{URL: "opensource.org/licenses/opl-2.1", ID: "OSET-PL-2.1"},
{URL: "opensource.org/licenses/osl-1.0", ID: "OSL-1.0"},
{URL: "opensource.org/licenses/osl-2.1", ID: "OSL-2.1"},
{URL: "opensource.org/licenses/osl-3.0", ID: "OSL-3.0"},
{URL: "opensource.org/licenses/php-3.0", ID: "PHP-3.0"},
{URL: "opensource.org/licenses/postgresql", ID: "PostgreSQL"},
{URL: "opensource.org/licenses/python-2.0", ID: "Python-2.0"},
{URL: "opensource.org/licenses/qpl-1.0", ID: "QPL-1.0"},
{URL: "opensource.org/licenses/rpl-1.1", ID: "RPL-1.1"},
{URL: "opensource.org/licenses/rpl-1.5", ID: "RPL-1.5"},
{URL: "opensource.org/licenses/rpsl-1.0", ID: "RPSL-1.0"},
{URL: "opensource.org/licenses/rscpl", ID: "RSCPL"},
{URL: "opensource.org/licenses/simpl-2.0", ID: "SimPL-2.0"},
{URL: "opensource.org/licenses/sissl", ID: "SISSL"},
{URL: "opensource.org/licenses/sleepycat", ID: "Sleepycat"},
{URL: "opensource.org/licenses/spl-1.0", ID: "SPL-1.0"},
{URL: "opensource.org/licenses/upl", ID: "UPL-1.0"},
{URL: "opensource.org/licenses/vsl-1.0", ID: "VSL-1.0"},
{URL: "opensource.org/licenses/w3c", ID: "W3C"},
{URL: "opensource.org/licenses/watcom-1.0", ID: "Watcom-1.0"},
{URL: "opensource.org/licenses/xnet", ID: "Xnet"},
{URL: "opensource.org/licenses/zlib", ID: "Zlib"},
{URL: "opensource.org/licenses/zpl-2.0", ID: "ZPL-2.0"},
{URL: "www.apache.org/licenses/license-2.0", ID: "Apache-2.0"},
{URL: "www.gnu.org/licenses/agpl.txt", ID: "AGPL-3.0"},
// {URL: "www.gnu.org/licenses/autoconf-exception-3.0.html", ID: "GPL-3.0-with-autoconf-exception"},
// {URL: "www.gnu.org/licenses/ecos-license.html", ID: "eCos-2.0"},
{URL: "www.gnu.org/licenses/fdl-1.3.txt", ID: "GFDL-1.3"},
// {URL: "www.gnu.org/licenses/gcc-exception-3.1.html", ID: "GPL-3.0-with-GCC-exception"},
{URL: "www.gnu.org/licenses/gpl-3.0-standalone.html", ID: "GPL-3.0"},
// {URL: "www.gnu.org/licenses/gpl-faq.html#fontexception", ID: "GPL-2.0-with-font-exception"},
{URL: "www.gnu.org/licenses/lgpl-3.0-standalone.html", ID: "LGPL-3.0"},
{URL: "www.gnu.org/licenses/old-licenses/fdl-1.1.txt", ID: "GFDL-1.1"},
{URL: "www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", ID: "GPL-1.0"},
{URL: "www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html", ID: "GPL-2.0"},
{URL: "www.gnu.org/licenses/old-licenses/lgpl-2.0-standalone.html", ID: "LGPL-2.0"},
{URL: "www.gnu.org/licenses/old-licenses/lgpl-2.1-standalone.html", ID: "LGPL-2.1"},
{URL: "www.gnu.org/prep/maintain/html_node/license-notices-for-other-files.html", ID: "FSFAP"},
// {URL: "www.gnu.org/software/classpath/license.html", ID: "GPL-2.0-with-classpath-exception"},
{URL: "www.opensource.org/licenses/agpl-3.0", ID: "AGPL-3.0"},
{URL: "www.opensource.org/licenses/apl-1.0", ID: "APL-1.0"},
{URL: "www.opensource.org/licenses/apache-2.0", ID: "Apache-2.0"},
{URL: "www.opensource.org/licenses/bsd-2-clause", ID: "BSD-2-Clause"},
{URL: "www.opensource.org/licenses/bsd-3-clause", ID: "BSD-3-Clause"},
{URL: "www.opensource.org/licenses/bsl-1.0", ID: "BSL-1.0"},
{URL: "www.opensource.org/licenses/cnri-python", ID: "CNRI-Python"},
{URL: "www.opensource.org/licenses/cpal-1.0", ID: "CPAL-1.0"},
{URL: "www.opensource.org/licenses/epl-1.0", ID: "EPL-1.0"},
{URL: "www.opensource.org/licenses/epl-2.0", ID: "EPL-2.0"},
{URL: "www.opensource.org/licenses/eudatagrid", ID: "EUDatagrid"},
{URL: "www.opensource.org/licenses/eupl-1.1", ID: "EUPL-1.1"},
{URL: "www.opensource.org/licenses/fair", ID: "Fair"},
{URL: "www.opensource.org/licenses/frameworx-1.0", ID: "Frameworx-1.0"},
{URL: "www.opensource.org/licenses/gpl-2.0", ID: "GPL-2.0"},
{URL: "www.opensource.org/licenses/gpl-3.0", ID: "GPL-3.0"},
{URL: "www.opensource.org/licenses/hpnd", ID: "HPND"},
{URL: "www.opensource.org/licenses/ipa", ID: "IPA"},
{URL: "www.opensource.org/licenses/ipl-1.0", ID: "IPL-1.0"},
{URL: "www.opensource.org/licenses/isc", ID: "ISC"},
{URL: "www.opensource.org/licenses/lgpl-2.1", ID: "LGPL-2.1"},
{URL: "www.opensource.org/licenses/lgpl-3.0", ID: "LGPL-3.0"},
{URL: "www.opensource.org/licenses/lpl-1.02", ID: "LPL-1.02"},
{URL: "www.opensource.org/licenses/lppl-1.3c", ID: "LPPL-1.3c"},
{URL: "www.opensource.org/licenses/mit", ID: "MIT"},
{URL: "www.opensource.org/licenses/mpl-1.1", ID: "MPL-1.1"},
{URL: "www.opensource.org/licenses/ms-pl", ID: "MS-PL"},
{URL: "www.opensource.org/licenses/ms-rl", ID: "MS-RL"},
{URL: "www.opensource.org/licenses/miros", ID: "MirOS"},
{URL: "www.opensource.org/licenses/motosoto", ID: "Motosoto"},
{URL: "www.opensource.org/licenses/multics", ID: "Multics"},
{URL: "www.opensource.org/licenses/nasa-1.3", ID: "NASA-1.3"},
{URL: "www.opensource.org/licenses/ncsa", ID: "NCSA"},
{URL: "www.opensource.org/licenses/ngpl", ID: "NGPL"},
{URL: "www.opensource.org/licenses/nosl3.0", ID: "NPOSL-3.0"},
{URL: "www.opensource.org/licenses/ntp", ID: "NTP"},
{URL: "www.opensource.org/licenses/naumen", ID: "Naumen"},
{URL: "www.opensource.org/licenses/oclc-2.0", ID: "OCLC-2.0"},
{URL: "www.opensource.org/licenses/ofl-1.1", ID: "OFL-1.1"},
{URL: "www.opensource.org/licenses/ogtsl", ID: "OGTSL"},
{URL: "www.opensource.org/licenses/osl-3.0", ID: "OSL-3.0"},
{URL: "www.opensource.org/licenses/php-3.0", ID: "PHP-3.0"},
{URL: "www.opensource.org/licenses/postgresql", ID: "PostgreSQL"},
{URL: "www.opensource.org/licenses/python-2.0", ID: "Python-2.0"},
{URL: "www.opensource.org/licenses/qpl-1.0", ID: "QPL-1.0"},
{URL: "www.opensource.org/licenses/rpl-1.5", ID: "RPL-1.5"},
{URL: "www.opensource.org/licenses/rpsl-1.0", ID: "RPSL-1.0"},
{URL: "www.opensource.org/licenses/rscpl", ID: "RSCPL"},
{URL: "www.opensource.org/licenses/spl-1.0", ID: "SPL-1.0"},
{URL: "www.opensource.org/licenses/simpl-2.0", ID: "SimPL-2.0"},
{URL: "www.opensource.org/licenses/sleepycat", ID: "Sleepycat"},
{URL: "www.opensource.org/licenses/vsl-1.0", ID: "VSL-1.0"},
{URL: "www.opensource.org/licenses/w3c", ID: "W3C"},
// {URL: "www.opensource.org/licenses/wxwindows", ID: "wxWindows"},
{URL: "www.opensource.org/licenses/watcom-1.0", ID: "Watcom-1.0"},
{URL: "www.opensource.org/licenses/zlib", ID: "Zlib"},
{URL: "www.opensource.org/licenses/afl-3.0", ID: "AFL-3.0"},
{URL: "www.opensource.org/licenses/artistic-license-2.0", ID: "Artistic-2.0"},
{URL: "www.opensource.org/licenses/attribution", ID: "AAL"},
{URL: "www.opensource.org/licenses/cddl1", ID: "CDDL-1.0"},
{URL: "www.opensource.org/licenses/nokia", ID: "Nokia"},
// {URL: "opensource.org/licenses/wxwindows", ID: "wxWindows"},
}