diff options
| -rw-r--r-- | scripts/.gitignore | 2 | ||||
| -rw-r--r-- | scripts/Makefile | 6 | ||||
| -rw-r--r-- | scripts/check-licenses.py | 425 | ||||
| -rwxr-xr-x | scripts/check-licenses.py~ | 34 | ||||
| -rwxr-xr-x | scripts/check-licenses2.py~ | 165 | ||||
| -rwxr-xr-x | scripts/manifest.py | 6 | ||||
| -rw-r--r-- | scripts/tests/generate-mirror-commands/reference-mirror.sh (renamed from scripts/tests/reference-mirror.sh) | 0 |
7 files changed, 634 insertions, 4 deletions
diff --git a/scripts/.gitignore b/scripts/.gitignore index 927775a..179c022 100644 --- a/scripts/.gitignore +++ b/scripts/.gitignore @@ -1,2 +1,2 @@ __pycache__ -tests/mirror.sh +tests/generate-mirror-commands/mirror.sh diff --git a/scripts/Makefile b/scripts/Makefile index 763d794..ad28c86 100644 --- a/scripts/Makefile +++ b/scripts/Makefile @@ -1,7 +1,7 @@ .PHONY: check -check: tests/mirror.sh - cmp $< tests/reference-mirror.sh +check: tests/generate-mirror-commands/mirror.sh + cmp $< $(dir $<)reference-mirror.sh -tests/mirror.sh: ../default.xml generate-mirror-commands.py +tests/generate-mirror-commands/mirror.sh: ../default.xml generate-mirror-commands.py ./generate-mirror-commands.py $< > $@ diff --git a/scripts/check-licenses.py b/scripts/check-licenses.py new file mode 100644 index 0000000..d70807e --- /dev/null +++ b/scripts/check-licenses.py @@ -0,0 +1,425 @@ +#!/usr/bin/env python3 +# Copyright (C) 2021 Denis 'GNUtoo' Carikli <GNUtoo@cyberdimension.org> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +import os +import re +import sh +import sys + +import manifest + +debug = False +print_headers = False +verbose = False + +_data = {} + +def has_android_free_license_filename(data, repository_path): + details = {} + found = False + for filename in data.get('files', []): + path = repository_path + os.sep + filename + + if os.path.exists(path): + details[path] = {} + details[path]['free_license'] = True + details[path]['type'] = 'license-filename' + if not found: + found = True + if not debug: + return True, details + + return found, details + +_license_filenames = { + 'license-filename': { + 'func' : has_android_free_license_filename, + 'files' : [ + 'MODULE_LICENSE_BSD_LIKE', + 'MODULE_LICENSE_APACHE2', + 'MODULE_LICENSE_BSD', + 'MODULE_LICENSE_MIT', + 'MODULE_LICENSE_BSD_APL2', + 'MODULE_LICENSE_GPL', + 'MODULE_LICENSE_PUBLIC_DOMAIN', + ], + 'sha1sums' : None, + }, +} +_data.update(_license_filenames) + +################# +# License files # +################# +license_files = [ + 'COPYING', + 'LICENSE', + 'LICENCE', + 'NIST-CONDITIONS-OF-USE', +] + +stock_license_files_checksums = [ + ############## + # Apache 2.0 # + ############## + '11d8a409876496183c8fba90eb70f25301875b67', + '5a7d7df655ba40478fae80a6abafc6afc36f9b6a', + '82f88802986ad28deac953dc06bd0bb52f3d012c', + '04f15afbbe53b69d0e4870e9308efae75efa773a', + '58853eb8199b5afe72a73a25fd8cf8c94285174b', + ######### + # Boost # + ######### + #external/catch2/LICENSE + '3cba29011be2b9d59f6204d6fa0a386b1b2dbd90', + ####### + # BSD # + ####### + # external/openssh/LICENCE, also contains + # various other permissive licenses + '0ea904438c8997ec028278e2d51a0af60ef7e698', + # external/angle/LICENSE + '37126a0eda0b30f44070f59e6833187e99a7eb83', + ################# + # BSD 2 clauses # + ################# + # external/bc/LICENSE + 'e340c40163eac1621be590383a187e06a55f7a46', + # external/neon_2_sse/LICENSE + '431443c5e39f2a2f7514b387082e5ac2877923bd', + ########################################################################### + # BSD 2 clauses + Apache2 + MIT/Expat + other weak free software licenses # + ########################################################################### + # prebuilts/maven_repo/bumptech/LICENSE + '1101898f21325ac882951894f17493b08925749a', + ################# + # BSD 3 clauses # + ################# + # prebuilts/go/linux-x86/LICENSE + 'd6a5f1ecaedd723c325a2063375b3517e808a2b5', + # external/capstone/LICENSE + '861af24907e399e873920dbbff1ea1dd73a9ba35', + # external/toolchain-utils/LICENSE + 'd9005d5cecc3c5eaa7afef623b99eb38b5cdf6ac', + # external/libsrtp2/LICENSE + 'e8a22b77b707f4d8fc8cb2f86c117fd8f7251635', + # ChromeOS authors + # external/crosvm/LICENSE, external/puffin/LICENSE + '1f0a15bbb0febb59ac19144ac84f6562770372ef', + # external/vixl/LICENCE + '94836c813237bec24a2b695dce00f85cad7cb3a5', + # modifications: "The Go Authors" + year + # external/golang-protobuf/LICENSE + 'aa9b240f558caed367795f667629ccbca28f20b2', + # external/libyuv/LICENSE + 'e9acea9922263c04ea4c231e258b25e6898ec932', + ########################### + # BSD 3 clauses + unicode # + ########################### + # external/google-breakpad/LICENSE + '86dabb511ec367808a04ba4b87c4a3e0b97765e2', + + ############### + # Expat / MIT # + ############### + # external/libepoxy/COPYING + '00f34512740377ad1f155eaa15936e472661c5e3', + # prebuilts/ktlint/LICENSE + 'da2a23609257d3856b866a33d337884785513c71', + # external/python/six/LICENSE + '5a09fe8ab2f714dc1fa204fc261eceb8d82962e8', + # external/googletest/LICENSE + '5a2314153eadadc69258a9429104cd11804ea304', + # external/fonttools/LICENSE + '3b4f969a237019242665f7adacc882c35a82c145', + # external/robolectric-shadows/LICENSE + '14e5e69109cfb70093b929e7191b6dab02b37c51', + ######## + # GPL2 # + ######## + # external/ethtool/COPYING + '4cc77b90af91e615a64ae04893fdffa7939db84c', + # external/oj-libjdwp/LICENSE + 'a4fb972c240d89131ee9e16b845cd302e0ecb05f', + ############## + # GPLv2 only # + ############## + # external/autotest/LICENSE + '86f2601747ba9f6ab13cf7eabfdfe720416965bb', + ########################################## + # GPLv2 + GPLv2||LGPL 2.1 for some parts # + ########################################## + # external/f2fs-tools/COPYING + 'a55afe19b58d728b586547acb09862752d4f1b76', + # external/libusb/COPYING + 'caeb68c46fa36651acf592771d09de7937926bb3', + # external/libfuse/LICENSE + '0028a619acd3ff14a6d35b2837bb85a3ad2bab2b', + ############### + # Imagemagick # + ############### + # external/ImageMagick/LICENSE + '1de15ef06b3465e1bb922ba9c69a2a67a0263455', + ################### + # LGPL || Apache2 # + ################### + # external/javaparser/LICENSE + 'caaf43ae64cd4674b8e99869d658870b65fd6733', + ######################## + # MPLv2 + 2 clause BSD # + ######################## + # "libkms++ is mostly under the MPLv2 license except for a snippet in + # modedb_cea.cpp which is derived from 2-clause BSD licensed files from the + # Linux kernel." + # external/libkmsxx/LICENSE + 'a4ac2bea4ed2d4726528801b8880415b70f26157', + ##################################################### + # (MPLv2 || GPLv2 || LGPLv2.1) +? ||? BSD 3 clauses # + ##################################################### + # external/chromium-libpac/LICENSE + '437ba59383a90fd0033eb7b77370eb6c399b72de', + ########### + # OFL-1.1 # + ########### + # TODO: The OFL licenses are almost all the same with some variations at the + # very begining with the font name and some minor variations, so it could be + # automatically detected with code to make the maintenance easier when + # porting Replicant to newer Android versions + # external/google-fonts/source-sans-pro/LICENSE + 'a55d8584dd1af1bd63bf607a6195d56beb8ee564', + # external/google-fonts/lato/LICENSE + '76897b37e127e2332a1a79aab2e0d6f30ccdc47a', + # external/google-fonts/arvo/LICENSE + '2ef38678dab952ed745c71e59e9a2f595cddb0f4', + # external/google-fonts/big-shoulders-text/LICENSE + '35765f09b7708df97741f83ca94978a8a742adfa', + # external/google-fonts/fraunces/LICENSE + '587180e421d550a5e4bf6a6e76275c21d486e8ef', + # external/google-fonts/arbutus-slab/LICENSE + 'd9c9ee6d894d5fd114fb317a2fda29823dd90d57', + # external/google-fonts/barlow/LICENSE + '92050ce5096f0a97a4140e5be847f25901d1f327', + # external/google-fonts/karla/LICENSE + '0c63ce44ac335ff9fab43f701affbb85475d9dda', + # external/google-fonts/lustria/LICENSE + '5d0d72fede5a8ae33cb8f75ae2c9814092ad19b2', + # external/google-fonts/rubik/LICENSE + 'bf47091f200e5c46fa3032c8a143bb197f2f6a22', + # external/google-fonts/zilla-slab/LICENSE + 'e81354927f121d3ba67ef5e48b34dc930cdeca06', + ######### + # PSF 2 # + ######### + # external/python/ipaddress/LICENSE + '3f92966b8a60875dca8d9e1d6b63ac7eb9e4178e', + ############## + # Python 2.1 # + ############## + '0c492b235e749a739628cdd18d8c860c6f70396b', + # external/python/cpython2/LICENSE + 'ee7904173585b2506078dc8dac150638f1b3e537', + # external/python/cpython3/LICENSE + 'f8ec80e0c4f67f0f460f6933dcbe99f36a82f023', + ########################## + # Unlicense || Expat/MIT # + ########################## + # external/rust/crates/byteorder/COPYING + 'dd445710e6e4caccc4f8a587a130eaeebe83f6f6', + + ####### + # X11 # + ####### + # external/shaderc/spirv-headers/LICENSE + '9a84200f47e09abfbde1a6b25028460451b23d03', +] + +modified_license_files_checksums = [ + ############## + # Apache 2.0 # + ############## + '11d8a409876496183c8fba90eb70f25301875b67', + # packages/apps/Dialer/LICENSE + # Has additional apache 2005-2008 copyright header + 'b3ca008f1479a18a234094ea7169bc2dbbd141bb', + # hardware/st/secure_element/LICENSE + '5166166f89e06525e7cddb92f37f8670b3bd628f', + # external/subsampling-scale-image-view/LICENSE + '8a16eeaa6a87c42f0d7346e1f641d060bbf1f27f', + ######### + # GPLv2 # + ######### + # libcore/LICENSE + '56423feb073860b0b2de8dd6dc9053c2e291f483', + ############# + # LGPLv 2.1 # + ############# + # prebuilts/checkstyle/LICENSE + 'afb804cc9151ceb4044111bd37da309b29e844f3', + # external/kmod/COPYING + 'f0a5efd7fe620e424bc7f8dcf809f2166ae4f372', + # external/libexif/COPYING + 'e8b46065a76eacf8b16ba95038a8dadf61dcc264', + # external/libnl/COPYING + 'acd27bde2d7451c37a297c578899e29794a3008d', + # external/libiio/LICENSE + 'adfb290d0f03d365c548d0834d3276a96665b76c', + +] + +def has_stock_free_license_file(data, repository_path): + details = {} + found = False + + for filename in data.get('files', []): + path = repository_path + os.sep + filename + + if os.path.exists(path): + sha1sum = sh.sha1sum(path).split(' ')[0] + + new_found = sha1sum in data.get('sha1sums', []) + + details[path] = {} + details[path]['free_license'] = new_found + details[path]['sha1sum'] = sha1sum + details[path]['type'] = 'stock-license-file' + + if not found: + found = new_found + if not debug: + return found, details + + return found, details + +stock_license_files = { + 'stock-license-file': { + 'func' : has_stock_free_license_file, + 'files': license_files, + 'sha1sums': stock_license_files_checksums, + }, +} +_data.update(stock_license_files) + +def has_modified_free_license_file(data, repository_path): + details = {} + found = False + + for filename in data.get('files', []): + path = repository_path + os.sep + filename + + if os.path.exists(path): + # Remove all whitespaces and line breaks + buf = open(path, 'r').read() + buf = buf.replace(' ', '') + buf = buf.replace('\r', '') + buf = buf.replace('\n', '') + + sha1sum = sh.sha1sum(_in=buf).split(' ')[0] + + new_found = sha1sum in data.get('sha1sums', []) + + details[path] = {} + details[path]['free_license'] = new_found + details[path]['sha1sum'] = sha1sum + details[path]['type'] = 'modified-license-file' + + if not found: + found = new_found + if not debug: + return found, details + + return found, details + +modified_license_files = { + 'modified-license-file': { + 'func' : has_modified_free_license_file, + 'files': license_files, + 'sha1sums': modified_license_files_checksums, + }, +} + +_data.update(modified_license_files) + +def has_free_license(data, repository_path): + found = False + details = [] + + for method, method_data in data.items(): + new_found, new_details = method_data['func'](data[method], repository_path) + details.append(new_details) + + if not found: + found = new_found + if not debug and found: + return found, details + + return found, details + +def print_todo(repository_path, details): + stock_licenses_sha1sum = None + modified_licenses_sha1sum = None + license_file_path = None + for data in details: + if len(data) == 0: + continue + for file_path, values in data.items(): + if values['type'] == 'stock-license-file': + stock_licenses_sha1sum = values['sha1sum'] + if license_file_path == None: + license_file_path = file_path + else: + assert(license_file_path == file_path) + elif values['type'] == 'modified-license-file': + modified_licenses_sha1sum = values['sha1sum'] + if license_file_path == None: + license_file_path = file_path + else: + assert(license_file_path == file_path) + + if license_file_path \ + and stock_licenses_sha1sum \ + and modified_licenses_sha1sum: + if print_headers: + print("| {} | {} | {} |".format("path", "file sha1sum", + "stripped file sha1sum")) + print("| {} | {} | {} |".format(license_file_path, + stock_licenses_sha1sum, + modified_licenses_sha1sum)) + +if __name__ == '__main__': + if len(sys.argv) != 2: + usage(sys.argv[0]) + + manifest_xml_path = sys.argv[1] + + m = manifest.Manifest(manifest_xml_path) + paths = m.list_repositories_paths() + + nr_unknown_repositories = len(paths) + for path in paths: + if verbose: + print(path) + found, details = has_free_license(_data, path) + if found: + nr_unknown_repositories -= 1 + if verbose: + print(found, details) + print() + if not found and len(details) > 0: + print_todo(path, details) + + print("Remaining: {} done: {} total: {}".format( + nr_unknown_repositories, + len(paths) - nr_unknown_repositories, + len(paths))) diff --git a/scripts/check-licenses.py~ b/scripts/check-licenses.py~ new file mode 100755 index 0000000..f3f022c --- /dev/null +++ b/scripts/check-licenses.py~ @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# Copyright (C) 2021 Denis 'GNUtoo' Carikli <GNUtoo@cyberdimension.org> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. + +import sys + +import manifest + +def usage(progname): + print ('Usage:') + print ('\t{} <path/to/default.xml>'.format(progname)) + sys.exit(1) + +if __name__ == '__main__': + if len(sys.argv) != 2: + usage(sys.argv[0]) + + manifest_xml_path = sys.argv[1] + + m = manifest.Manifest(manifest_xml_path) + m.parse() + m.close() diff --git a/scripts/check-licenses2.py~ b/scripts/check-licenses2.py~ new file mode 100755 index 0000000..f1973f4 --- /dev/null +++ b/scripts/check-licenses2.py~ @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +# Copyright (C) 2021 Denis 'GNUtoo' Carikli <GNUtoo@cyberdimension.org> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. + +_data = {} + +def has_android_free_license_filename(data, repository_path): + pass + +_license_filenames = { + 'license-filename': { + 'func' : has_android_free_license_filename, + 'files' : [ + 'MODULE_LICENSE_BSD_LIKE', + 'MODULE_LICENSE_APACHE2', + 'MODULE_LICENSE_BSD', + 'MODULE_LICENSE_MIT', + 'MODULE_LICENSE_BSD_APL2', + 'MODULE_LICENSE_GPL', + 'MODULE_LICENSE_PUBLIC_DOMAIN', + ], + 'sha1sums' : None, + }, +} +_data.update(_license_filenames) + +################# +# License files # +################# +license_files = [ + 'COPYING', + 'LICENSE', + 'LICENCE', + 'NIST-CONDITIONS-OF-USE', +] + +stock_license_files_checksums = [ + ############## + # Apache 2.0 # + ############## + '11d8a409876496183c8fba90eb70f25301875b67', + '5a7d7df655ba40478fae80a6abafc6afc36f9b6a', + '82f88802986ad28deac953dc06bd0bb52f3d012c', + '04f15afbbe53b69d0e4870e9308efae75efa773a', + '58853eb8199b5afe72a73a25fd8cf8c94285174b', + ####### + # BSD # + ####### + # external/openssh/LICENCE, also contains + # various other permissive licenses + '0ea904438c8997ec028278e2d51a0af60ef7e698', + ############### + # Expat / MIT # + ############### + # external/libepoxy/COPYING + '00f34512740377ad1f155eaa15936e472661c5e3', + ########### + # OFL-1.1 # + ########### + # external/google-fonts/source-sans-pro/LICENSE + 'a55d8584dd1af1bd63bf607a6195d56beb8ee564', + # external/google-fonts/lato/LICENSE + '76897b37e127e2332a1a79aab2e0d6f30ccdc47a', + # external/google-fonts/arvo/LICENSE + '2ef38678dab952ed745c71e59e9a2f595cddb0f4', + # external/google-fonts/big-shoulders-text/LICENSE + '35765f09b7708df97741f83ca94978a8a742adfa', + # external/google-fonts/fraunces/LICENSE + '587180e421d550a5e4bf6a6e76275c21d486e8ef', + ############## + # Python 2.1 # + ############## + '0c492b235e749a739628cdd18d8c860c6f70396b', +] + +modified_license_files_checksums = [ + # Apache 2.0 + '11d8a409876496183c8fba90eb70f25301875b67', +] + +def has_stock_free_license_file(path): + pass + +stock_license_files = { + 'stock-license-file': { + 'func' : has_stock_free_license_file, + 'files': license_files, + 'sha1sums': stock_license_files_checksums, + }, +} +_data.update(stock_license_files) + +def has_modified_free_license_file(path): + pass + +modified_license_files = { + 'modified-license-file': { + 'func' : has_modified_free_license_file, + 'files': license_files, + 'sha1sums': modified_license_files_checksums, + }, +} + +_data.update(modified_license_files) + + +if __name__ == '__main__': + verbose = True + if len(sys.argv) != 2: + usage(sys.argv[0]) + + manifest_xml_path = sys.argv[1] + + m = manifest.Manifest(manifest_xml_path) + paths = m.list_repositories_paths() + + nr_unknown_repositories = 0 + nr_unknown_licenses = 0 + for path in paths: + l = RepositoryLicense(path) + found, details = l.has_free_license() + if not found and verbose: + nr_unknown_repositories += 1 + if len(details) > 0: + for k, v in details.items(): + printed_header = False + if v['type'] in ['stock-license-file', + 'modified-license-file']: + if not printed_header: + print("{}:".format(path)) + printed_header = True + print("- {} {}".format(k, v['sha1sum'])) + nr_unknown_licenses += 1 + # We'll handle repositories with READMEs when we + # will have handled all the licenses files + # else: + # print("- TODO: look at the READMEs") + # We'll handle these repositories when we + # will have handled all the licenses files + # else: + # print("{}".format(path)) + + if verbose: + if nr_unknown_repositories > 0 or nr_unknown_licenses > 0: + print() + print("Total:") + print("------") + if nr_unknown_repositories > 0: + print("- {} repositories to check".format(nr_unknown_repositories)) + if nr_unknown_licenses > 0: + print("- {} licenses to check".format(nr_unknown_licenses)) + + diff --git a/scripts/manifest.py b/scripts/manifest.py index d1188e6..065ee7f 100755 --- a/scripts/manifest.py +++ b/scripts/manifest.py @@ -184,5 +184,11 @@ class Manifest(object): print(command) print('') + def list_repositories_paths(self): + paths = [] + for child in self.root.iter('project'): + paths.append(self.get_project_property(child, 'path')) + return paths + def close(self): self.xml_file.close() diff --git a/scripts/tests/reference-mirror.sh b/scripts/tests/generate-mirror-commands/reference-mirror.sh index 7d9ed43..7d9ed43 100644 --- a/scripts/tests/reference-mirror.sh +++ b/scripts/tests/generate-mirror-commands/reference-mirror.sh |