scripts: Move Manifest in its own file

This enables to reuse the manifest parsing code other python
scripts.

Signed-off-by: Denis 'GNUtoo' Carikli <GNUtoo@cyberdimension.org>

3 files changed, 193 insertions, 172 deletions

diff --git a/scripts/.gitignore b/scripts/.gitignore
index 452bdbe..927775a 100644
--- a/scripts/.gitignore
+++ b/scripts/.gitignore
@@ -1 +1,2 @@
+__pycache__
 tests/mirror.sh
diff --git a/scripts/generate-mirror-commands.py b/scripts/generate-mirror-commands.py
index 946c33f..bf7a969 100755
--- a/scripts/generate-mirror-commands.py
+++ b/scripts/generate-mirror-commands.py
@@ -14,189 +14,21 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-import re
 import sys
 
-# Etree isn't subject to any data leak vulnerabilities:
-# +---------------------------+------------+-----------------------------------+
-# | Issue                     | Etree      | Issue description                 |
-# +---------------------------+------------+-----------------------------------+
-# | billion laughs            | Vulnerable | DDOS by huge CPU and memory usage |
-# +---------------------------+------------+-----------------------------------+
-# | quadratic blowup          | Vulnerable | DDOS by huge CPU and memory usage |
-# +---------------------------+------------+-----------------------------------+
-# | external entity expansion | Safe (1)   | Data leak                         |
-# +---------------------------+------------+-----------------------------------+
-# | DTD retrieval             | Safe       | Data leak                         |
-# +---------------------------+------------+-----------------------------------+
-# | decompression bomb        | Safe       | DDOS by huge amount of CPU        |
-# +---------------------------+------------+-----------------------------------+
-# (1) xml.etree.ElementTree doesn’t expand external entities and raises a
-#     ParserError when an entity occurs.
-#
-# Other XML parsers like sax, minidom, pulldom, xmlrpc have similar security
-# properties: they are not vulnerable (anymore) to data leaks but they are still
-# vulnerable to DDOS attacks.
-# Reference: https://docs.python.org/3.8/library/xml.html#xml-vulnerabilities
-import xml.etree.ElementTree
+import manifest
 
 def usage(progname):
     print ('Usage:')
     print ('\t{} <path/to/default.xml>'.format(progname))
     sys.exit(1)
 
-class Manifest(object):
-    def __init__(self, manifest_xml_path):
-        self.remotes = {}
-        self.xml_file = open(manifest_xml_path, 'r')
-        tree = xml.etree.ElementTree.parse(self.xml_file)
-        self.root = tree.getroot()
-        self.defaults = self._get_defaults(self.root)
-
-    def _get_defaults(self, root):
-        for child in root.iter('default'):
-            return child
-
-    def get_remote(self, name):
-        if name in self.remotes.keys():
-            return self.remotes.get(name)
-
-        for child in self.root.iter('remote'):
-            if child.get('name') == name:
-                self.remotes[name] = child
-                return child
-
-    def get_project_property(self, project, prop):
-        if prop in project.keys():
-            return project.get(prop)
-        else:
-            return self.defaults.get(prop)
-
-    def get_clone_url(self, elm):
-        repo_path = re.sub('/*$', '', self.get_project_property(elm, 'name'))
-        remote_name = self.get_project_property(elm, 'remote')
-        remote_url = re.sub('/*$', '',
-                            self.get_remote(remote_name).get('fetch'))
-
-        return "{}/{}".format(remote_url, repo_path)
-
-    def get_revision(self, elm):
-        if 'revision' in elm.keys():
-            return elm.get('revision')
-
-        remote = self.get_remote(self.get_project_property(elm, 'remote'))
-        if 'revision' in remote:
-            return remote.get('revision')
-        elif self.defaults.get('remote') == remote.get('name'):
-            return self.defaults.get('revision')
-
-        assert(False)
-
-    def get_base_directory(self, given_remote):
-        # Since the commands are generated it's a good idea to do our
-        # best not to produce potentially dangerous commands. The
-        # downside is that the list above will need to be updated
-        # automatically
-        #
-        # TODO: android-x86  ccache  F-Droid LineageOS
-        whitelist = {
-            'aosp' : {
-                'fetch'   : 'https://android.googlesource.com',
-                'dirname' : 'mirrors/AOSP',
-            },
-        }
-
-        assert(given_remote.get('name') in whitelist)
-        whiltelist_renote = whitelist.get(given_remote.get('name'))
-        whitelist_fetch_url = re.sub('/*$', '', whiltelist_renote.get('fetch'))
-        given_remote_fetch_url = re.sub('/*$', '', given_remote.get('fetch'))
-        assert (whitelist_fetch_url == given_remote_fetch_url)
-
-        return whitelist.get(given_remote.get('name')).get('dirname')
-
-    def is_revision_whitelisted(self, elm):
-        remote = self.get_project_property(elm, 'remote')
-        revision = self.get_revision(elm)
-        if remote == 'aosp' and revision == 'refs/tags/android-11.0.0_r17':
-            return True
-        if remote == 'freedesktop' and \
-           revision == '32819fe45972e0c706423d71075788a5885f7b86':
-            return True
-
-        return False
-
-    def is_revision_blacklisted(self, elm):
-        remote = self.get_project_property(elm, 'remote')
-        revision = self.get_revision(elm)
-        if revision == 'master':
-            return True
-        # TODO: Add support for generating a correct dirname
-        # for the freedesktop mirrors
-        elif remote == 'freedesktop':
-            return True
-
-        return False
-
-    def get_clone_commands(self, elm):
-        if self.is_revision_blacklisted(elm):
-            return []
-
-        commands = []
-        url = self.get_clone_url(elm)
-        remote = self.get_remote(self.get_project_property(elm, 'remote'))
-        base_directory = self.get_base_directory(remote)
-        repo_directory = self.get_project_property(elm,
-                                                   'name').replace('/', '_')
-        revision = self.get_revision(elm)
-
-        # Guard against updating the master branch of mirrors that could be used
-        # by several Replicant versions at the same time by forcing users to
-        # manually declare revisions here. The revisions have to be either
-        # whitelisted or blacklisted else we abort.
-        if not self.is_revision_whitelisted(elm) and \
-           not self.is_revision_blacklisted(elm):
-            print('/!\ The "{}" revision of {}/{} is not known by {}'.format(
-                revision, base_directory, repo_directory, sys.argv[0]))
-            print('    Please add it either to the whitelist in {}'.format(
-                'is_revision_whitelisted'))
-            print('    or to the blacklist in {}'.format(
-                'is_revision_blacklisted'))
-            assert(False)
-
-        commands.append("if [ ! -d {}/{}.git ] ; then".format(
-            base_directory, repo_directory))
-
-        commands.append("    git clone --mirror {} {}/{}.git".format(
-            url, base_directory, repo_directory))
-
-        commands.append("else")
-
-        commands.append("    git -C {}/{}.git fetch {} {}".format(
-            base_directory, repo_directory, url, revision))
-
-        commands.append("    touch {}/{}.git/git-daemon-export-ok".format(
-            base_directory, repo_directory))
-
-        commands.append("fi")
-
-        return commands
-
-    def parse(self):
-        for child in self.root.iter('project'):
-            print("# {}".format(self.get_project_property(child, 'name')))
-            for command in self.get_clone_commands(child):
-                print(command)
-            print('')
-
-    def close(self):
-        self.xml_file.close()
-
 if __name__ == '__main__':
     if len(sys.argv) != 2:
            usage(sys.argv[0])
 
     manifest_xml_path = sys.argv[1]
 
-    manifest = Manifest(manifest_xml_path)
-    manifest.parse()
-    manifest.close()
+    m = manifest.Manifest(manifest_xml_path)
+    m.generate_mirror_commands()
+    m.close()
diff --git a/scripts/manifest.py b/scripts/manifest.py
new file mode 100755
index 0000000..d1188e6
--- /dev/null
+++ b/scripts/manifest.py
@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+# Copyright (C) 2021 Denis 'GNUtoo' Carikli <GNUtoo@cyberdimension.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import re
+import sys
+
+# Etree isn't subject to any data leak vulnerabilities:
+# +---------------------------+------------+-----------------------------------+
+# | Issue                     | Etree      | Issue description                 |
+# +---------------------------+------------+-----------------------------------+
+# | billion laughs            | Vulnerable | DDOS by huge CPU and memory usage |
+# +---------------------------+------------+-----------------------------------+
+# | quadratic blowup          | Vulnerable | DDOS by huge CPU and memory usage |
+# +---------------------------+------------+-----------------------------------+
+# | external entity expansion | Safe (1)   | Data leak                         |
+# +---------------------------+------------+-----------------------------------+
+# | DTD retrieval             | Safe       | Data leak                         |
+# +---------------------------+------------+-----------------------------------+
+# | decompression bomb        | Safe       | DDOS by huge amount of CPU        |
+# +---------------------------+------------+-----------------------------------+
+# (1) xml.etree.ElementTree doesn’t expand external entities and raises a
+#     ParserError when an entity occurs.
+#
+# Other XML parsers like sax, minidom, pulldom, xmlrpc have similar security
+# properties: they are not vulnerable (anymore) to data leaks but they are still
+# vulnerable to DDOS attacks.
+# Reference: https://docs.python.org/3.8/library/xml.html#xml-vulnerabilities
+import xml.etree.ElementTree
+
+class Manifest(object):
+    def __init__(self, manifest_xml_path):
+        self.remotes = {}
+        self.xml_file = open(manifest_xml_path, 'r')
+        tree = xml.etree.ElementTree.parse(self.xml_file)
+        self.root = tree.getroot()
+        self.defaults = self._get_defaults(self.root)
+
+    def _get_defaults(self, root):
+        for child in root.iter('default'):
+            return child
+
+    def get_remote(self, name):
+        if name in self.remotes.keys():
+            return self.remotes.get(name)
+
+        for child in self.root.iter('remote'):
+            if child.get('name') == name:
+                self.remotes[name] = child
+                return child
+
+    def get_project_property(self, project, prop):
+        if prop in project.keys():
+            return project.get(prop)
+        else:
+            return self.defaults.get(prop)
+
+    def get_clone_url(self, elm):
+        repo_path = re.sub('/*$', '', self.get_project_property(elm, 'name'))
+        remote_name = self.get_project_property(elm, 'remote')
+        remote_url = re.sub('/*$', '',
+                            self.get_remote(remote_name).get('fetch'))
+
+        return "{}/{}".format(remote_url, repo_path)
+
+    def get_revision(self, elm):
+        if 'revision' in elm.keys():
+            return elm.get('revision')
+
+        remote = self.get_remote(self.get_project_property(elm, 'remote'))
+        if 'revision' in remote:
+            return remote.get('revision')
+        elif self.defaults.get('remote') == remote.get('name'):
+            return self.defaults.get('revision')
+
+        assert(False)
+
+    def get_base_directory(self, given_remote):
+        # Since the commands are generated it's a good idea to do our
+        # best not to produce potentially dangerous commands. The
+        # downside is that the list above will need to be updated
+        # automatically
+        #
+        # TODO: android-x86  ccache  F-Droid LineageOS
+        whitelist = {
+            'aosp' : {
+                'fetch'   : 'https://android.googlesource.com',
+                'dirname' : 'mirrors/AOSP',
+            },
+        }
+
+        assert(given_remote.get('name') in whitelist)
+        whiltelist_renote = whitelist.get(given_remote.get('name'))
+        whitelist_fetch_url = re.sub('/*$', '', whiltelist_renote.get('fetch'))
+        given_remote_fetch_url = re.sub('/*$', '', given_remote.get('fetch'))
+        assert (whitelist_fetch_url == given_remote_fetch_url)
+
+        return whitelist.get(given_remote.get('name')).get('dirname')
+
+    def is_revision_whitelisted(self, elm):
+        remote = self.get_project_property(elm, 'remote')
+        revision = self.get_revision(elm)
+        if remote == 'aosp' and revision == 'refs/tags/android-11.0.0_r17':
+            return True
+        if remote == 'freedesktop' and \
+           revision == '32819fe45972e0c706423d71075788a5885f7b86':
+            return True
+
+        return False
+
+    def is_revision_blacklisted(self, elm):
+        remote = self.get_project_property(elm, 'remote')
+        revision = self.get_revision(elm)
+        if revision == 'master':
+            return True
+        # TODO: Add support for generating a correct dirname
+        # for the freedesktop mirrors
+        elif remote == 'freedesktop':
+            return True
+
+        return False
+
+    def get_clone_commands(self, elm):
+        if self.is_revision_blacklisted(elm):
+            return []
+
+        commands = []
+        url = self.get_clone_url(elm)
+        remote = self.get_remote(self.get_project_property(elm, 'remote'))
+        base_directory = self.get_base_directory(remote)
+        repo_directory = self.get_project_property(elm,
+                                                   'name').replace('/', '_')
+        revision = self.get_revision(elm)
+
+        # Guard against updating the master branch of mirrors that could be used
+        # by several Replicant versions at the same time by forcing users to
+        # manually declare revisions here. The revisions have to be either
+        # whitelisted or blacklisted else we abort.
+        if not self.is_revision_whitelisted(elm) and \
+           not self.is_revision_blacklisted(elm):
+            print('/!\ The "{}" revision of {}/{} is not known by this python script'.format(
+                revision, base_directory, repo_directory, self.progname))
+            print('    Please add it either to the whitelist in {}'.format(
+                'is_revision_whitelisted'))
+            print('    or to the blacklist in {}'.format(
+                'is_revision_blacklisted'))
+            print('    in the manifest.py file')
+            assert(False)
+
+        commands.append("if [ ! -d {}/{}.git ] ; then".format(
+            base_directory, repo_directory))
+
+        commands.append("    git clone --mirror {} {}/{}.git".format(
+            url, base_directory, repo_directory))
+
+        commands.append("else")
+
+        commands.append("    git -C {}/{}.git fetch {} {}".format(
+            base_directory, repo_directory, url, revision))
+
+        commands.append("    touch {}/{}.git/git-daemon-export-ok".format(
+            base_directory, repo_directory))
+
+        commands.append("fi")
+
+        return commands
+
+    def generate_mirror_commands(self):
+        for child in self.root.iter('project'):
+            print("# {}".format(self.get_project_property(child, 'name')))
+            for command in self.get_clone_commands(child):
+                print(command)
+            print('')
+
+    def close(self):
+        self.xml_file.close()