Add script to help mirror repositories

Signed-off-by: Denis 'GNUtoo' Carikli <GNUtoo@cyberdimension.org>

1 files changed, 169 insertions, 0 deletions

diff --git a/scripts/generate-mirror-commands.py b/scripts/generate-mirror-commands.py
new file mode 100755
index 0000000..6349bee
--- /dev/null
+++ b/scripts/generate-mirror-commands.py
@@ -0,0 +1,169 @@
+#!/usr/bin/env python3
+# Copyright (C) 2021 Denis 'GNUtoo' Carikli <GNUtoo@cyberdimension.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import re
+import sys
+
+# Etree isn't subject to any data leak vulnerabilities:
+# +---------------------------+------------+-----------------------------------+
+# | Issue                     | Etree      | Issue description                 |
+# +---------------------------+------------+-----------------------------------+
+# | billion laughs            | Vulnerable | DDOS by huge CPU and memory usage |
+# +---------------------------+------------+-----------------------------------+
+# | quadratic blowup          | Vulnerable | DDOS by huge CPU and memory usage |
+# +---------------------------+------------+-----------------------------------+
+# | external entity expansion | Safe (1)   | Data leak                         |
+# +---------------------------+------------+-----------------------------------+
+# | DTD retrieval             | Safe       | Data leak                         |
+# +---------------------------+------------+-----------------------------------+
+# | decompression bomb        | Safe       | DDOS by huge amount of CPU        |
+# +---------------------------+------------+-----------------------------------+
+# (1) xml.etree.ElementTree doesn’t expand external entities and raises a
+#     ParserError when an entity occurs.
+#
+# Other XML parsers like sax, minidom, pulldom, xmlrpc have similar security
+# properties: they are not vulnerable (anymore) to data leaks but they are still
+# vulnerable to DDOS attacks.
+# Reference: https://docs.python.org/3.8/library/xml.html#xml-vulnerabilities
+import xml.etree.ElementTree
+
+import jinja2
+
+def usage(progname):
+    print ('Usage:')
+    print ('\t{} <path/to/default.xml>'.format(progname))
+    sys.exit(1)
+
+class Manifest(object):
+    def __init__(self, manifest_xml_path):
+        self.remotes = {}
+        self.xml_file = open(manifest_xml_path, 'r')
+        tree = xml.etree.ElementTree.parse(self.xml_file)
+        self.root = tree.getroot()
+        self.defaults = self._get_defaults(self.root)
+
+    def _get_defaults(self, root):
+        for child in root.iter('default'):
+            return child
+
+    def get_remote(self, name):
+        if name in self.remotes.keys():
+            return self.remotes.get(name)
+
+        for child in self.root.iter('remote'):
+            if child.get('name') == name:
+                self.remotes[name] = child
+                return child
+
+    def get_project_property(self, project, prop):
+        if prop in project.keys():
+            return project.get(prop)
+        else:
+            return self.defaults.get(prop)
+
+    def get_clone_url(self, elm):
+        repo_path = re.sub('/*$', '', self.get_project_property(elm, 'name'))
+        remote_name = self.get_project_property(elm, 'remote')
+        remote_url = re.sub('/*$', '',
+                            self.get_remote(remote_name).get('fetch'))
+
+        return "{}/{}".format(remote_url, repo_path)
+
+    def get_revision(self, elm):
+        if 'revision' in elm.keys():
+            return elm.get('revision')
+
+        remote = self.get_remote(self.get_project_property(elm, 'remote'))
+        if 'revision' in remote:
+            return remote.get('revision')
+        elif self.defaults.get('remote') == remote.get('name'):
+            return self.defaults.get('revision')
+
+        assert(False)
+
+    def get_base_directory(self, given_remote):
+        # Since the commands are generated it's a good idea to do our
+        # best not to produce potentially dangerous commands. The
+        # downside is that the list above will need to be updated
+        # automatically
+        #
+        # TODO: android-x86  ccache  F-Droid LineageOS
+        whitelist = {
+            'aosp' : {
+                'fetch'   : 'https://android.googlesource.com',
+                'dirname' : 'mirrors/AOSP',
+            },
+            'freedesktop' : {
+                'fetch' : 'https://gitlab.freedesktop.org',
+                'dirname' : 'mirrors/freesmartphone.org',
+            },
+        }
+
+        assert(given_remote.get('name') in whitelist)
+        whiltelist_renote = whitelist.get(given_remote.get('name'))
+        whitelist_fetch_url = re.sub('/*$', '', whiltelist_renote.get('fetch'))
+        given_remote_fetch_url = re.sub('/*$', '', given_remote.get('fetch'))
+        assert (whitelist_fetch_url == given_remote_fetch_url)
+
+        return whitelist.get(given_remote.get('name')).get('dirname')
+
+    def get_clone_commands(self, elm):
+        commands = []
+
+        url = self.get_clone_url(elm)
+        remote = self.get_remote(self.get_project_property(elm, 'remote'))
+        base_directory = self.get_base_directory(remote)
+        repo_directory = self.get_project_property(elm,
+                                                   'name').replace('/', '_')
+        revision = self.get_revision(elm)
+
+        commands.append("if [ ! -d {}/{}.git ] ; then".format(
+            base_directory, repo_directory))
+
+        commands.append("    git clone --mirror {} {}/{}.git".format(
+            url, base_directory, repo_directory))
+
+        commands.append("else")
+
+        commands.append("    git -C {}/{}.git fetch {} {}".format(
+            base_directory, repo_directory, url, revision))
+
+        commands.append("    touch {}/{}.git/git-daemon-export-ok".format(
+            base_directory, repo_directory))
+
+        commands.append("fi")
+
+        return commands
+
+    def parse(self):
+        for child in self.root.iter('project'):
+            print("# {}".format(self.get_project_property(child, 'name')))
+            for command in self.get_clone_commands(child):
+                print(command)
+            print('')
+
+    def close(self):
+        self.xml_file.close()
+
+if __name__ == '__main__':
+    if len(sys.argv) != 2:
+           usage(sys.argv[0])
+
+    manifest_xml_path = sys.argv[1]
+
+    manifest = Manifest(manifest_xml_path)
+    manifest.parse()
+    manifest.close()