Move tzdata scripts from bionic to external/icu

The scripts affect files in external/icu, bionic
and the generated files also affect libcore. The files must
be updated together so there is no "obvious" home.

OEM developers seem to want to update ICU
themselves and have been asking how. Moving the
scripts to external/icu and splitting the ICU generation
code into a sub-script they can run makes some sense.

This change includes some small changes to keep the
scripts working. A later commit will contain more extensive
changes.

Bug: 23419215

Change-Id: I056104ff133c8e35e8ab65a51e1abddec44fcfb7

2 files changed, 462 insertions, 0 deletions

diff --git a/tools/ZoneCompactor.java b/tools/ZoneCompactor.java
new file mode 100644
index 000000000..2d598fec0
--- /dev/null
+++ b/tools/ZoneCompactor.java
@@ -0,0 +1,192 @@
+
+import java.io.*;
+import java.util.*;
+
+// usage: java ZoneCompiler <setup file> <data directory> <output directory> <tzdata version>
+//
+// Compile a set of tzfile-formatted files into a single file containing an index.
+//
+// The compilation is controlled by a setup file, which is provided as a
+// command-line argument.  The setup file has the form:
+//
+// Link <toName> <fromName>
+// ...
+// <zone filename>
+// ...
+//
+// Note that the links must be declared prior to the zone names.
+// A zone name is a filename relative to the source directory such as
+// 'GMT', 'Africa/Dakar', or 'America/Argentina/Jujuy'.
+//
+// Use the 'zic' command-line tool to convert from flat files
+// (such as 'africa' or 'northamerica') to a directory
+// hierarchy suitable for this tool (containing files such as 'data/Africa/Abidjan').
+//
+
+public class ZoneCompactor {
+  // Maximum number of characters in a zone name, including '\0' terminator.
+  private static final int MAXNAME = 40;
+
+  // Zone name synonyms.
+  private Map<String,String> links = new HashMap<String,String>();
+
+  // File offsets by zone name.
+  private Map<String,Integer> offsets = new HashMap<String,Integer>();
+
+  // File lengths by zone name.
+  private Map<String,Integer> lengths = new HashMap<String,Integer>();
+
+  // Concatenate the contents of 'inFile' onto 'out'.
+  private static void copyFile(File inFile, OutputStream out) throws Exception {
+    byte[] ret = new byte[0];
+
+    InputStream in = new FileInputStream(inFile);
+    byte[] buf = new byte[8192];
+    while (true) {
+      int nbytes = in.read(buf);
+      if (nbytes == -1) {
+        break;
+      }
+      out.write(buf, 0, nbytes);
+
+      byte[] nret = new byte[ret.length + nbytes];
+      System.arraycopy(ret, 0, nret, 0, ret.length);
+      System.arraycopy(buf, 0, nret, ret.length, nbytes);
+      ret = nret;
+    }
+    out.flush();
+  }
+
+  public ZoneCompactor(String setupFile, String dataDirectory, String zoneTabFile, String outputDirectory, String version) throws Exception {
+    // Read the setup file and concatenate all the data.
+    ByteArrayOutputStream allData = new ByteArrayOutputStream();
+    BufferedReader reader = new BufferedReader(new FileReader(setupFile));
+    String s;
+    int offset = 0;
+    while ((s = reader.readLine()) != null) {
+      s = s.trim();
+      if (s.startsWith("Link")) {
+        StringTokenizer st = new StringTokenizer(s);
+        st.nextToken();
+        String to = st.nextToken();
+        String from = st.nextToken();
+        links.put(from, to);
+      } else {
+        String link = links.get(s);
+        if (link == null) {
+          File sourceFile = new File(dataDirectory, s);
+          long length = sourceFile.length();
+          offsets.put(s, offset);
+          lengths.put(s, (int) length);
+
+          offset += length;
+          copyFile(sourceFile, allData);
+        }
+      }
+    }
+    reader.close();
+
+    // Fill in fields for links.
+    Iterator<String> it = links.keySet().iterator();
+    while (it.hasNext()) {
+      String from = it.next();
+      String to = links.get(from);
+
+      offsets.put(from, offsets.get(to));
+      lengths.put(from, lengths.get(to));
+    }
+
+    // Create/truncate the destination file.
+    RandomAccessFile f = new RandomAccessFile(new File(outputDirectory, "tzdata"), "rw");
+    f.setLength(0);
+
+    // Write the header.
+
+    // byte[12] tzdata_version -- 'tzdata2012f\0'
+    // int index_offset -- so we can slip in extra header fields in a backwards-compatible way
+    // int data_offset
+    // int zonetab_offset
+
+    // tzdata_version
+    f.write(toAscii(new byte[12], version));
+
+    // Write dummy values for the three offsets, and remember where we need to seek back to later
+    // when we have the real values.
+    int index_offset_offset = (int) f.getFilePointer();
+    f.writeInt(0);
+    int data_offset_offset = (int) f.getFilePointer();
+    f.writeInt(0);
+    int zonetab_offset_offset = (int) f.getFilePointer();
+    f.writeInt(0);
+
+    int index_offset = (int) f.getFilePointer();
+
+    // Write the index.
+    ArrayList<String> sortedOlsonIds = new ArrayList<String>();
+    sortedOlsonIds.addAll(offsets.keySet());
+    Collections.sort(sortedOlsonIds);
+    it = sortedOlsonIds.iterator();
+    while (it.hasNext()) {
+      String zoneName = it.next();
+      if (zoneName.length() >= MAXNAME) {
+        throw new RuntimeException("zone filename too long: " + zoneName.length());
+      }
+
+      // Follow the chain of links to work out where the real data for this zone lives.
+      String actualZoneName = zoneName;
+      while (links.get(actualZoneName) != null) {
+        actualZoneName = links.get(actualZoneName);
+      }
+
+      f.write(toAscii(new byte[MAXNAME], zoneName));
+      f.writeInt(offsets.get(actualZoneName));
+      f.writeInt(lengths.get(actualZoneName));
+      f.writeInt(0); // Used to be raw GMT offset. No longer used.
+    }
+
+    int data_offset = (int) f.getFilePointer();
+
+    // Write the data.
+    f.write(allData.toByteArray());
+
+    int zonetab_offset = (int) f.getFilePointer();
+
+    // Copy the zone.tab.
+    reader = new BufferedReader(new FileReader(zoneTabFile));
+    while ((s = reader.readLine()) != null) {
+      if (!s.startsWith("#")) {
+        f.writeBytes(s);
+        f.write('\n');
+      }
+    }
+    reader.close();
+
+    // Go back and fix up the offsets in the header.
+    f.seek(index_offset_offset);
+    f.writeInt(index_offset);
+    f.seek(data_offset_offset);
+    f.writeInt(data_offset);
+    f.seek(zonetab_offset_offset);
+    f.writeInt(zonetab_offset);
+
+    f.close();
+  }
+
+  private static byte[] toAscii(byte[] dst, String src) {
+    for (int i = 0; i < src.length(); ++i) {
+      if (src.charAt(i) > '~') {
+        throw new RuntimeException("non-ASCII string: " + src);
+      }
+      dst[i] = (byte) src.charAt(i);
+    }
+    return dst;
+  }
+
+  public static void main(String[] args) throws Exception {
+    if (args.length != 5) {
+      System.err.println("usage: java ZoneCompactor <setup file> <data directory> <zone.tab file> <output directory> <tzdata version>");
+      System.exit(0);
+    }
+    new ZoneCompactor(args[0], args[1], args[2], args[3], args[4]);
+  }
+}
diff --git a/tools/update-tzdata.py b/tools/update-tzdata.py
new file mode 100755
index 000000000..8caa8efa3
--- /dev/null
+++ b/tools/update-tzdata.py
@@ -0,0 +1,270 @@
+#!/usr/bin/python
+
+"""Updates the timezone data held in bionic and ICU."""
+
+import ftplib
+import glob
+import httplib
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tarfile
+import tempfile
+
+regions = ['africa', 'antarctica', 'asia', 'australasia',
+           'etcetera', 'europe', 'northamerica', 'southamerica',
+           # These two deliberately come last so they override what came
+           # before (and each other).
+           'backward', 'backzone' ]
+
+def CheckDirExists(dir, dirname):
+  if not os.path.isdir(dir):
+    print "Couldn't find %s (%s)!" % (dirname, dir)
+    sys.exit(1)
+
+def GetAndroidRootOrDie():
+  value = os.environ.get('ANDROID_BUILD_TOP')
+  if not value:
+    print "ANDROID_BUILD_TOP not defined: run envsetup.sh / lunch"
+    sys.exit(1);
+  CheckDirExists(value, '$ANDROID_BUILD_TOP')
+  return value
+
+# Find the bionic directory, searching upward from this script.
+android_build_top = GetAndroidRootOrDie()
+bionic_dir = os.path.realpath('%s/bionic' % android_build_top)
+bionic_libc_zoneinfo_dir = '%s/libc/zoneinfo' % bionic_dir
+CheckDirExists(bionic_libc_zoneinfo_dir, 'bionic/libc/zoneinfo')
+tools_dir = '%s/external/icu/tools' % android_build_top
+CheckDirExists(tools_dir, 'external/icu/tools')
+print 'Found bionic in %s ...' % bionic_dir
+
+# Find the icu directory.
+icu_dir = os.path.realpath('%s/external/icu' % android_build_top)
+icu4c_dir = os.path.realpath('%s/icu4c/source' % icu_dir)
+icu4j_dir = os.path.realpath('%s/icu4j' % icu_dir)
+CheckDirExists(icu4c_dir, 'external/icu/icu4c/source')
+CheckDirExists(icu4j_dir, 'external/icu/icu4j')
+print 'Found icu in %s ...' % icu_dir
+
+
+def GetCurrentTzDataVersion():
+  return open('%s/tzdata' % bionic_libc_zoneinfo_dir).read().split('\x00', 1)[0]
+
+
+def WriteSetupFile():
+  """Writes the list of zones that ZoneCompactor should process."""
+  links = []
+  zones = []
+  for region in regions:
+    for line in open('extracted/%s' % region):
+      fields = line.split()
+      if fields:
+        if fields[0] == 'Link':
+          links.append('%s %s %s' % (fields[0], fields[1], fields[2]))
+          zones.append(fields[2])
+        elif fields[0] == 'Zone':
+          zones.append(fields[1])
+  zones.sort()
+
+  setup = open('setup', 'w')
+  for link in sorted(set(links)):
+    setup.write('%s\n' % link)
+  for zone in sorted(set(zones)):
+    setup.write('%s\n' % zone)
+  setup.close()
+
+
+def SwitchToNewTemporaryDirectory():
+  tmp_dir = tempfile.mkdtemp('-tzdata')
+  os.chdir(tmp_dir)
+  print 'Created temporary directory "%s"...' % tmp_dir
+
+
+def FtpRetrieveFile(ftp, filename):
+  ftp.retrbinary('RETR %s' % filename, open(filename, 'wb').write)
+
+
+def FtpRetrieveFileAndSignature(ftp, data_filename):
+  """Downloads and repackages the given data from the given FTP server."""
+  print 'Downloading data...'
+  FtpRetrieveFile(ftp, data_filename)
+
+  print 'Downloading signature...'
+  signature_filename = '%s.asc' % data_filename
+  FtpRetrieveFile(ftp, signature_filename)
+
+
+def HttpRetrieveFile(http, path, output_filename):
+  http.request("GET", path)
+  f = open(output_filename, 'wb')
+  f.write(http.getresponse().read())
+  f.close()
+
+
+def HttpRetrieveFileAndSignature(http, data_filename):
+  """Downloads and repackages the given data from the given HTTP server."""
+  path = "/time-zones/repository/releases/%s" % data_filename
+
+  print 'Downloading data...'
+  HttpRetrieveFile(http, path, data_filename)
+
+  print 'Downloading signature...'
+  signature_filename = '%s.asc' % data_filename
+  HttpRetrievefile(http, "%s.asc" % path, signature_filename)
+
+
+def BuildIcuToolsAndData(data_filename):
+  # Keep track of the original cwd so we can go back to it at the end.
+  original_working_dir = os.getcwd()
+
+  # Create a directory to run 'make' from.
+  icu_working_dir = '%s/icu' % original_working_dir
+  os.mkdir(icu_working_dir)
+  os.chdir(icu_working_dir)
+
+  # Build the ICU tools.
+  print 'Configuring ICU tools...'
+  subprocess.check_call(['%s/runConfigureICU' % icu4c_dir, 'Linux'])
+
+  # Run the ICU tools.
+  os.chdir('tools/tzcode')
+
+  # The tz2icu tool only picks up icuregions and icuzones in they are in the CWD
+  for icu_data_file in [ 'icuregions', 'icuzones']:
+    icu_data_file_source = '%s/tools/tzcode/%s' % (icu4c_dir, icu_data_file)
+    icu_data_file_symlink = './%s' % icu_data_file
+    os.symlink(icu_data_file_source, icu_data_file_symlink)
+
+  shutil.copyfile('%s/%s' % (original_working_dir, data_filename), data_filename)
+  print 'Making ICU data...'
+  # The Makefile assumes the existence of the bin directory.
+  os.mkdir('%s/bin' % icu_working_dir)
+  subprocess.check_call(['make'])
+
+  # Copy the source file to its ultimate destination.
+  icu_txt_data_dir = '%s/data/misc' % icu4c_dir
+  print 'Copying zoneinfo64.txt to %s ...' % icu_txt_data_dir
+  shutil.copy('zoneinfo64.txt', icu_txt_data_dir)
+
+  # Regenerate the .dat file.
+  os.chdir(icu_working_dir)
+  subprocess.check_call(['make', 'INCLUDE_UNI_CORE_DATA=1', '-j32'])
+
+  # Copy the .dat file to its ultimate destination.
+  icu_dat_data_dir = '%s/stubdata' % icu4c_dir
+  datfiles = glob.glob('data/out/tmp/icudt??l.dat')
+  if len(datfiles) != 1:
+    print 'ERROR: Unexpectedly found %d .dat files (%s). Halting.' % (len(datfiles), datfiles)
+    sys.exit(1)
+  datfile = datfiles[0]
+  print 'Copying %s to %s ...' % (datfile, icu_dat_data_dir)
+  shutil.copy(datfile, icu_dat_data_dir)
+
+  # Generate the ICU4J .jar files
+  os.chdir('%s/data' % icu_working_dir)
+  subprocess.check_call(['make', 'icu4j-data'])
+
+  # Copy the ICU4J .jar files to their ultimate destination.
+  icu_jar_data_dir = '%s/main/shared/data' % icu4j_dir
+  jarfiles = glob.glob('out/icu4j/*.jar')
+  if len(jarfiles) != 2:
+    print 'ERROR: Unexpectedly found %d .jar files (%s). Halting.' % (len(jarfiles), jarfiles)
+    sys.exit(1)
+  for jarfile in jarfiles:
+    print 'Copying %s to %s ...' % (jarfile, icu_jar_data_dir)
+    shutil.copy(jarfile, icu_jar_data_dir)
+
+  # Switch back to the original working cwd.
+  os.chdir(original_working_dir)
+
+
+def CheckSignature(data_filename):
+  signature_filename = '%s.asc' % data_filename
+  print 'Verifying signature...'
+  # If this fails for you, you probably need to import Paul Eggert's public key:
+  # gpg --recv-keys ED97E90E62AA7E34
+  subprocess.check_call(['gpg', '--trusted-key=ED97E90E62AA7E34', '--verify',
+                         signature_filename, data_filename])
+
+
+def BuildBionicToolsAndData(data_filename):
+  new_version = re.search('(tzdata.+)\\.tar\\.gz', data_filename).group(1)
+
+  print 'Extracting...'
+  os.mkdir('extracted')
+  tar = tarfile.open(data_filename, 'r')
+  tar.extractall('extracted')
+
+  print 'Calling zic(1)...'
+  os.mkdir('data')
+  zic_inputs = [ 'extracted/%s' % x for x in regions ]
+  zic_cmd = ['zic', '-d', 'data' ]
+  zic_cmd.extend(zic_inputs)
+  subprocess.check_call(zic_cmd)
+
+  WriteSetupFile()
+
+  print 'Calling ZoneCompactor to update bionic to %s...' % new_version
+  subprocess.check_call(['javac', '-d', '.',
+                         '%s/ZoneCompactor.java' % tools_dir])
+  subprocess.check_call(['java', 'ZoneCompactor',
+                         'setup', 'data', 'extracted/zone.tab',
+                         bionic_libc_zoneinfo_dir, new_version])
+
+
+# Run with no arguments from any directory, with no special setup required.
+# See http://www.iana.org/time-zones/ for more about the source of this data.
+def main():
+  print 'Looking for new tzdata...'
+
+  tzdata_filenames = []
+
+  # The FTP server lets you download intermediate releases, and also lets you
+  # download the signatures for verification, so it's your best choice.
+  use_ftp = True
+
+  if use_ftp:
+    ftp = ftplib.FTP('ftp.iana.org')
+    ftp.login()
+    ftp.cwd('tz/releases')
+    for filename in ftp.nlst():
+      if filename.startswith('tzdata20') and filename.endswith('.tar.gz'):
+        tzdata_filenames.append(filename)
+    tzdata_filenames.sort()
+  else:
+    http = httplib.HTTPConnection('www.iana.org')
+    http.request("GET", "/time-zones")
+    index_lines = http.getresponse().read().split('\n')
+    for line in index_lines:
+      m = re.compile('.*href="/time-zones/repository/releases/(tzdata20\d\d\c\.tar\.gz)".*').match(line)
+      if m:
+        tzdata_filenames.append(m.group(1))
+
+  # If you're several releases behind, we'll walk you through the upgrades
+  # one by one.
+  current_version = GetCurrentTzDataVersion()
+  current_filename = '%s.tar.gz' % current_version
+  for filename in tzdata_filenames:
+    if filename > current_filename:
+      print 'Found new tzdata: %s' % filename
+      SwitchToNewTemporaryDirectory()
+      if use_ftp:
+        FtpRetrieveFileAndSignature(ftp, filename)
+      else:
+        HttpRetrieveFileAndSignature(http, filename)
+
+      CheckSignature(filename)
+      BuildIcuToolsAndData(filename)
+      BuildBionicToolsAndData(filename)
+      print 'Look in %s and %s for new data files' % (bionic_dir, icu_dir)
+      sys.exit(0)
+
+  print 'You already have the latest tzdata (%s)!' % current_version
+  sys.exit(0)
+
+
+if __name__ == '__main__':
+  main()