Speed-up CircleCI builds for M68K and PowerPC. (#94)

* Split out download action.
* Use CircleCI cache to store downloads.
* Run some make commands in parallel (-j option).

.circleci/DATESTAMP

@@ -0,0 +1 @@
+20171112-2

.circleci/config.yml

@@ -10,9 +10,15 @@ jobs:
     <<: *defaults
     steps:
       - checkout
+      - restore_cache:
+          keys:
+            - m68k-cache-{{ .Branch }}-{{ checksum ".circleci/DATESTAMP" }}
+      - run: './toolchain-m68k --quiet download'
+      - save_cache:
+          key: m68k-cache-{{ .Branch }}-{{ checksum ".circleci/DATESTAMP" }}
+          paths:
+            - .build-m68k
       - run: './toolchain-m68k --quiet build'
-      - store_artifacts:
-          path: m68k-amigaos
       - save_cache:
           key: m68k-amigaos-{{ .Branch }}-{{ .Revision }}
           paths: m68k-amigaos
@@ -29,9 +35,15 @@ jobs:
     <<: *defaults
     steps:
       - checkout
+      - restore_cache:
+          keys:
+            - ppc-cache-{{ .Branch }}-{{ checksum ".circleci/DATESTAMP" }}
+      - run: './toolchain-ppc --quiet download'
+      - save_cache:
+          key: ppc-cache-{{ .Branch }}-{{ checksum ".circleci/DATESTAMP" }}
+          paths:
+            - .build-ppc
       - run: './toolchain-ppc --quiet build'
-      - store_artifacts:
-          path: ppc-amigaos
       - save_cache:
           key: ppc-amigaos-{{ .Branch }}-{{ .Revision }}
           paths: ppc-amigaos

common.py

@@ -10,6 +10,7 @@ import contextlib
 from distutils import spawn, sysconfig
 import fileinput
 import os
+from multiprocessing import cpu_count
 import shutil
 import site
 import subprocess
@@ -417,7 +418,7 @@ def fetch(name, url):
     else:
       info('File "%s" already downloaded.', name)
   elif url.startswith('svn'):
-    execute('svn', 'export', '--quiet', url, name)
+    execute('svn', 'export', url, name)
   elif url.startswith('git'):
     if not path.exists(name):
       execute('git', 'clone', url, name)
@@ -489,7 +490,7 @@ def configure(name, *confopts, **kwargs):
 
 
 @recipe('make', 2)
-def make(name, target=None, makefile=None, **makevars):
+def make(name, target=None, makefile=None, parallel=False, **makevars):
   info('running make "%s"', target)
 
   with cwd(path.join('{build}', name)):
@@ -498,6 +499,8 @@ def make(name, target=None, makefile=None, **makevars):
       args = [target] + args
     if makefile is not None:
       args = ['-f', makefile] + args
+    if parallel:
+      args = ['-j%d' % cpu_count()] + args
     execute('make', *args)
 
 

toolchain-m68k

@@ -241,6 +241,19 @@ def touch_genfiles(dst):
       touch(h_file)
 
 
+def download():
+  with cwd('{archives}'):
+    for url in URLS:
+      if type(url) == tuple:
+        url, name = url[0], url[1]
+      else:
+        name = path.basename(url)
+      fetch(name, url)
+
+  execute('git', 'submodule', 'init');
+  execute('git', 'submodule', 'update');
+
+
 def build():
   for var in environ.keys():
     if var not in ['_', 'LOGNAME', 'HOME', 'SHELL', 'TMPDIR', 'PWD']:
@@ -310,28 +323,19 @@ def build():
   unpack('python-lha', work_dir='{build}')
   pysetup('python-lha')
 
-  with cwd('{archives}'):
-    for url in URLS:
-      if type(url) == tuple:
-        url, name = url[0], url[1]
-      else:
-        name = path.basename(url)
-      fetch(name, url)
-
-  execute('git', 'submodule', 'init');
-  execute('git', 'submodule', 'update');
+  download()
 
   unpack('{automake}')
 
   unpack('{m4}')
   configure('{m4}', '--prefix={host}')
-  make('{m4}')
+  make('{m4}', parallel=True)
   make('{m4}', 'install')
 
   unpack('{gawk}')
   update_autotools('{sources}/{gawk}')
   configure('{gawk}', '--prefix={host}')
-  make('{gawk}')
+  make('{gawk}', parallel=True)
   make('{gawk}', 'install')
 
   unpack('{flex}')
@@ -342,19 +346,19 @@ def build():
   unpack('{bison}')
   update_autotools('{sources}/{bison}/config')
   configure('{bison}', '--prefix={host}')
-  make('{bison}')
+  make('{bison}', parallel=True)
   make('{bison}', 'install')
 
   unpack('{texinfo}')
   update_autotools('{sources}/{texinfo}/build-aux')
   configure('{texinfo}', '--prefix={host}')
-  make('{texinfo}')
+  make('{texinfo}', parallel=True)
   make('{texinfo}', 'install')
 
   unpack('{autoconf}')
   update_autotools('{sources}/{autoconf}')
   configure('{autoconf}', '--prefix={host}')
-  make('{autoconf}')
+  make('{autoconf}', parallel=True)
   make('{autoconf}', 'install')
 
   prepare_target()
@@ -418,7 +422,7 @@ def build():
               '--target=m68k-amigaos',
               from_dir='{submodules}/{binutils}')
     touch_genfiles('{submodules}/{binutils}')
-    make('{binutils}')
+    make('{binutils}', parallel=True)
     make('{binutils}', 'install-binutils')
     make('{binutils}', 'install-gas')
     make('{binutils}', 'install-ld')
@@ -442,6 +446,7 @@ def build():
     touch_genfiles('{submodules}/{gcc}')
     touch('{submodules}/{gcc}/gcc/c-parse.gperf')
     touch('{submodules}/{gcc}/gcc/configure')
+    # parallel build fails for all-gcc
     make('{gcc}', 'all-gcc',
          MAKEINFO='makeinfo', CFLAGS_FOR_TARGET='-noixemul')
     make('{gcc}', 'install-gcc',
@@ -493,11 +498,11 @@ def build():
   make('{libdebug}', 'install')
 
   unpack('{clib2}', work_dir='{build}', top_dir='library')
-  make('{clib2}', makefile='GNUmakefile.68k')
+  make('{clib2}', makefile='GNUmakefile.68k', parallel=True)
   install_clib2()
 
   with env(CC=CC, CXX=CXX, CFLAGS=FLAGS, CXXFLAGS=FLAGS):
-    make('{gcc}', 'all-target',
+    make('{gcc}', 'all-target', parallel=True,
          MAKEINFO='makeinfo', CFLAGS_FOR_TARGET='-noixemul')
     make('{gcc}', 'install-target',
          MAKEINFO='makeinfo', CFLAGS_FOR_TARGET='-noixemul')
@@ -719,7 +724,7 @@ if __name__ == "__main__":
   parser = argparse.ArgumentParser(description='Build cross toolchain.')
   parser.add_argument('action',
                       choices=['build', 'list-sdk', 'install-sdk', 'clean',
-                               'test'],
+                               'test', 'download'],
                       default='build', help='perform action')
   parser.add_argument('args', metavar='ARGS', type=str, nargs='*',
                       help='action arguments')

toolchain-ppc

@@ -48,6 +48,16 @@ def prepare_sdk():
   unpack('SDK/newlib', top_dir='newlib', dst_dir=path.join(target, 'newlib'))
 
 
+def download():
+  with cwd('{archives}'):
+    for url in URLS:
+      if type(url) == tuple:
+        url, name = url[0], url[1]
+      else:
+        name = path.basename(url)
+      fetch(name, url)
+
+
 def build():
   for var in environ.keys():
     if var not in ['_', 'LOGNAME', 'HOME', 'SHELL', 'TMPDIR', 'PWD']:
@@ -107,21 +117,14 @@ def build():
   unpack('python-lha', work_dir='{build}')
   pysetup('python-lha')
 
-  with cwd('{archives}'):
-    for url in URLS:
-      if type(url) == tuple:
-        url, name = url[0], url[1]
-      else:
-        name = path.basename(url)
-      fetch(name, url)
+  download()
 
   unpack('{automake}')
 
-
   unpack('{texinfo}')
   update_autotools('{sources}/{texinfo}/build-aux')
   configure('{texinfo}', '--prefix={host}')
-  make('{texinfo}')
+  make('{texinfo}', parallel=True)
   make('{texinfo}', 'install')
 
   unpack('{gmp}')
@@ -129,7 +132,7 @@ def build():
   configure('{gmp}',
             '--disable-shared',
             '--prefix={host}')
-  make('{gmp}')
+  make('{gmp}', parallel=True)
   make('{gmp}', 'install')
 
   unpack('{mpfr}')
@@ -138,7 +141,7 @@ def build():
             '--disable-shared',
             '--prefix={host}',
             '--with-gmp={host}')
-  make('{mpfr}')
+  make('{mpfr}', parallel=True)
   make('{mpfr}', 'install')
 
   unpack('{mpc}')
@@ -148,7 +151,7 @@ def build():
             '--prefix={host}',
             '--with-gmp={host}',
             '--with-mpfr={host}')
-  make('{mpc}')
+  make('{mpc}', parallel=True)
   make('{mpc}', 'install')
 
   unpack('{isl}')
@@ -157,7 +160,7 @@ def build():
             '--disable-shared',
             '--prefix={host}',
             '--with-gmp-prefix={host}')
-  make('{isl}')
+  make('{isl}', parallel=True)
   make('{isl}', 'install')
 
   unpack('{cloog}')
@@ -168,7 +171,7 @@ def build():
             '--with-isl=system',
             '--with-gmp-prefix={host}',
             '--with-isl-prefix={host}')
-  make('{cloog}')
+  make('{cloog}', parallel=True)
   make('{cloog}', 'install')
 
   with env(CFLAGS='-Wno-error'):
@@ -177,7 +180,7 @@ def build():
               '--prefix={prefix}',
               '--target={target}',
               from_dir='{archives}/{binutils}')
-    make('{binutils}')
+    make('{binutils}', parallel=True)
     make('{binutils}', 'install')
 
   prepare_sdk()
@@ -231,7 +234,7 @@ if __name__ == "__main__":
 
   parser = argparse.ArgumentParser(description='Build cross toolchain.')
   parser.add_argument('action',
-                      choices=['build', 'clean'],
+                      choices=['build', 'clean', 'download'],
                       default='build', help='perform action')
   parser.add_argument('args', metavar='ARGS', type=str, nargs='*',
                       help='action arguments')