WIP: Parallel builds

These changes run the export_libs, resources, and rom phases in parallel, producing a 2.4 times speed-up on a quad core system. They currently only work on the Linux Port of RISC OS, and not in the desktop which will result in sharing an AMB.

It relies on an external routine on WriteV to group output by component to make it readable.

My first attempt by just modifying sb_system as suggested by comment was unreliable in various ways.

Todo:

• Sort out the problem of the SharedCLibrary headers being export during the export_libs phase and therefore potentially to late for building C libraries. I have created a SharedRISC_OSLib_Headers pseudo-component to output the headers at the end of export_hdrs phase, but there may be a better solution.
• Support plain Unix builds, mostly fixing sb_parallel_lock_csd.
• Use a constant for IXSupport_LinuxSyscall, or perhaps the HAL interface directly.
• Use the Linux system calls header, rather than it's private implementation of the Linux system call veneers.
• Parallelise the remaining phases.

c/build

  *
  */
 
-/* ANSI Includes */
+/* ISO Includes */
 #include <stdio.h>
 #include <stdarg.h>
+#include <stdbool.h>
 #include <stdlib.h>
 #include <string.h>
 #include <ctype.h>
   result = sb_chdir(buffer);
   if (result) {
     error(0, "cannot locate '%s'", buffer);
+    return 0;
   }
+  sb_parallel_lock_csd();
   return !result;
 }
 
         case HAL:
         case DATA:
         case EXP:
-          if (build_select_module_directory(component, module)) {
+          PARALLEL(export_libs) if (build_select_module_directory(component, module)) {
             if (amu_machinef("-k export PHASE=%s COMPONENT=%s %s%s %s",
               phase,
               component->name,
       case HAL:
       case DATA:
       case RES:
-        if (build_select_module_directory(component, module))
+        PARALLEL(true) if (build_select_module_directory(component, module))
         {
           /* We have been given a list of territories - iterate down them. */
           unsigned int loop;
         case KERNEL:
         case HAL:
         case DATA:
-        if (build_select_module_directory(component, module)) {
+        PARALLEL(true) if (build_select_module_directory(component, module)) {
   	  /*
   	   * do we want to force a clean build due to changes
   	   */
   	  if (component->force_clean) {
             sb_printf("Forcing a clean build due to switch/option changes...\n");
-  	    if (amu_machinef("-k clean COMPONENT=%s %s%s %s",
+	    if (amu_machinef("-k clean COMPONENT=%s %s%s %s",
   		component->name,
   		module->filename?"TARGET=":"",
   		module->filename?module->filename:"",
   	    if (component->options) {
   	      fprintf(fh_shadow, "-options %s", component->options);
   	    }
-  	    fprintf(fh_shadow, "\n");
-  	  }
+	    fprintf(fh_shadow, "\n");
+            fflush(fh_shadow);
+	  }
 
   	  if (amu_machinef("-k rom COMPONENT=%s %s%s %s",
   		component->name,

c/riscos

  * globals
  */
 static int spooling = FALSE;
+static int children;
+static int max_children;
 
 
 #ifdef __riscos
 static FILE *logfile;
 #endif
 
+#ifdef __riscos
+/*
+ * Linux system calls on RISC OS
+ */
+static int IXSupport_LinuxSyscall;
+
+static inline int fork(void) {
+  return _swi(IXSupport_LinuxSyscall, _RETURN(0) | _IN(7), 2);
+}
+
+static inline int wait(int *status) {
+  return _swi(IXSupport_LinuxSyscall, _RETURN(0) | _INR(0, 3) | _IN(7), -1, status, 0, 0, 114);
+}
+
+static inline void _exit(int code) {
+  _swi(IXSupport_LinuxSyscall, _RETURN(0) | _IN(0) | _IN(7), code, 1);
+}
+
+static inline int flock(int fd, int operation) {
+  return _swi(IXSupport_LinuxSyscall, _RETURN(0) | _INR(0, 1) | _IN(7), fd, operation, 143);
+}
+
+#define LOCK_EX 2
+
+#endif
+
+
+
 /*
  * change working directory
  */
 #endif
   sb_preload_env();
   sb_init_territories();
+
+  const char *c = sb_getenv("ROOL$MaxJobs");
+  if (c) max_children = atoi(c);
+  if (max_children) {
+#ifdef __riscos
+    _swi(OS_SWINumberFromString, _IN(1) | _OUT(0), "IXSupport_LinuxSyscall", &IXSupport_LinuxSyscall);
+#endif
+  }
 }
 
 #ifndef __riscos
 #endif
 }
 
-void sb_system_sync(void)
-{
-#ifndef __riscos
-  /* Nothing to do.  If we allow sb_system to fork and leave the task running in
-   * the background, we need to wait() for all the child processes to terminate
-   * here before returning.
-   */
-#endif
-}
-
 /* Function to duplicate a string (allocate memory and copy bytewise)
  *
  * Originally: sbrodie
 
   return result;
 }
+
+static void sb_parallel_flush(void) {
+  fflush(stdout);
+#ifdef __riscos
+  // Enable or flush block buffering by BufferWriteC
+  _kernel_oswrch(7);
+#endif
+}
+
+static void sb_parallel_panic(void) {
+  sb_parallel_flush();
+  _exit(1);
+}
+
+int sb_parallel_start(bool ok) {
+
+  // Exit early if parallel build not enabled
+  if (!max_children) return 1;
+
+  // Wait until there are not too many children
+  while(children >= (ok ? max_children : 1)) {
+    int status = 0;
+    if (wait(&status) < 0) error(0, "Wait() returned error");
+    --children;
+    if (status) error(0, "Child returned non-zero return code");
+  }
+
+  // Exit if parallel build not okay for this part (or just waiting)
+  if (!ok) return 1;
+
+  int pid = fork();
+  if (pid < 0) {
+    // Fork failed, build in single process.
+    return 1;
+  } else if (pid == 0) {
+    // In child process
+    children = -1;
+    error_database = NULL;
+    sb_parallel_flush();
+    atexit(sb_parallel_panic);
+    return 2;
+  } else {
+    // In parent process
+    ++children;
+    return 0;
+  }
+}
+
+void sb_parallel_end(int flag) {
+  if (flag > 1) {
+    sb_parallel_flush();
+    _exit(error_database != NULL);
+  }
+}
+
+void sb_parallel_lock_csd(void) {
+  if (children == -1) {
+    unsigned file = _swi(OS_Find, _INR(0, 1) | _RETURN(0), 0x43, "@");
+    file = _swi(OS_FSControl, _INR(0, 1) | _RETURN(1), 21, file);
+    if (file > 0x8000) file = *(unsigned *)file;
+    flock(file, LOCK_EX);
+  }
+}

c/srcbuild

     }
     sb_printf("Starting phase %s ...\n", action->name);
     (*(action->call)) ();
+    sb_system_sync();
     banner('=');
   }
 

h/riscos

 
 #ifndef srcbuild_riscos_h
 #define srcbuild_riscos_h
+#include <stdbool.h>
 
 extern int  sb_chdir(const char *dir);
 extern int  sb_mkdir(const char *dir, const int mode);
 extern char **sb_putenv(void);
 extern char *sb_strdup(const char *str);
 
+extern int  sb_parallel_start(bool ok);
+extern void sb_parallel_end(int flag);
+extern void sb_parallel_lock_csd(void);
+
 extern int  sb_system(char *command);
-extern void sb_system_sync(void);
+static inline void sb_system_sync(void) { sb_parallel_start(0); }
 
 extern void sb_init(void);
 
 #endif
 
 
+#define PARALLEL(OK) for(int sb_parallel_flag = sb_parallel_start(OK); sb_parallel_flag; sb_parallel_end(sb_parallel_flag), sb_parallel_flag = 0)
+
+
 #endif

h/srcbuild

 extern HashId join_entries[];
 extern int join_db_hwm;
 extern podule_info_t podule_info;
+extern error_entry_t *error_database;
 
 #endif