OS_ChangeDynamicArea performance optimisations

Detail:
  s/ChangeDyn:
    - Apply various optimisations to OS_ChangeDynamicArea to reduce the execution time when performing large grows/shrinks.
    - Optimisations can be toggled on/off with FastCDA_* flags for debugging.
    - On a 1GHz 512MB BB-xM, the initial *FreePool call now takes 0.15s instead of 13.46s. On a 512MB Iyonix the time has dropped from 1.18s to 0.23s.
    - Growing screen memory (on BB-xM) has also seen significant gains - between 2x and 4x speedup, depending on what state the source pages are in.
    - Added/updated documentation for a few functions and made more use of ROUTs for safety
  s/ARM600, s/VMSAv6:
    - Update BangCamUpdate, etc. to add support for the PageFlags_Unsafe flag that OS_ChangeDynamicArea uses to bypass cache/TLB maintenance in some situations
    - Avoid BangCamUpdate calling BangL2PT to map out the page if the page isn't mapped in (avoids unnecessary cache/TLB flush)
  s/ArthurSWIs:
    - Add extra ASSERT for safety
  s/AMBcontrol/memory
    - Fix incorrect assumption that the usable size of a heap block is always 8 less than the value stored in the header. Even with the old 8 byte aligned allocations the usable size will always be 4 bytes less than the value in the header. This code would have resulted in some slight memory wasteage, as AMBcontrol will have always tried growing the block four bytes bigger than needed.
Admin:
  Tested on Iyonix & BB-xM


Version 5.35, 4.79.2.146. Tagged as 'Kernel-5_35-4_79_2_146'

VersionASM

@@ -13,11 +13,11 @@
                         GBLS    Module_ComponentPath
 Module_MajorVersion     SETS    "5.35"
 Module_Version          SETA    535
-Module_MinorVersion     SETS    "4.79.2.145"
-Module_Date             SETS    "08 Apr 2012"
-Module_ApplicationDate  SETS    "08-Apr-12"
+Module_MinorVersion     SETS    "4.79.2.146"
+Module_Date             SETS    "15 Apr 2012"
+Module_ApplicationDate  SETS    "15-Apr-12"
 Module_ComponentName    SETS    "Kernel"
 Module_ComponentPath    SETS    "castle/RiscOS/Sources/Kernel"
-Module_FullVersion      SETS    "5.35 (4.79.2.145)"
-Module_HelpVersion      SETS    "5.35 (08 Apr 2012) 4.79.2.145"
+Module_FullVersion      SETS    "5.35 (4.79.2.146)"
+Module_HelpVersion      SETS    "5.35 (15 Apr 2012) 4.79.2.146"
                         END

VersionNum

@@ -5,19 +5,19 @@
  *
  */
 #define Module_MajorVersion_CMHG        5.35
-#define Module_MinorVersion_CMHG        4.79.2.145
-#define Module_Date_CMHG                08 Apr 2012
+#define Module_MinorVersion_CMHG        4.79.2.146
+#define Module_Date_CMHG                15 Apr 2012
 
 #define Module_MajorVersion             "5.35"
 #define Module_Version                  535
-#define Module_MinorVersion             "4.79.2.145"
-#define Module_Date                     "08 Apr 2012"
+#define Module_MinorVersion             "4.79.2.146"
+#define Module_Date                     "15 Apr 2012"
 
-#define Module_ApplicationDate          "08-Apr-12"
+#define Module_ApplicationDate          "15-Apr-12"
 
 #define Module_ComponentName            "Kernel"
 #define Module_ComponentPath            "castle/RiscOS/Sources/Kernel"
 
-#define Module_FullVersion              "5.35 (4.79.2.145)"
-#define Module_HelpVersion              "5.35 (08 Apr 2012) 4.79.2.145"
+#define Module_FullVersion              "5.35 (4.79.2.146)"
+#define Module_HelpVersion              "5.35 (15 Apr 2012) 4.79.2.146"
 #define Module_LibraryVersionInfo       "5:35"

s/AMBControl/Memory

@@ -50,7 +50,7 @@ AMB_BlockResize ROUT
         ADD     r3,r3,#AMBblockQ - 1
         BIC     r3,r3,#AMBblockQ - 1
         LDR     r1,[r2,#-4]                  ;pick up OS_Heap's size word (naughty!)
-        SUB     r1,r1,#8                     ;heap size will be 8 more than quantised size
+        SUB     r1,r1,#4                     ;heap size will be (at least) 4 more than quantised size
         SUBS    r3,r3,r1                     ;required size change
         MOVNE   r0, #HeapReason_ExtendBlock
         BLNE    DoSysHeapOpWithExtension

s/ARM600

@@ -218,7 +218,8 @@ BangCamUpdate ROUT
         LDR     r1, [r1, #CamEntriesPointer]
         ADD     r1, r1, r2, LSL #3              ; point at cam entry (logaddr, PPL)
         LDMIA   r1, {r0, r6}                    ; r0 = current logaddress, r6 = current PPL
-        STMIA   r1, {r3, r11}                   ; store new address, PPL
+        BIC     r4, r11, #PageFlags_Unsafe
+        STMIA   r1, {r3, r4}                    ; store new address, PPL
         Push    "r0, r6"                        ; save old logical address, PPL
         LDR     r1, =ZeroPage+PhysRamTable      ; go through phys RAM table
         MOV     r6, r2                          ; make copy of r2 (since that must be preserved)
@@ -248,11 +249,15 @@ BangCamUpdate ROUT
         TEQ     r4, r0, LSR #12                 ; if equal to physical address of page being moved
         BNE     %FT20                           ; if not there, then just put in new page
 
+        AND     r4, r11, #PageFlags_Unsafe
         Push    "r0, r3, r11, r14"              ; save phys.addr, new log.addr, new PPL, lr
         ADD     r3, sp, #4*4
         LDMIA   r3, {r3, r11}                   ; reload old logical address, old PPL
+        LDR     r0, =DuffEntry                  ; Nothing to do if wasn't mapped in
+        ORR     r11, r11, r4
+        TEQ     r3, r0
         MOV     r0, #0                          ; cause translation fault
-        BL      BangL2PT                        ; map page out
+        BLNE    BangL2PT                        ; map page out
         Pull    "r0, r3, r11, r14"
 20
         ADD     sp, sp, #8                      ; junk old logical address, PPL
@@ -340,6 +345,9 @@ BangL2PT                                        ; internal entry point used only
         Push    "lr"
         MOV     r6, r0
 
+        TST     r11, #PageFlags_Unsafe
+        BNE     %FT30
+
         TST     r11, #DynAreaFlags_DoublyMapped
         BNE     BangL2PT_sledgehammer           ;if doubly mapped, don't try to be clever
 

s/ArthurSWIs

@@ -1083,6 +1083,7 @@ fakeservicecall
        MOV      r0, r0
        LDR      r10, =ZeroPage
        LDRB     r9, [r10, #FIQclaim_interlock]
+       ASSERT   (ZeroPage :AND: 255) = 0
        STRB     r10, [r10, #FIQclaim_interlock]
 
    [ FIQDebug

s/ChangeDyn

@@ -14,6 +14,89 @@
 ;
         TTL   => ChangeDyn
 
+        ; OS_ChangeDynamicArea optimisations:
+
+        GBLL  FastCDA_UpFront ; Do all cache/TLB maintenance upfront instead of on a per-page basis
+FastCDA_UpFront SETL {TRUE}
+
+        GBLL  FastCDA_FIQs ; Don't thrash ClaimFIQ/ReleaseFIQ in DoTheGrowPagesSpecified
+FastCDA_FIQs SETL {TRUE}
+
+        GBLL  FastCDA_NoPhysical ; Don't use RISCOS_AccessPhysicalAddress/RISCOS_ReleasePhysicalAddress in DoTheGrowPagesSpecified. Instead, map pages straight to destination address.
+FastCDA_NoPhysical SETL {TRUE}
+
+        GBLL  FastCDA_CorruptFreePool ; Contents of free pool doesn't need preserving in DoTheGrowPagesSpecified
+FastCDA_CorruptFreePool SETL {TRUE}
+
+        GBLL  FastCDA_Unnecessary ; Avoid unnecessary MMU_ChangingEntry calls in DoTheGrowPagesSpecified
+FastCDA_Unnecessary SETL {TRUE}
+
+        ; DoTheGrowPagesSpecified profiling code
+        ; Written to use Cortex-A8 cycle count performance counter - will need modifying for other CPUs!
+
+        GBLL  FastCDA_Prof
+FastCDA_Prof SETL {FALSE}
+
+      [ FastCDA_Prof
+        ; Squeeze profiling workspace into "free space after envstring"
+                                 ^ ExtendedROMFooter+4
+        ! 0, "FastCDA_Prof workspace at ":CC::STR:@
+FastCDA_Prof_DoTheGrowInit           # 4
+FastCDA_Prof_MarkRequired            # 4
+FastCDA_Prof_PagesUnsafe             # 4
+FastCDA_Prof_DoublyCheckCacheability # 4
+FastCDA_Prof_DoublyMovePages         # 4
+FastCDA_Prof_FindSpare               # 4
+FastCDA_Prof_ClaimFIQ                # 4
+FastCDA_Prof_AccessPhysical          # 4
+FastCDA_Prof_CopyPage                # 4
+FastCDA_Prof_ReleasePhysical         # 4
+FastCDA_Prof_MoveReplacement         # 4
+FastCDA_Prof_MoveNeeded              # 4
+FastCDA_Prof_ReleaseFIQ              # 4
+FastCDA_Prof_PagesSafe               # 4
+FastCDA_Prof_CallPreGrow             # 4
+FastCDA_Prof_CallPostGrow            # 4
+FastCDA_Prof_MMUChanging             # 4
+FastCDA_Prof_MMUChangingUncached     # 4
+FastCDA_Prof_ChangingEntry           # 4
+        ASSERT @ <= &500
+      ]
+
+        MACRO
+        FastCDA_ProfInit $temp
+      [ FastCDA_Prof
+        MVN     $temp,#0
+        MCR     p15,0,$temp,c9,c12,2
+        MOV     $temp,#1<<31
+        MCR     p15,0,$temp,c9,c12,1
+        MOV     $temp,#7
+        MCR     p15,0,$temp,c9,c12,0
+      ]
+        MEND
+
+        MACRO
+        FastCDA_ProfStart $var,$temp,$temp2,$temp3,$cc
+      [ FastCDA_Prof
+        LDR$cc  $temp,=ZeroPage+FastCDA_Prof_$var
+        LDR$cc  $temp2,[$temp]
+        MRC$cc  p15,0,$temp3,c9,c13,0
+        SUB$cc  $temp2,$temp2,$temp3
+        STR$cc  $temp2,[$temp]
+      ]
+        MEND
+
+        MACRO
+        FastCDA_ProfEnd $var,$temp,$temp2,$temp3,$cc
+      [ FastCDA_Prof
+        MRC$cc  p15,0,$temp3,c9,c13,0
+        LDR$cc  $temp,=ZeroPage+FastCDA_Prof_$var
+        LDR$cc  $temp2,[$temp]
+        ADD$cc  $temp2,$temp2,$temp3
+        STR$cc  $temp2,[$temp]
+      ]
+        MEND
+
 ;******************************************************************************
 ; ChangeDynamic SWI
 ; In  : R0 =  0 => System Heap,
@@ -279,7 +362,7 @@ GetPageFlagsForR0IntoR6 Entry "R0-R2, R4-R5, R7"
 ; MoveCAMatR0toR3
 ; in:   r0 = old logaddr
 ;       r3 = new logaddr
-;       r9 = MEMC CR
+;       r9 = offset from 1st to 2nd copy of doubly mapped area (either source or dest, but not both)
 ;       r11 = page protection level
 ;
 ; out:  r2 = physical page number of page moved, unless there was a serious error
@@ -364,11 +447,36 @@ CamMapBroke
         &       0
         =       "!!!! CAM Map Corrupt !!!!", 0
         ALIGN
+; Call_CAM_Mapping
+; in:   r2 = physical page number
+;       r3 = logical address (2nd copy of doubly mapped area)
+;       r9 = offset from 1st to 2nd copy of doubly mapped area (either source or dest, but not both)
+;       r11 = PPL + CB bits
 Call_CAM_Mapping
         Push    "r0, r1, r4, r6, lr"
         BL      BangCamUpdate
         Pull    "r0, r1, r4, r6, pc"
 
+  [ FastCDA_UpFront
+; CheckCacheabilityR0ByMinusR2
+; in:   r0 = end of area
+;       r2 = size of area (must be nonzero)
+; out:  r6 has DynAreaFlags_NotCacheable set if entire region noncacheable
+;              Flag clear if at least one page is cacheable
+;       r0 points to start of area
+;
+;       4K page size assumed!
+CheckCacheabilityR0ByMinusR2 ROUT
+        Entry   "r2"
+10
+        SUB     r0, r0, #4096
+        BL      GetPageFlagsForR0IntoR6
+        SUBS    r2, r2, #4096
+        TSTNE   r6, #DynAreaFlags_NotCacheable
+        BNE     %BT10
+        SUB     r0, r0, r2
+        EXIT
+  ]
 
 ; +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 ; In    r0 bits 0..6 = area number
@@ -719,6 +827,7 @@ PageFlags_Unavailable           *  1 :SHL: 20                      ; physical pa
 ; Temporary flags only used by kernel
 ;
 PageFlags_Required              *  1 :SHL: 21                      ; physical page asked for by handler
+PageFlags_Unsafe                *  1 :SHL: 22                      ; skip cache/TLB maintenance in BangCamUpdate. flag not saved to CAM map.
 
 
 DynamicAreaSWI Entry
@@ -3007,6 +3116,8 @@ SysHeapString
 ChangeDynamicSWI ROUT
         Push    "r0, r2-r9, r10, lr"
 
+        FastCDA_ProfInit r3
+
         LDR     r10, =ZeroPage                  ; check we're not in an IRQ
         LDR     lr, [r10, #IRQsema]
         TEQ     lr, #0
@@ -3082,7 +3193,7 @@ daq_cda_od6done
         BEQ     IssueServiceMemoryMoved         ; zero pages! (r0 = area number, r1 = size change (0))
         BPL     AreaGrow
 
-AreaShrink
+AreaShrink ROUT
         RSB     r1, r1, #0                      ; make size change positive
  [ DebugCDA2
         DREG    r0, "Shrinking area ", cc
@@ -3177,10 +3288,35 @@ AreaShrink
         LDR     r3, [r12, #DANode_Size]
         ADD     r1, r1, r3                      ; r1 -> address of 1st extra page
 
+      [ FastCDA_UpFront
+        Push    "r0-r1"
+        BL      CheckCacheabilityR0ByMinusR2
+        LDR     r4, [r11, #DANode_Flags]
+        ADR     lr, %FT19
+        TST     r4, #DynAreaFlags_DoublyMapped
+        LDR     r4, =ZeroPage
+        BNE     %FT18
+        ; Interacting with singly-mapped region - use regular logic
+        TST     r6, #DynAreaFlags_NotCacheable
+        MOV     r1, r2, LSR #12
+        ARMop   MMU_ChangingEntries, EQ, tailcall, r4
+        ARMop   MMU_ChangingUncachedEntries, NE, tailcall, r4
+18
+        ; Interacting with doubly-mapped region - use sledgehammer logic
+        TST     r6, #DynAreaFlags_NotCacheable
+        ARMop   MMU_Changing, EQ, tailcall, r4
+        ARMop   MMU_ChangingUncached, NE, tailcall, r4
+19
+        Pull    "r0-r1"
+      ]
+
         LDR     lr, =DynAreaFlags_AccessMask
         MOV     r4, r2
         LDR     r6, [r12, #DANode_Flags]        ; r6 = dst flags
         AND     r6, r6, lr
+      [ FastCDA_UpFront
+        ORR     r6, r6, #PageFlags_Unsafe
+      ]
 20
         SUB     r0, r0, r5                      ; pre-decrement source pointer
  [ DebugCDA2
@@ -3221,6 +3357,9 @@ AreaShrink
         SUB     r1, r1, r5
  [ 1 = 1
         BL      GetPageFlagsForR0IntoR6
+ ]
+ [ FastCDA_UpFront
+        ORR     r6, r6, #PageFlags_Unsafe
  ]
         BL      MovePageAtR0ToR1WithAccessR6
         SUBS    r4, r4, r5
@@ -3241,7 +3380,7 @@ AreaShrink
         LDR     r0, [r11, #DANode_Number]       ; reload dynamic area number
         B       IssueServiceMemoryMoved
 
-AreaGrow
+AreaGrow ROUT
  [ DebugCDA2
         DREG    r0, "Growing area ", cc
         DREG    r1, " by "
@@ -3540,7 +3679,8 @@ ISMM_BatCloak
 ;       C=0 => failed to move as much as we wanted
 ;       C=1 => succeeded in moving as much as we wanted
 
-TryToShrinkShrinkables Entry "r0,r1,r10"
+TryToShrinkShrinkables ROUT
+        Entry "r0,r1,r10"
         LDR     lr, [r11, #DANode_Number]
         TEQ     lr, #ChangeDyn_FreePool
         EXIT    NE                              ; if src <> free pool, exit with C, V flags intact
@@ -3604,26 +3744,33 @@ TryToShrinkShrinkables Entry "r0,r1,r10"
 
                 ^       0, sp
 NumEntries      #       4                       ; Number of entries to do for this chunk
-DestAddr        #       4                       ; Log addr of 1st page being added to dest
-DestFlags       #       4                       ; Page flags for destination area
 TotalAmount     #       4                       ; Total size of grow for this chunk (ie entry value of r3)
-SavedPSR        #       4                       ; PSR before IRQs disabled
-Offset1To2      #       4                       ; Offset from 1st to 2nd bank
 
 DoTheGrowNotSpecifiedStackSize * :INDEX: @      ; amount of stack needed for 'not specified' version
 
+DestAddr        #       4                       ; Log addr of 1st page being added to dest
+DestFlags       #       4                       ; Page flags for destination area
+SavedPSR        #       4                       ; PSR before IRQs disabled
+Offset1To2      #       4                       ; Offset from 1st to 2nd bank
 PageBlock1      #       PageBlockSize           ; 1st page block, for original page numbers and phys. addrs
 PageBlock2      #       PageBlockSize           ; 2nd page block, for new page numbers and phys. addrs
 
 DoTheGrowStackSize *    :INDEX: @
 
-DoTheGrow Entry "r3,r5,r10-r12", DoTheGrowStackSize
+; Offset1To2 is only used by the first half of the routine. Reuse the space as flags for the second half:
+                    ^   :INDEX: Offset1To2, sp
+NeedToMoveFlag      #   1                       ; Whether we still need to move the current page
+                    #   3                       ; (spare)
+
+DoTheGrow ROUT
+        Entry "r3,r5,r10-r12", DoTheGrowStackSize
 
 ; First fill in the page block with -1 in the physical page number words
 
         STR     r2, NumEntries                  ; save number of entries for use later
         STR     r7, TotalAmount                 ; save amount growing by
 
+        FastCDA_ProfStart DoTheGrowInit, r0, r1, lr
         ADR     r1, PageBlock1                  ; point at 1st page block on stack
         ADD     lr, r2, r2, LSL #1              ; lr = number of words in page block
         ADD     lr, r1, lr, LSL #2              ; lr -> off end of page block
@@ -3633,6 +3780,7 @@ DoTheGrow Entry "r3,r5,r10-r12", DoTheGrowStackSize
         STR     r0, [lr, #PageBlockSize]        ; and put -1 in 2nd page block as well
         TEQ     lr, r1                          ; until the end
         BNE     %BT10
+        FastCDA_ProfEnd DoTheGrowInit, r0, r3, lr
 
 ; Now call the pre-grow handler
 
@@ -3675,6 +3823,23 @@ DoTheGrow Entry "r3,r5,r10-r12", DoTheGrowStackSize
 
         MOVS    r4, r3                          ; amount to do
         BEQ     %FT20                           ; [none, so skip all this]
+
+      [ FastCDA_UpFront
+        ; Perform sledgehammer logic upfront
+        Push    "r0,r2,r6"
+        MOV     r0, r1
+        MOV     r2, r3
+        BL      CheckCacheabilityR0ByMinusR2
+        TST     r6, #DynAreaFlags_NotCacheable
+        LDR     r0, =ZeroPage
+        ADR     lr, %FT14
+        ARMop   MMU_Changing, EQ, tailcall, r0
+        ARMop   MMU_ChangingUncached, NE, tailcall, r0
+14
+        Pull    "r0,r2,r6"
+        ORR     r6, r6, #PageFlags_Unsafe
+      ]
+
         Push    "r0, r1"
         SUB     r0, r1, r3                      ; src starts at start of 1st copy = start of 2nd - old size
         SUB     r1, r0, r2                      ; dst start = src start - amount of room needed
@@ -3690,6 +3855,21 @@ DoTheGrow Entry "r3,r5,r10-r12", DoTheGrowStackSize
         ADD     r9, r3, r2                      ; set up offset from 1st copy to 2nd copy (= new size)
 25
         ADD     r1, r1, r3                      ; r1 -> address of 1st extra page
+   [ FastCDA_UpFront
+        ; Flush src region from cache
+        Push    "r0-r1,r6"
+        BL      CheckCacheabilityR0ByMinusR2
+        TST     r6, #DynAreaFlags_NotCacheable
+        LDR     r4, =ZeroPage
+        MOV     r1, r2, LSR #12
+        ADR     lr, %FT29
+        ARMop   MMU_ChangingEntries, EQ, tailcall, r4
+        ARMop   MMU_ChangingUncachedEntries, NE, tailcall, r4
+29
+        Pull    "r0-r1,r6"
+        ; Now BangCam for all the pages
+        ORR     r6, r6, #PageFlags_Unsafe
+   ]
         MOV     r4, #0                          ; amount done so far
         MOV     r10, r2                         ; move amount to do into r10, as routine returns page number in r2
         ADR     r3, PageBlock1                  ; point at 1st entry we have to update
@@ -3718,7 +3898,7 @@ DoTheGrow Entry "r3,r5,r10-r12", DoTheGrowStackSize
         CLRV
         EXIT
 
-37
+DoTheGrowPageUnavailable ROUT
 
 ; Come here if a required page is not available
 ; First we need to go back thru all the part of the page block we've already done,
@@ -3751,11 +3931,12 @@ DoTheGrow Entry "r3,r5,r10-r12", DoTheGrowStackSize
 
         MakeErrorBlock  CantGetPhysMem
 
-DoTheGrowPagesSpecified
+DoTheGrowPagesSpecified ROUT
 
 ; First check if any of the pages requested are unavailable
 ; At the same time as we're doing this, we fill in the log. and phys. addresses in the block
 
+        FastCDA_ProfStart MarkRequired, r0, r6, lr
         LDR     r0, =ZeroPage
         LDR     r0, [r0, #CamEntriesPointer]
         LDR     r6, =L2PT
@@ -3767,7 +3948,7 @@ DoTheGrowPagesSpecified
         STR     r8, [r1, #PageBlockSize+4-12]   ; and in 2nd page block
 
         TST     lr, #PageFlags_Unavailable :OR: PageFlags_Required ; if page in use by someone else, or by us, then return error
-        BNE     %BT37
+        BNE     DoTheGrowPageUnavailable
         ORR     lr, lr, #PageFlags_Required     ; set bit in flags to say page will be needed
         STR     lr, [r4, #4]                    ; and store back
 
@@ -3785,13 +3966,16 @@ DoTheGrowPagesSpecified
 
         SUBS    r2, r2, #1
         BNE     %BT40
+        FastCDA_ProfEnd MarkRequired, r0, r6, lr
 
 ; now issue Service_PagesUnsafe
 
+        FastCDA_ProfStart PagesUnsafe, r0, r6, lr
         ADR     r2, PageBlock1                  ; r2 -> 1st page block
         LDR     r3, NumEntries                  ; r3 = number of entries in page block
         MOV     r1, #Service_PagesUnsafe
         BL      Issue_Service
+        FastCDA_ProfEnd PagesUnsafe, r0, r6, lr
 
 ; now move the pages
 
@@ -3824,7 +4008,44 @@ DoTheGrowPagesSpecified
 
         MOVS    r4, r3                          ; amount to do
         BEQ     %FT50                           ; [none, so skip all this]
+
+    [ FastCDA_UpFront
+        ; Perform sledgehammer logic upfront
+        Push    "r0,r2,r6"
+        FastCDA_ProfStart DoublyCheckCacheability, r0, r6, lr
+        MOV     r0, r1
+        MOV     r2, r3
+        BL      CheckCacheabilityR0ByMinusR2
+        FastCDA_ProfEnd DoublyCheckCacheability, r0, r2, lr
+        TST     r6, #DynAreaFlags_NotCacheable
+        LDR     r2, =ZeroPage
+      [ FastCDA_Prof
+        LDREQ   lr,[r2,#FastCDA_Prof_MMUChanging]
+        LDRNE   lr,[r2,#FastCDA_Prof_MMUChangingUncached]
+        MRC     p15,0,r0,c9,c13,0
+        SUB     lr,lr,r0
+        STREQ   lr,[r2,#FastCDA_Prof_MMUChanging]
+        STRNE   lr,[r2,#FastCDA_Prof_MMUChangingUncached]
+      ]
+        ADR     lr, %FT44
+        ARMop   MMU_Changing, EQ, tailcall, r2
+        ARMop   MMU_ChangingUncached, NE, tailcall, r2
+44
+      [ FastCDA_Prof
+        MRC     p15,0,r0,c9,c13,0
+        TST     r6, #DynAreaFlags_NotCacheable
+        LDREQ   lr,[r2,#FastCDA_Prof_MMUChanging]
+        LDRNE   lr,[r2,#FastCDA_Prof_MMUChangingUncached]
+        ADD     lr,lr,r0
+        STREQ   lr,[r2,#FastCDA_Prof_MMUChanging]
+        STRNE   lr,[r2,#FastCDA_Prof_MMUChangingUncached]
+      ]
+        Pull    "r0,r2,r6"
+        ORR     r6, r6, #PageFlags_Unsafe
+    ]
+
         Push    "r0, r1"
+        FastCDA_ProfStart DoublyMovePages, r0, r9, lr
         SUB     r0, r1, r3                      ; src starts at start of 1st copy = start of 2nd - old size
         SUB     r1, r0, r2                      ; dst start = src start - amount of room needed
         MOV     r9, #0                          ; no funny business while moving these pages
@@ -3834,6 +4055,7 @@ DoTheGrowPagesSpecified
         ADD     r1, r1, r5                      ; advance dst ptr
         SUBS    r4, r4, r5                      ; one less page to move
         BNE     %BT45                           ; loop if more
+        FastCDA_ProfEnd DoublyMovePages, r0, r9, lr
         Pull    "r0, r1"                        ; restore original regs
 50
         ADD     r9, r3, r2                      ; set up offset from 1st copy to 2nd copy (= new size)
@@ -3851,6 +4073,7 @@ DoTheGrowPagesSpecified
 ; Now before we start, we must construct the second page block, with replacement page numbers
 
 ;        DLINE   "Start of 1st loop"
+        FastCDA_ProfStart FindSpare, r6, r1, lr
 
 60
         LDR     r6, [r8], #12                   ; r6 = page number required
@@ -3907,11 +4130,21 @@ DoTheGrowPagesSpecified
 
         SUBS    r9, r9, #1                      ; one less entry to do
         BNE     %BT60
+        FastCDA_ProfEnd FindSpare, r7, r1, lr
 
         MOV     r7, r3                          ; r7 -> camentries
 
 ; Now we can go onto the 2nd loop which actually moves the pages
 
+     [ FastCDA_FIQs
+        ; Claim FIQs for this entire loop
+        ; (With the old behaviour, for large grows, total time in ReleaseFIQ could be several centiseconds, since the kernel reinstalls the default handler each time)
+        FastCDA_ProfStart ClaimFIQ, r6, r1, lr
+        MOV     r1, #Service_ClaimFIQ
+        BL      Issue_Service
+        FastCDA_ProfEnd ClaimFIQ, r6, r1, lr
+     ]
+
         LDR     r1, DestAddr
         MOV     r4, #0                          ; amount done
         MOV     r0, r7                          ; point r0 at camentries
@@ -3922,9 +4155,22 @@ DoTheGrowPagesSpecified
         MRS     r14, CPSR
         STR     r14, SavedPSR                   ; save old PSR (note: stack must be flat when we do this!)
 
+    [ FastCDA_NoPhysical
+        ; Grab the flags for the page we're replacing; NoPhysical optimisation means the page may get mapped to its target pos earlier than before, causing the flags in the CAM map to be "wrong" when we read them back out later on
+        LDR     r11, [r8, #0]                   ; need to get PPL for page being replaced
+        ADD     lr, r0, #4                      ; point at PPLs, not addresses
+        LDR     r11, [lr, r11, LSL #3]
+        MOV     lr, #1
+        STRB    lr, NeedToMoveFlag
+    ]
+
         Push    "r0-r4,r7-r12"                  ; save regs used during copy
+    [ :LNOT: FastCDA_FIQs
+        FastCDA_ProfStart ClaimFIQ, r6, r1, lr
         MOV     r1, #Service_ClaimFIQ
         BL      Issue_Service
+        FastCDA_ProfEnd ClaimFIQ, r6, r1, lr
+    ]
 
         WritePSRc I_bit+SVC_mode, r6            ; disable IRQs round here (we don't want interrupt code to update
                                                 ; the old mapping behind us while we're trying to copy it)
@@ -3941,24 +4187,50 @@ DoTheGrowPagesSpecified
         ; The old scheme, always copying from other mapping, had interrupt cache coherency hole, at least for
         ; ARM with writeback cache (bug in 3.7, fixed in Ursula, then lost)
 
+        LDR     r6, [r8, #4]                    ;logical address of src page
+
+      [ FastCDA_CorruptFreePool
+        ; If the source is in the free pool, we don't need to preserve its contents
+        LDR     r3, =ZeroPage+FreePoolDANode
+        SUB     r2, r6, #FreePoolAddress
+        LDR     r3, [r3, #DANode_MaxSize]
+        CMP     r2, r3
+        BLO     %FT74
+      ]
+
         LDR     r0, [r0, lr, LSL #3]            ; r0 = log. address for replacement page (NB use logical address to write to, for cache consistency)
 
-        LDR     r6, [r8, #4]                    ;logical address of src page
         LDR     r3, =Nowhere
         TEQ     r6, r3                          ;will be 'Nowhere' if not mapped in
         BNE     %FT71
 
+      [ FastCDA_NoPhysical
+        ; No need to use AccessPhysicalAddress here - if the required page isn't mapped in, all we need to do is map it in at the target address and use that mapping to copy the data out
+        LDR     r2, [r8, #0]
+        LDR     r3, [sp, #4] ; Get stacked r1
+        LDR     r11, [sp, #11*4+:INDEX:DestFlags]
+        FastCDA_ProfStart MoveNeeded, lr, r4, r7
+        BL      Call_CAM_Mapping                ; move needed page to destination
+        FastCDA_ProfEnd MoveNeeded, lr, r4, r7
+        MOV     r6, r3                          ; r6 = logical address of src for copy
+        MOV     lr, #0
+        STRB    lr, [sp, #11*4+:INDEX:NeedToMoveFlag]
+      |
         ASSERT  HAL
         SUB     sp, sp, #4                      ; for oldp
         Push    "r0,r1"
+        FastCDA_ProfStart AccessPhysical, r0, r1, lr
         MOV     r0, #0
         LDR     r1, [r8, #8]                    ; r1 = physical address of src for copy
         ADD     r2, sp, #8                      ; must use physical address, as page may be mapped to nowhere along with others
         BL      RISCOS_AccessPhysicalAddress
         MOV     r6, r0                          ; r6 = logical address of src for copy
+        FastCDA_ProfEnd AccessPhysical, r0, r1, lr
         Pull    "r0,r1"
+      ]
 
 71
+        FastCDA_ProfStart CopyPage, r2, r3, r4
         ADD     lr, r6, r5                      ; lr = end src address
 72
         LDMIA   r6!, {r2, r3, r4, r7, r9, r10, r11, r12}
@@ -3966,12 +4238,24 @@ DoTheGrowPagesSpecified
         TEQ     r6, lr
         BNE     %BT72
 
+        FastCDA_ProfEnd CopyPage, r2, r3, r4
+
+     [ :LNOT: FastCDA_NoPhysical
         LDR     r0, [r8, #4]                    ;logical address of src page
         LDR     r3, =Nowhere
         TEQ     r0, r3
 
         Pull    "r0", EQ                        ; oldp
+      [ FastCDA_Prof
+        BNE     %FT73
+        FastCDA_ProfStart ReleasePhysical, r2, r3, r4
+      ]
         BLEQ    RISCOS_ReleasePhysicalAddress
+      [ FastCDA_Prof
+        FastCDA_ProfEnd ReleasePhysical, r2, r3, r4
+73
+      ]
+     ]
 
 ; now check if page we're replacing is in L2PT, and if so then adjust L1PT entries (4 of these)
 
@@ -4010,38 +4294,71 @@ DoTheGrowPagesSpecified
         ; to avoid serious grief in the awkward cases. Fortunately, these page substitutions are relatively
         ; rare, so performance is not critical.
 
+ [ :LNOT: FastCDA_NoPhysical
         LDR     lr, =ZeroPage
         LDR     lr, [lr, #CamEntriesPointer]    ; lr -> soft cam map
         ADD     lr, lr, #4                      ; point at PPLs, not addresses
         LDR     r2, [r8, #0]                    ; need to get PPL for page being replaced
         LDR     r11, [lr, r2, LSL #3]
+ ]
         BIC     r11, r11, #PageFlags_Required   ; knock off bits that indicate that it was a required page
 
         ADD     lr, r8, #PageBlockSize
         LDMIA   lr, {r2, r3}                    ; get page number, logical address
 
+        LDR     lr, =Nowhere                    ; There's no point in cleaning the nowhere page, and on some architectures it'll even trigger an abort handler due to the lack of mapping
+        TEQ     r3, lr
+        BEQ     %FT75
+      [ FastCDA_Unnecessary
+        ; We only need to clean the cache/TLB if the page is cacheable
+        ; For uncacheable pages, BangL2PT will flush the TLB for us just before the mapping is updated
+        TST     r11, #DynAreaFlags_NotCacheable
+        BNE     %FT75
+      ]
         Push    "r0, r4"
-        LDR     r4, =Nowhere                    ; There's no point in cleaning the nowhere page, and on some architectures it'll even trigger an abort handler due to the lack of mapping
         MOV     r0, r3
-        TEQ     r3, r4
-        LDRNE   r4, =ZeroPage
-        ARMop   MMU_ChangingEntry,NE,,r4
+      [ FastCDA_Prof
+        FastCDA_ProfStart ChangingEntry, r6, lr, r4
+      ]
+        LDR     r4, =ZeroPage
+        ARMop   MMU_ChangingEntry,,,r4
+      [ FastCDA_Prof
+        FastCDA_ProfEnd ChangingEntry, r6, lr, r4
+      ]
         Pull    "r0, r4"
+75
 
+        FastCDA_ProfStart MoveReplacement, r6, lr, r5
         BL      Call_CAM_Mapping                ; move replacement page in
+        FastCDA_ProfEnd MoveReplacement, r6, lr, r5
 76
         LDR     r2, [r8, #0]
         MOV     r3, r1
         LDR     r11, DestFlags
+      [ FastCDA_NoPhysical
+        LDRB    lr, NeedToMoveFlag
+        TEQ     lr, #0
+        BEQ     %FT77                           ; don't bother if page already been moved to dest
+      ]
+        FastCDA_ProfStart MoveNeeded, r6, lr, r5
         BL      Call_CAM_Mapping                ; move needed page to destination
+        FastCDA_ProfEnd MoveNeeded, r6, lr, r5
 
+77
         LDR     lr, SavedPSR
         MSR     CPSR_cf, lr
 
+      [ :LNOT: FastCDA_FIQs
         Push    "r1"
+        FastCDA_ProfStart ReleaseFIQ, r1, lr, r5
         MOV     r1, #Service_ReleaseFIQ
         BL      Issue_Service
+        FastCDA_ProfEnd ReleaseFIQ, r1, lr, r5
         Pull    "r1"
+      ]
+      [ FastCDA_Prof
+        MOV     r5, #4096
+      ]
 
         ADD     r1, r1, r5                      ; advance dest ptr
         ADD     r4, r4, r5                      ; increment amount done
@@ -4049,17 +4366,26 @@ DoTheGrowPagesSpecified
         CMP     r4, r7                          ; have we done all?
         BNE     %BT70                           ; [no, so loop]
 
+     [ FastCDA_FIQs
+        FastCDA_ProfStart ReleaseFIQ, r1, lr, r2
+        MOV     r1, #Service_ReleaseFIQ
+        BL      Issue_Service
+        FastCDA_ProfEnd ReleaseFIQ, r1, lr, r2
+     ]
+
         LDR     r3, [r12, #DANode_Size]
         ADD     r3, r3, r7
         STR     r3, [r12, #DANode_Size]         ; store increased destination size
 
 ; now issue Service_PagesSafe
 
+        FastCDA_ProfStart PagesSafe, r1, r2, r3
         LDR     r2, NumEntries
         ADR     r3, PageBlock1
         ADR     r4, PageBlock2
         MOV     r1, #Service_PagesSafe
         BL      Issue_Service
+        FastCDA_ProfEnd PagesSafe, r1, r2, r3
 
 ; now call Post_Grow handler
 
@@ -4090,7 +4416,8 @@ DoTheGrowPagesSpecified
 ; Note: Removal is from one area only, the calling routine breaks the chunk at free/app boundary.
 
 
-DoTheGrowNotSpecified Entry "r3,r5,r10-r12", DoTheGrowNotSpecifiedStackSize
+DoTheGrowNotSpecified ROUT
+        Entry "r3,r5,r10-r12", DoTheGrowNotSpecifiedStackSize
 
         STR     r2, NumEntries                  ; save number of entries for use later
         STR     r7, TotalAmount                 ; save amount growing by
@@ -4124,6 +4451,23 @@ DoTheGrowNotSpecified Entry "r3,r5,r10-r12", DoTheGrowNotSpecifiedStackSize
 
         MOVS    r4, r3                          ; amount to do
         BEQ     %FT20                           ; [none, so skip all this]
+
+      [ FastCDA_UpFront
+        ; Perform sledgehammer logic upfront
+        Push    "r0,r2,r6"
+        MOV     r0, r1
+        MOV     r2, r3
+        BL      CheckCacheabilityR0ByMinusR2
+        TST     r6, #DynAreaFlags_NotCacheable
+        LDR     r0, =ZeroPage
+        ADR     lr, %FT14
+        ARMop   MMU_Changing, EQ, tailcall, r0
+        ARMop   MMU_ChangingUncached, NE, tailcall, r0
+14
+        Pull    "r0,r2,r6"
+        ORR     r6, r6, #PageFlags_Unsafe
+      ]
+
         Push    "r0, r1"
         SUB     r0, r1, r3                      ; src starts at start of 1st copy = start of 2nd - old size
         SUB     r1, r0, r2                      ; dst start = src start - amount of room needed
@@ -4139,6 +4483,21 @@ DoTheGrowNotSpecified Entry "r3,r5,r10-r12", DoTheGrowNotSpecifiedStackSize
         ADD     r9, r3, r2                      ; set up offset from 1st copy to 2nd copy (= new size)
 25
         ADD     r1, r1, r3                      ; r1 -> address of 1st extra page
+   [ FastCDA_UpFront
+        ; Flush src region from cache
+        Push    "r0-r1,r6"
+        BL      CheckCacheabilityR0ByMinusR2
+        TST     r6, #DynAreaFlags_NotCacheable
+        LDR     r4, =ZeroPage
+        MOV     r1, r2, LSR #12
+        ADR     lr, %FT29
+        ARMop   MMU_ChangingEntries, EQ, tailcall, r4
+        ARMop   MMU_ChangingUncachedEntries, NE, tailcall, r4
+29
+        Pull    "r0-r1,r6"
+        ; Now BangCam for all the pages
+        ORR     r6, r6, #PageFlags_Unsafe
+   ]
         MOV     r4, #0                          ; amount done so far
         MOV     r10, r2                         ; move amount to do into r10
 30
@@ -4182,7 +4541,8 @@ DoTheGrowNotSpecified Entry "r3,r5,r10-r12", DoTheGrowNotSpecifiedStackSize
 ;       endif
 ;
 
-CheckAppSpace Entry "r0-r3"
+CheckAppSpace ROUT
+        Entry "r0-r3"
         LDR     r2, =ZeroPage
         LDR     r3, [r2, #AplWorkSize]
         LDR     r2, [r2, #Curr_Active_Object]
@@ -4336,12 +4696,14 @@ CallPreGrow Entry "r0,r4, r12"
         CMP     r0, #0                                  ; if none (V=0)
         EXIT    EQ                                      ; then exit
 
+        FastCDA_ProfStart CallPreGrow, r0, r4, lr
         MOV     r0, #DAHandler_PreGrow                  ; r0 = reason code
         LDR     r4, [r12, #DANode_Size]                 ; r4 = current size
         ASSERT  DANode_Handler = DANode_Workspace +4
         ADD     r12, r12, #DANode_Workspace
         MOV     lr, pc
         LDMIA   r12, {r12, pc}                          ; load workspace pointer and jump to handler
+        FastCDA_ProfEnd CallPreGrow, r12, r4, lr
         EXIT    VC                                      ; if no error then exit
 
         TEQ     r0, #0                                  ; if generic error returned
@@ -4371,12 +4733,14 @@ CallPostGrow Entry "r0,r3,r4, r12"
         CMP     r0, #0                                  ; if none (V=0)
         EXIT    EQ                                      ; then exit
 
+        FastCDA_ProfStart CallPostGrow, r0, r4, lr
         MOV     r0, #DAHandler_PostGrow                 ; r0 = reason code
         LDR     r4, [r12, #DANode_Size]                 ; r4 = new size
         ASSERT  DANode_Handler = DANode_Workspace +4
         ADD     r12, r12, #DANode_Workspace
         MOV     lr, pc
         LDMIA   r12, {r12, pc}                          ; load workspace pointer and jump to handler
+        FastCDA_ProfEnd CallPostGrow, r12, r4, lr
         EXIT
 
  [ ShrinkableDAs

s/VMSAv6

@@ -75,7 +75,8 @@ BangCamUpdate ROUT
         LDR     r1, [r1, #CamEntriesPointer]
         ADD     r1, r1, r2, LSL #3              ; point at cam entry (logaddr, PPL)
         LDMIA   r1, {r0, r6}                    ; r0 = current logaddress, r6 = current PPL
-        STMIA   r1, {r3, r11}                   ; store new address, PPL
+        BIC     r4, r11, #PageFlags_Unsafe
+        STMIA   r1, {r3, r4}                    ; store new address, PPL
         Push    "r0, r6"                        ; save old logical address, PPL
         LDR     r1, =ZeroPage+PhysRamTable      ; go through phys RAM table
         MOV     r6, r2                          ; make copy of r2 (since that must be preserved)
@@ -105,11 +106,15 @@ BangCamUpdate ROUT
         TEQ     r4, r0, LSR #12                 ; if equal to physical address of page being moved
         BNE     %FT20                           ; if not there, then just put in new page
 
+        AND     r4, r11, #PageFlags_Unsafe
         Push    "r0, r3, r11, r14"              ; save phys.addr, new log.addr, new PPL, lr
         ADD     r3, sp, #4*4
         LDMIA   r3, {r3, r11}                   ; reload old logical address, old PPL
+        LDR     r0, =DuffEntry                  ; Nothing to do if wasn't mapped in
+        ORR     r11, r11, r4
+        TEQ     r3, r0
         MOV     r0, #0                          ; cause translation fault
-        BL      BangL2PT                        ; map page out
+        BLNE    BangL2PT                        ; map page out
         Pull    "r0, r3, r11, r14"
 20
         ADD     sp, sp, #8                      ; junk old logical address, PPL
@@ -197,6 +202,9 @@ BangL2PT                                        ; internal entry point used only
         Push    "lr"
         MOV     r6, r0
 
+        TST     r11, #PageFlags_Unsafe
+        BNE     %FT30
+
         TST     r11, #DynAreaFlags_DoublyMapped
         BNE     BangL2PT_sledgehammer           ;if doubly mapped, don't try to be clever