Merge remote-tracking branch 'github/cm-11.0' into HEAD

8 files changed, 319 insertions, 192 deletions

diff --git a/libc/Android.mk b/libc/Android.mk
index c538aa733..1b02b78de 100644
--- a/libc/Android.mk
+++ b/libc/Android.mk
@@ -370,7 +370,6 @@ libc_common_src_files += \
 	string/strncpy.c \
 	bionic/strchr.cpp \
 	string/strrchr.c \
-	bionic/memchr.c \
 	bionic/memrchr.c \
 	string/index.c \
 	bionic/strnlen.c \
@@ -412,6 +411,8 @@ libc_static_common_src_files += \
     bionic/pthread_create.cpp \
     bionic/pthread_key.cpp \
 
+libc_common_src_files += \
+    bionic/memchr.c
 endif # x86
 
 ifeq ($(TARGET_ARCH),mips)
diff --git a/libc/arch-arm/cortex-a15/bionic/memchr.S b/libc/arch-arm/cortex-a15/bionic/memchr.S
new file mode 100644
index 000000000..918bc69e2
--- /dev/null
+++ b/libc/arch-arm/cortex-a15/bionic/memchr.S
@@ -0,0 +1,153 @@
+/* Copyright (c) 2010-2011, Linaro Limited
+   All rights reserved.
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions
+   are met:
+
+      * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+      * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+
+      * Neither the name of Linaro Limited nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+   HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+   Written by Dave Gilbert <david.gilbert@linaro.org>
+   Adapted to work inside Bionic by Bernhard Rosenkränzer <bero@linaro.org>
+
+   This memchr routine is optimised on a Cortex-A9 and should work on
+   all ARMv7 processors.   It has a fast past for short sizes, and has
+   an optimised path for large data sets; the worst case is finding the
+   match early in a large data set.
+
+ */
+
+// Prototype: void *memchr(const void *s, int c, size_t n);
+
+#include <machine/asm.h>
+#include "private/libc_events.h"
+
+	.syntax unified
+	.arch armv7-a
+
+@ this lets us check a flag in a 00/ff byte easily in either endianness
+#ifdef __ARMEB__
+#define CHARTSTMASK(c) 1<<(31-(c*8))
+#else
+#define CHARTSTMASK(c) 1<<(c*8)
+#endif
+	.text
+	.thumb
+
+@ ---------------------------------------------------------------------------
+ENTRY(memchr)
+	.thumb_func
+	.align 2
+	.p2align 4,,15
+	@ r0 = start of memory to scan
+	@ r1 = character to look for
+	@ r2 = length
+	@ returns r0 = pointer to character or NULL if not found
+	and	r1,r1,#0xff	@ Don't think we can trust the caller to actually pass a char
+
+	cmp	r2,#16		@ If it's short don't bother with anything clever
+	blt	20f 
+
+	tst	r0, #7		@ If it's already aligned skip the next bit
+	beq	10f
+
+	@ Work up to an aligned point
+5:
+	ldrb	r3, [r0],#1
+	subs	r2, r2, #1
+	cmp	r3, r1
+	beq	50f		@ If it matches exit found
+	tst	r0, #7
+	bne	5b		@ If not aligned yet then do next byte
+	
+10:
+	@ At this point, we are aligned, we know we have at least 8 bytes to work with
+	push	{r4,r5,r6,r7}
+	orr	r1, r1, r1, lsl #8	@ expand the match word across to all bytes
+	orr	r1, r1, r1, lsl #16
+	bic	r4, r2, #7	@ Number of double words to work with
+	mvns	r7, #0		@ all F's
+	movs	r3, #0
+	
+15:
+	ldmia	r0!,{r5,r6}
+	subs	r4, r4, #8
+	eor	r5,r5, r1	@ Get it so that r5,r6 have 00's where the bytes match the target
+	eor	r6,r6, r1
+	uadd8	r5, r5, r7	@ Parallel add 0xff - sets the GE bits for anything that wasn't 0
+	sel	r5, r3, r7	@ bytes are 00 for none-00 bytes, or ff for 00 bytes - NOTE INVERSION
+	uadd8	r6, r6, r7	@ Parallel add 0xff - sets the GE bits for anything that wasn't 0
+	sel	r6, r5, r7	@ chained....bytes are 00 for none-00 bytes, or ff for 00 bytes - NOTE INVERSION
+	cbnz	r6, 60f
+	bne	15b		@ (Flags from the subs above) If not run out of bytes then go around again
+
+	pop	{r4,r5,r6,r7}
+	and	r1,r1,#0xff	@ Get r1 back to a single character from the expansion above
+	and	r2,r2,#7	@ Leave the count remaining as the number after the double words have been done
+ 
+20:
+	cbz	r2, 40f		@ 0 length or hit the end already then not found
+
+21:  @ Post aligned section, or just a short call
+	ldrb	r3,[r0],#1
+	subs	r2,r2,#1
+	eor	r3,r3,r1	@ r3 = 0 if match - doesn't break flags from sub
+	cbz	r3, 50f
+	bne	21b		@ on r2 flags
+
+40:
+	movs	r0,#0		@ not found
+	bx	lr
+
+50:
+	subs	r0,r0,#1	@ found
+	bx	lr
+
+60:  @ We're here because the fast path found a hit - now we have to track down exactly which word it was
+	@ r0 points to the start of the double word after the one that was tested
+	@ r5 has the 00/ff pattern for the first word, r6 has the chained value
+	cmp	r5, #0
+	itte	eq
+	moveq	r5, r6		@ the end is in the 2nd word
+	subeq	r0,r0,#3	@ Points to 2nd byte of 2nd word
+	subne	r0,r0,#7	@ or 2nd byte of 1st word
+
+	@ r0 currently points to the 3rd byte of the word containing the hit
+	tst	r5, # CHARTSTMASK(0)	@ 1st character
+	bne	61f
+	adds	r0,r0,#1
+	tst	r5, # CHARTSTMASK(1)	@ 2nd character
+	ittt	eq
+	addeq	r0,r0,#1
+	tsteq	r5, # (3<<15)		@ 2nd & 3rd character
+	@ If not the 3rd must be the last one
+	addeq	r0,r0,#1
+
+61:
+	pop	{r4,r5,r6,r7}
+	subs	r0,r0,#1
+	bx	lr
+END(memchr)
diff --git a/libc/arch-arm/cortex-a15/bionic/strlen.S b/libc/arch-arm/cortex-a15/bionic/strlen.S
index 08f6d193b..75f23745b 100644
--- a/libc/arch-arm/cortex-a15/bionic/strlen.S
+++ b/libc/arch-arm/cortex-a15/bionic/strlen.S
@@ -1,165 +1,146 @@
-/*
- * Copyright (C) 2013 The Android Open Source Project
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *  * Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- *  * Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in
- *    the documentation and/or other materials provided with the
- *    distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
- * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
+/* Copyright (c) 2010-2011,2013 Linaro Limited
+   All rights reserved.
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions
+   are met:
+
+      * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+      * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+
+      * Neither the name of Linaro Limited nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+   HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
+
 /*
- * Copyright (c) 2013 ARM Ltd
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the company may not be used to endorse or promote
- *    products derived from this software without specific prior written
- *    permission.
- *
- * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+   Assumes:
+   ARMv6T2, AArch32
+
+   Adapted to Bionic by Bernhard Rosenkränzer <bero@linaro.org>
  */
 
 #include <machine/asm.h>
 
-    .syntax unified
-
-    .thumb
-    .thumb_func
+#ifdef __ARMEB__
+#define S2LO		lsl
+#define S2HI		lsr
+#else
+#define S2LO		lsr
+#define S2HI		lsl
+#endif
+
+	.text
+	/* This code requires Thumb.  */
+	.thumb
+	.syntax unified
+
+/* Parameters and result.  */
+#define srcin		r0
+#define result		r0
+
+/* Internal variables.  */
+#define src		r1
+#define data1a		r2
+#define data1b		r3
+#define const_m1	r12
+#define const_0		r4
+#define tmp1		r4		/* Overlaps const_0  */
+#define tmp2		r5
 
 ENTRY(strlen)
-    pld     [r0, #0]
-    mov     r1, r0
-
-    ands    r3, r0, #7
-    beq     mainloop
-
-    // Align to a double word (64 bits).
-    rsb     r3, r3, #8
-    lsls    ip, r3, #31
-    beq     align_to_32
-
-    ldrb    r2, [r1], #1
-    cbz     r2, update_count_and_return
-
-align_to_32:
-    bcc     align_to_64
-    ands    ip, r3, #2
-    beq     align_to_64
-
-    ldrb    r2, [r1], #1
-    cbz     r2, update_count_and_return
-    ldrb    r2, [r1], #1
-    cbz     r2, update_count_and_return
-
-align_to_64:
-    tst     r3, #4
-    beq     mainloop
-    ldr     r3, [r1], #4
-
-    sub     ip, r3, #0x01010101
-    bic     ip, ip, r3
-    ands    ip, ip, #0x80808080
-    bne     zero_in_second_register
-
-    .p2align 2
-mainloop:
-    ldrd    r2, r3, [r1], #8
-
-    pld     [r1, #64]
-
-    sub     ip, r2, #0x01010101
-    bic     ip, ip, r2
-    ands    ip, ip, #0x80808080
-    bne     zero_in_first_register
-
-    sub     ip, r3, #0x01010101
-    bic     ip, ip, r3
-    ands    ip, ip, #0x80808080
-    bne     zero_in_second_register
-    b       mainloop
-
-update_count_and_return:
-    sub     r0, r1, r0
-    sub     r0, r0, #1
-    bx      lr
-
-zero_in_first_register:
-    sub     r0, r1, r0
-    lsls    r3, ip, #17
-    bne     sub8_and_return
-    bcs     sub7_and_return
-    lsls    ip, ip, #1
-    bne     sub6_and_return
-
-    sub     r0, r0, #5
-    bx      lr
-
-sub8_and_return:
-    sub     r0, r0, #8
-    bx      lr
-
-sub7_and_return:
-    sub     r0, r0, #7
-    bx      lr
-
-sub6_and_return:
-    sub     r0, r0, #6
-    bx      lr
-
-zero_in_second_register:
-    sub     r0, r1, r0
-    lsls    r3, ip, #17
-    bne     sub4_and_return
-    bcs     sub3_and_return
-    lsls    ip, ip, #1
-    bne     sub2_and_return
-
-    sub     r0, r0, #1
-    bx      lr
-
-sub4_and_return:
-    sub     r0, r0, #4
-    bx      lr
-
-sub3_and_return:
-    sub     r0, r0, #3
-    bx      lr
-
-sub2_and_return:
-    sub     r0, r0, #2
-    bx      lr
+	.p2align 6
+	pld	[srcin, #0]
+	strd	r4, r5, [sp, #-8]!
+	bic	src, srcin, #7
+	mvn	const_m1, #0
+	ands	tmp1, srcin, #7		/* (8 - bytes) to alignment.  */
+	pld	[src, #32]
+	bne.w	misaligned8
+	mov	const_0, #0
+	mov	result, #-8
+loop_aligned:
+	/* Bytes 0-7.  */
+	ldrd	data1a, data1b, [src]
+	pld	[src, #64]
+	add	result, result, #8
+start_realigned:
+	uadd8	data1a, data1a, const_m1	/* Saturating GE<0:3> set.  */
+	sel	data1a, const_0, const_m1	/* Select based on GE<0:3>.  */
+	uadd8	data1b, data1b, const_m1
+	sel	data1b, data1a, const_m1	/* Only used if d1a == 0.  */
+	cbnz	data1b, null_found
+
+	/* Bytes 8-15.  */
+	ldrd	data1a, data1b, [src, #8]
+	uadd8	data1a, data1a, const_m1	/* Saturating GE<0:3> set.  */
+	add	result, result, #8
+	sel	data1a, const_0, const_m1	/* Select based on GE<0:3>.  */
+	uadd8	data1b, data1b, const_m1
+	sel	data1b, data1a, const_m1	/* Only used if d1a == 0.  */
+	cbnz	data1b, null_found
+
+	/* Bytes 16-23.  */
+	ldrd	data1a, data1b, [src, #16]
+	uadd8	data1a, data1a, const_m1	/* Saturating GE<0:3> set.  */
+	add	result, result, #8
+	sel	data1a, const_0, const_m1	/* Select based on GE<0:3>.  */
+	uadd8	data1b, data1b, const_m1
+	sel	data1b, data1a, const_m1	/* Only used if d1a == 0.  */
+	cbnz	data1b, null_found
+
+	/* Bytes 24-31.  */
+	ldrd	data1a, data1b, [src, #24]
+	add	src, src, #32
+	uadd8	data1a, data1a, const_m1	/* Saturating GE<0:3> set.  */
+	add	result, result, #8
+	sel	data1a, const_0, const_m1	/* Select based on GE<0:3>.  */
+	uadd8	data1b, data1b, const_m1
+	sel	data1b, data1a, const_m1	/* Only used if d1a == 0.  */
+	cmp	data1b, #0
+	beq	loop_aligned
+
+null_found:
+	cmp	data1a, #0
+	itt	eq
+	addeq	result, result, #4
+	moveq	data1a, data1b
+#ifndef __ARMEB__
+	rev	data1a, data1a
+#endif
+	clz	data1a, data1a
+	ldrd	r4, r5, [sp], #8
+	add	result, result, data1a, lsr #3	/* Bits -> Bytes.  */
+	bx	lr
+
+misaligned8:
+	ldrd	data1a, data1b, [src]
+	and	tmp2, tmp1, #3
+	rsb	result, tmp1, #0
+	lsl	tmp2, tmp2, #3			/* Bytes -> bits.  */
+	tst	tmp1, #4
+	pld	[src, #64]
+	S2HI	tmp2, const_m1, tmp2
+	orn	data1a, data1a, tmp2
+	itt	ne
+	ornne	data1b, data1b, tmp2
+	movne	data1a, const_m1
+	mov	const_0, #0
+	b	start_realigned
 END(strlen)
diff --git a/libc/arch-arm/cortex-a15/cortex-a15.mk b/libc/arch-arm/cortex-a15/cortex-a15.mk
index 483a5214a..bc45cccd0 100644
--- a/libc/arch-arm/cortex-a15/cortex-a15.mk
+++ b/libc/arch-arm/cortex-a15/cortex-a15.mk
@@ -8,5 +8,6 @@ $(call libc-add-cpu-variant-src,STRCPY,arch-arm/cortex-a15/bionic/strcpy.S)
 $(call libc-add-cpu-variant-src,STRLEN,arch-arm/cortex-a15/bionic/strlen.S)
 $(call libc-add-cpu-variant-src,__STRCAT_CHK,arch-arm/cortex-a15/bionic/__strcat_chk.S)
 $(call libc-add-cpu-variant-src,__STRCPY_CHK,arch-arm/cortex-a15/bionic/__strcpy_chk.S)
+$(call libc-add-cpu-variant-src,MEMCHR,arch-arm/cortex-a15/bionic/memchr.S)
 
 include bionic/libc/arch-arm/generic/generic.mk
diff --git a/libc/arch-arm/generic/generic.mk b/libc/arch-arm/generic/generic.mk
index f9a1e48b6..b3ed69dbf 100644
--- a/libc/arch-arm/generic/generic.mk
+++ b/libc/arch-arm/generic/generic.mk
@@ -8,3 +8,4 @@ $(call libc-add-cpu-variant-src,STRCPY,arch-arm/generic/bionic/strcpy.S)
 $(call libc-add-cpu-variant-src,STRLEN,arch-arm/generic/bionic/strlen.c)
 $(call libc-add-cpu-variant-src,__STRCAT_CHK,bionic/__strcat_chk.cpp)
 $(call libc-add-cpu-variant-src,__STRCPY_CHK,bionic/__strcpy_chk.cpp)
+$(call libc-add-cpu-variant-src,MEMCHR,bionic/memchr.c)
diff --git a/libc/arch-arm/krait/bionic/memmove.S b/libc/arch-arm/krait/bionic/memmove.S
index cfa06cef4..b7b77ce7e 100644
--- a/libc/arch-arm/krait/bionic/memmove.S
+++ b/libc/arch-arm/krait/bionic/memmove.S
@@ -94,28 +94,15 @@ _memmove_words:
 	cmpge	r12, r0
 	it	le
 	ble	memcpy
-	cmp	r2, #4
-	it	le
-	ble	.Lneon_b2f_smallcopy_loop
+	cmp	r2, #63
+	ble	.Lneon_b2f_smallcopy
 	push	{r0, lr}
 	add	r0, r0, r2
 	add	r1, r1, r2
-	cmp	r2, #64
-	it	ge
-	bge	.Lneon_b2f_copy_64
-	cmp	r2, #32
-	it	ge
-	bge	.Lneon_b2f_copy_32
-	cmp	r2, #8
-	it	ge
-	bge	.Lneon_b2f_copy_8
-	b	.Lneon_b2f_copy_1
-.Lneon_b2f_copy_64:
 	mov	r12, r2, lsr #6
-	add	r1, r1, #32
 	add	r0, r0, #32
+	add	r1, r1, #32
 	cmp	r12, #PLDTHRESH
-	it	le
 	ble	.Lneon_b2f_copy_64_loop_nopld
 	sub	r12, #PLDOFFS
 	sub	lr, r1, #(PLDOFFS)*PLDSIZE
@@ -129,7 +116,6 @@ _memmove_words:
 	subs	r12, r12, #1
 	vst1.32	{q0, q1}, [r0]!
 	vst1.32	{q2, q3}, [r0]
-	it	ne
 	bne	.Lneon_b2f_copy_64_loop_outer
 	mov	r12, #PLDOFFS
 .Lneon_b2f_copy_64_loop_nopld:
@@ -140,15 +126,12 @@ _memmove_words:
 	subs	r12, r12, #1
 	vst1.32	{q8, q9}, [r0]!
 	vst1.32	{q10, q11}, [r0]
-	it	ne
 	bne	.Lneon_b2f_copy_64_loop_nopld
 	ands	r2, r2, #0x3f
-	it	eq
 	beq	.Lneon_memmove_done
 	sub	r1, r1, #32
 	sub	r0, r0, #32
 	cmp	r2, #32
-	it	lt
 	blt	.Lneon_b2f_copy_8
 .Lneon_b2f_copy_32:
 	sub	r1, r1, #32
@@ -156,12 +139,9 @@ _memmove_words:
 	vld1.32	{q0, q1}, [r1]
 	vst1.32	{q0, q1}, [r0]
 	ands	r2, r2, #0x1f
-	it	eq
 	beq	.Lneon_memmove_done
-.Lneon_b2f_copy_finish:
 .Lneon_b2f_copy_8:
 	movs	r12, r2, lsr #0x3
-	it	eq
 	beq	.Lneon_b2f_copy_1
 .Lneon_b2f_copy_8_loop:
 	sub	r1, r1, #8
@@ -169,30 +149,39 @@ _memmove_words:
 	vld1.32	{d0}, [r1]
 	subs	r12, r12, #1
 	vst1.32	{d0}, [r0]
-	it	ne
 	bne	.Lneon_b2f_copy_8_loop
-.Lneon_b2f_copy_1:
 	ands	r2, r2, #0x7
-	it	eq
 	beq	.Lneon_memmove_done
+.Lneon_b2f_copy_1:
 	sub	r1, r1, r2
 	sub	r0, r0, r2
-.Lneon_b2f_copy_1_loop:
+	ands	r12, r2, #1
+	beq	.Lneon_b2f_copy_halfword_loop
 	subs	r2, r2, #1
 	ldrb	r3, [r1, r2]
 	strb	r3, [r0, r2]
-	it	ne
-	bne	.Lneon_b2f_copy_1_loop
+	beq	.Lneon_memmove_done
+.Lneon_b2f_copy_halfword_loop:
+	subs	r2, r2, #2
+	ldrh	r3, [r1, r2]
+	strh	r3, [r0, r2]
+	bne	.Lneon_b2f_copy_halfword_loop
 .Lneon_memmove_done:
 	pop	{r0, pc}
-.Lneon_b2f_smallcopy_loop:
+.Lneon_b2f_smallcopy:
+	ands	r12, r2, #1
+	beq	.Lneon_b2f_halfword_small_loop
 	subs	r2, r2, #1
 	ldrb	r3, [r1, r2]
 	strb	r3, [r0, r2]
-	it	ne
-	bne	.Lneon_b2f_smallcopy_loop
+	it	eq
+	bxeq	lr
+.Lneon_b2f_halfword_small_loop:
+	subs	r2, r2, #2
+	ldrh	r3, [r1, r2]
+	strh	r3, [r0, r2]
+	bne	.Lneon_b2f_halfword_small_loop
 	bx	lr
         .cfi_endproc
 END(memmove)
 
-
diff --git a/libc/arch-arm/krait/krait.mk b/libc/arch-arm/krait/krait.mk
index d1f06084e..868ad4c0e 100644
--- a/libc/arch-arm/krait/krait.mk
+++ b/libc/arch-arm/krait/krait.mk
@@ -14,4 +14,4 @@ $(call libc-add-cpu-variant-src,STRCAT,arch-arm/cortex-a15/bionic/strcat.S)
 $(call libc-add-cpu-variant-src,STRCPY,arch-arm/cortex-a15/bionic/strcpy.S)
 $(call libc-add-cpu-variant-src,STRLEN,arch-arm/cortex-a15/bionic/strlen.S)
 
-#include bionic/libc/arch-arm/generic/generic.mk
+include bionic/libc/arch-arm/generic/generic.mk
diff --git a/libc/arch-arm/scorpion/scorpion.mk b/libc/arch-arm/scorpion/scorpion.mk
index 52bf78bfa..fb893f0f8 100644
--- a/libc/arch-arm/scorpion/scorpion.mk
+++ b/libc/arch-arm/scorpion/scorpion.mk
@@ -2,6 +2,7 @@ $(call libc-add-cpu-variant-src,MEMCPY,arch-arm/cortex-a15/bionic/memcpy.S)
 $(call libc-add-cpu-variant-src,MEMSET,arch-arm/krait/bionic/memset.S)
 $(call libc-add-cpu-variant-src,STRCMP,arch-arm/krait/bionic/strcmp.S)
 $(call libc-add-cpu-variant-src,MEMMOVE,arch-arm/krait/bionic/memmove.S)
+$(call libc-add-cpu-variant-src,BCOPY,)
 # Use cortex-a15 versions of strcat/strcpy/strlen.
 $(call libc-add-cpu-variant-src,STRCAT,arch-arm/cortex-a15/bionic/strcat.S)
 $(call libc-add-cpu-variant-src,STRCPY,arch-arm/cortex-a15/bionic/strcpy.S)
@@ -9,4 +10,4 @@ $(call libc-add-cpu-variant-src,STRLEN,arch-arm/cortex-a15/bionic/strlen.S)
 $(call libc-add-cpu-variant-src,__STRCAT_CHK,arch-arm/cortex-a15/bionic/__strcat_chk.S)
 $(call libc-add-cpu-variant-src,__STRCPY_CHK,arch-arm/cortex-a15/bionic/__strcpy_chk.S)
 
-#include bionic/libc/arch-arm/generic/generic.mk
+include bionic/libc/arch-arm/generic/generic.mk