/*
   *  linux/arch/alpha/lib/memcpy.c
   *
   *  Copyright (C) 1995  Linus Torvalds
   */
  
  /*
   * This is a reasonably optimized memcpy() routine.
   */
  
  /*
   * Note that the C code is written to be optimized into good assembly. However,
   * at this point gcc is unable to sanely compile "if (n >= 0)", resulting in a
   * explicit compare against 0 (instead of just using the proper "blt reg, xx" or
   * "bge reg, xx"). I hope alpha-gcc will be fixed to notice this eventually..
   */
  
  #include <linux/types.h>
  
  /*
   * This should be done in one go with ldq_u*2/mask/stq_u. Do it
   * with a macro so that we can fix it up later..
   */
  #define ALIGN_DEST_TO8_UP(d,s,n) \
  	while (d & 7) { \
  		if (n <= 0) return; \
  		n--; \
  		*(char *) d = *(char *) s; \
  		d++; s++; \
  	}
  #define ALIGN_DEST_TO8_DN(d,s,n) \
  	while (d & 7) { \
  		if (n <= 0) return; \
  		n--; \
  		d--; s--; \
  		*(char *) d = *(char *) s; \
  	}
  
  /*
   * This should similarly be done with ldq_u*2/mask/stq. The destination
   * is aligned, but we don't fill in a full quad-word
   */
  #define DO_REST_UP(d,s,n) \
  	while (n > 0) { \
  		n--; \
  		*(char *) d = *(char *) s; \
  		d++; s++; \
  	}
  #define DO_REST_DN(d,s,n) \
  	while (n > 0) { \
  		n--; \
  		d--; s--; \
  		*(char *) d = *(char *) s; \
  	}
  
  /*
   * This should be done with ldq/mask/stq. The source and destination are
   * aligned, but we don't fill in a full quad-word
   */
  #define DO_REST_ALIGNED_UP(d,s,n) DO_REST_UP(d,s,n)
  #define DO_REST_ALIGNED_DN(d,s,n) DO_REST_DN(d,s,n)
  
  /*
   * This does unaligned memory copies. We want to avoid storing to
   * an unaligned address, as that would do a read-modify-write cycle.
   * We also want to avoid double-reading the unaligned reads.
   *
   * Note the ordering to try to avoid load (and address generation) latencies.
   */
  static inline void __memcpy_unaligned_up (unsigned long d, unsigned long s,
  					  long n)
  {
  	ALIGN_DEST_TO8_UP(d,s,n);
  	n -= 8;			/* to avoid compare against 8 in the loop */
  	if (n >= 0) {
  		unsigned long low_word, high_word;
  		__asm__("ldq_u %0,%1":"=r" (low_word):"m" (*(unsigned long *) s));
  		do {
  			unsigned long tmp;
  			__asm__("ldq_u %0,%1":"=r" (high_word):"m" (*(unsigned long *)(s+8)));
  			n -= 8;
  			__asm__("extql %1,%2,%0"
  				:"=r" (low_word)
  				:"r" (low_word), "r" (s));
  			__asm__("extqh %1,%2,%0"
  				:"=r" (tmp)
  				:"r" (high_word), "r" (s));
  			s += 8;
  			*(unsigned long *) d = low_word | tmp;
  			d += 8;
  			low_word = high_word;
  		} while (n >= 0);
  	}
  	n += 8;
  	DO_REST_UP(d,s,n);
  }
  
  static inline void __memcpy_unaligned_dn (unsigned long d, unsigned long s,
  					  long n)
  {
  	/* I don't understand AXP assembler well enough for this. -Tim */
  	s += n;
  	d += n;
  	while (n--)
  		* (char *) --d = * (char *) --s;
  }
  
  /*
   * Hmm.. Strange. The __asm__ here is there to make gcc use an integer register
   * for the load-store. I don't know why, but it would seem that using a floating
   * point register for the move seems to slow things down (very small difference,
   * though).
   *
   * Note the ordering to try to avoid load (and address generation) latencies.
   */
  static inline void __memcpy_aligned_up (unsigned long d, unsigned long s,
  					long n)
  {
  	ALIGN_DEST_TO8_UP(d,s,n);
  	n -= 8;
  	while (n >= 0) {
  		unsigned long tmp;
  		__asm__("ldq %0,%1":"=r" (tmp):"m" (*(unsigned long *) s));
  		n -= 8;
  		s += 8;
  		*(unsigned long *) d = tmp;
  		d += 8;
  	}
  	n += 8;
  	DO_REST_ALIGNED_UP(d,s,n);
  }
  static inline void __memcpy_aligned_dn (unsigned long d, unsigned long s,
  					long n)
  {
  	s += n;
  	d += n;
  	ALIGN_DEST_TO8_DN(d,s,n);
  	n -= 8;
  	while (n >= 0) {
  		unsigned long tmp;
  		s -= 8;
  		__asm__("ldq %0,%1":"=r" (tmp):"m" (*(unsigned long *) s));
  		n -= 8;
  		d -= 8;
  		*(unsigned long *) d = tmp;
  	}
  	n += 8;
  	DO_REST_ALIGNED_DN(d,s,n);
  }
  
  void * memcpy(void * dest, const void *src, size_t n)
  {
  	if (!(((unsigned long) dest ^ (unsigned long) src) & 7)) {
  		__memcpy_aligned_up ((unsigned long) dest, (unsigned long) src,
  				     n);
  		return dest;
  	}
  	__memcpy_unaligned_up ((unsigned long) dest, (unsigned long) src, n);
  	return dest;
  }
  
  /* For backward modules compatibility, define __memcpy.  */
  asm("__memcpy = memcpy; .globl __memcpy");