/*
   *  linux/arch/arm/mm/context.c
   *
   *  Copyright (C) 2002-2003 Deep Blue Solutions Ltd, all rights reserved.
   *  Copyright (C) 2012 ARM Limited
   *
   *  Author: Will Deacon <will.deacon@arm.com>
   *
   * This program is free software; you can redistribute it and/or modify
   * it under the terms of the GNU General Public License version 2 as
   * published by the Free Software Foundation.
   */
  #include <linux/init.h>
  #include <linux/sched.h>
  #include <linux/mm.h>
  #include <linux/smp.h>
  #include <linux/percpu.h>
  
  #include <asm/mmu_context.h>
  #include <asm/smp_plat.h>
  #include <asm/thread_notify.h>
  #include <asm/tlbflush.h>
  #include <asm/proc-fns.h>
  
  /*
   * On ARMv6, we have the following structure in the Context ID:
   *
   * 31                         7          0
   * +-------------------------+-----------+
   * |      process ID         |   ASID    |
   * +-------------------------+-----------+
   * |              context ID             |
   * +-------------------------------------+
   *
   * The ASID is used to tag entries in the CPU caches and TLBs.
   * The context ID is used by debuggers and trace logic, and
   * should be unique within all running processes.
   *
   * In big endian operation, the two 32 bit words are swapped if accessed
   * by non-64-bit operations.
   */
  #define ASID_FIRST_VERSION	(1ULL << ASID_BITS)
  #define NUM_USER_ASIDS		ASID_FIRST_VERSION
  
  static DEFINE_RAW_SPINLOCK(cpu_asid_lock);
  static atomic64_t asid_generation = ATOMIC64_INIT(ASID_FIRST_VERSION);
  static DECLARE_BITMAP(asid_map, NUM_USER_ASIDS);
  
  static DEFINE_PER_CPU(atomic64_t, active_asids);
  static DEFINE_PER_CPU(u64, reserved_asids);
  static cpumask_t tlb_flush_pending;
  
  #ifdef CONFIG_ARM_ERRATA_798181
  void a15_erratum_get_cpumask(int this_cpu, struct mm_struct *mm,
  			     cpumask_t *mask)
  {
  	int cpu;
  	unsigned long flags;
  	u64 context_id, asid;
  
  	raw_spin_lock_irqsave(&cpu_asid_lock, flags);
  	context_id = mm->context.id.counter;
  	for_each_online_cpu(cpu) {
  		if (cpu == this_cpu)
  			continue;
  		/*
  		 * We only need to send an IPI if the other CPUs are
  		 * running the same ASID as the one being invalidated.
  		 */
  		asid = per_cpu(active_asids, cpu).counter;
  		if (asid == 0)
  			asid = per_cpu(reserved_asids, cpu);
  		if (context_id == asid)
  			cpumask_set_cpu(cpu, mask);
  	}
  	raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);
  }
  #endif
  
  #ifdef CONFIG_ARM_LPAE
  /*
   * With LPAE, the ASID and page tables are updated atomicly, so there is
   * no need for a reserved set of tables (the active ASID tracking prevents
   * any issues across a rollover).
   */
  #define cpu_set_reserved_ttbr0()
  #else
  static void cpu_set_reserved_ttbr0(void)
  {
  	u32 ttb;
  	/*
  	 * Copy TTBR1 into TTBR0.
  	 * This points at swapper_pg_dir, which contains only global
  	 * entries so any speculative walks are perfectly safe.
  	 */
  	asm volatile(
  	"	mrc	p15, 0, %0, c2, c0, 1		@ read TTBR1
  "
  	"	mcr	p15, 0, %0, c2, c0, 0		@ set TTBR0
  "
  	: "=r" (ttb));
  	isb();
  }
  #endif
  
  #ifdef CONFIG_PID_IN_CONTEXTIDR
  static int contextidr_notifier(struct notifier_block *unused, unsigned long cmd,
  			       void *t)
  {
  	u32 contextidr;
  	pid_t pid;
  	struct thread_info *thread = t;
  
  	if (cmd != THREAD_NOTIFY_SWITCH)
  		return NOTIFY_DONE;
  
  	pid = task_pid_nr(thread->task) << ASID_BITS;
  	asm volatile(
  	"	mrc	p15, 0, %0, c13, c0, 1
  "
  	"	and	%0, %0, %2
  "
  	"	orr	%0, %0, %1
  "
  	"	mcr	p15, 0, %0, c13, c0, 1
  "
  	: "=r" (contextidr), "+r" (pid)
  	: "I" (~ASID_MASK));
  	isb();
  
  	return NOTIFY_OK;
  }
  
  static struct notifier_block contextidr_notifier_block = {
  	.notifier_call = contextidr_notifier,
  };
  
  static int __init contextidr_notifier_init(void)
  {
  	return thread_register_notifier(&contextidr_notifier_block);
  }
  arch_initcall(contextidr_notifier_init);
  #endif
  
  static void flush_context(unsigned int cpu)
  {
  	int i;
  	u64 asid;
  
  	/* Update the list of reserved ASIDs and the ASID bitmap. */
  	bitmap_clear(asid_map, 0, NUM_USER_ASIDS);
  	for_each_possible_cpu(i) {
  		if (i == cpu) {
  			asid = 0;
  		} else {
  			asid = atomic64_xchg(&per_cpu(active_asids, i), 0);
  			/*
  			 * If this CPU has already been through a
  			 * rollover, but hasn't run another task in
  			 * the meantime, we must preserve its reserved
  			 * ASID, as this is the only trace we have of
  			 * the process it is still running.
  			 */
  			if (asid == 0)
  				asid = per_cpu(reserved_asids, i);
  			__set_bit(asid & ~ASID_MASK, asid_map);
  		}
  		per_cpu(reserved_asids, i) = asid;
  	}
  
  	/* Queue a TLB invalidate and flush the I-cache if necessary. */
  	cpumask_setall(&tlb_flush_pending);
  
  	if (icache_is_vivt_asid_tagged())
  		__flush_icache_all();
  }
  
  static int is_reserved_asid(u64 asid)
  {
  	int cpu;
  	for_each_possible_cpu(cpu)
  		if (per_cpu(reserved_asids, cpu) == asid)
  			return 1;
  	return 0;
  }
  
  static u64 new_context(struct mm_struct *mm, unsigned int cpu)
  {
  	static u32 cur_idx = 1;
  	u64 asid = atomic64_read(&mm->context.id);
  	u64 generation = atomic64_read(&asid_generation);
  
  	if (asid != 0 && is_reserved_asid(asid)) {
  		/*
  		 * Our current ASID was active during a rollover, we can
  		 * continue to use it and this was just a false alarm.
  		 */
  		asid = generation | (asid & ~ASID_MASK);
  	} else {
  		/*
  		 * Allocate a free ASID. If we can't find one, take a
  		 * note of the currently active ASIDs and mark the TLBs
  		 * as requiring flushes. We always count from ASID #1,
  		 * as we reserve ASID #0 to switch via TTBR0 and to
  		 * avoid speculative page table walks from hitting in
  		 * any partial walk caches, which could be populated
  		 * from overlapping level-1 descriptors used to map both
  		 * the module area and the userspace stack.
  		 */
  		asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, cur_idx);
  		if (asid == NUM_USER_ASIDS) {
  			generation = atomic64_add_return(ASID_FIRST_VERSION,
  							 &asid_generation);
  			flush_context(cpu);
  			asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, 1);
  		}
  		__set_bit(asid, asid_map);
  		cur_idx = asid;
  		asid |= generation;
  		cpumask_clear(mm_cpumask(mm));
  	}
  
  	return asid;
  }
  
  void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk)
  {
  	unsigned long flags;
  	unsigned int cpu = smp_processor_id();
  	u64 asid;
  
  	if (unlikely(mm->context.vmalloc_seq != init_mm.context.vmalloc_seq))
  		__check_vmalloc_seq(mm);
  
  	/*
  	 * We cannot update the pgd and the ASID atomicly with classic
  	 * MMU, so switch exclusively to global mappings to avoid
  	 * speculative page table walking with the wrong TTBR.
  	 */
  	cpu_set_reserved_ttbr0();
  
  	asid = atomic64_read(&mm->context.id);
  	if (!((asid ^ atomic64_read(&asid_generation)) >> ASID_BITS)
  	    && atomic64_xchg(&per_cpu(active_asids, cpu), asid))
  		goto switch_mm_fastpath;
  
  	raw_spin_lock_irqsave(&cpu_asid_lock, flags);
  	/* Check that our ASID belongs to the current generation. */
  	asid = atomic64_read(&mm->context.id);
  	if ((asid ^ atomic64_read(&asid_generation)) >> ASID_BITS) {
  		asid = new_context(mm, cpu);
  		atomic64_set(&mm->context.id, asid);
  	}
  
  	if (cpumask_test_and_clear_cpu(cpu, &tlb_flush_pending)) {
  		local_flush_bp_all();
  		local_flush_tlb_all();
  	}
  
  	atomic64_set(&per_cpu(active_asids, cpu), asid);
  	cpumask_set_cpu(cpu, mm_cpumask(mm));
  	raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);
  
  switch_mm_fastpath:
  	cpu_switch_mm(mm->pgd, mm);
  }