/*
   * Based on arch/arm/mm/fault.c
   *
   * Copyright (C) 1995  Linus Torvalds
   * Copyright (C) 1995-2004 Russell King
   * Copyright (C) 2012 ARM Ltd.
   *
   * This program is free software; you can redistribute it and/or modify
   * it under the terms of the GNU General Public License version 2 as
   * published by the Free Software Foundation.
   *
   * This program is distributed in the hope that it will be useful,
   * but WITHOUT ANY WARRANTY; without even the implied warranty of
   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   * GNU General Public License for more details.
   *
   * You should have received a copy of the GNU General Public License
   * along with this program.  If not, see <http://www.gnu.org/licenses/>.
   */
  
  #include <linux/module.h>
  #include <linux/signal.h>
  #include <linux/mm.h>
  #include <linux/hardirq.h>
  #include <linux/init.h>
  #include <linux/kprobes.h>
  #include <linux/uaccess.h>
  #include <linux/page-flags.h>
  #include <linux/sched.h>
  #include <linux/highmem.h>
  #include <linux/perf_event.h>
  
  #include <asm/exception.h>
  #include <asm/debug-monitors.h>
  #include <asm/system_misc.h>
  #include <asm/pgtable.h>
  #include <asm/tlbflush.h>
  
  static const char *fault_name(unsigned int esr);
  
  /*
   * Dump out the page tables associated with 'addr' in mm 'mm'.
   */
  void show_pte(struct mm_struct *mm, unsigned long addr)
  {
  	pgd_t *pgd;
  
  	if (!mm)
  		mm = &init_mm;
  
  	pr_alert("pgd = %p
  ", mm->pgd);
  	pgd = pgd_offset(mm, addr);
  	pr_alert("[%08lx] *pgd=%016llx", addr, pgd_val(*pgd));
  
  	do {
  		pud_t *pud;
  		pmd_t *pmd;
  		pte_t *pte;
  
  		if (pgd_none(*pgd) || pgd_bad(*pgd))
  			break;
  
  		pud = pud_offset(pgd, addr);
  		if (pud_none(*pud) || pud_bad(*pud))
  			break;
  
  		pmd = pmd_offset(pud, addr);
  		printk(", *pmd=%016llx", pmd_val(*pmd));
  		if (pmd_none(*pmd) || pmd_bad(*pmd))
  			break;
  
  		pte = pte_offset_map(pmd, addr);
  		printk(", *pte=%016llx", pte_val(*pte));
  		pte_unmap(pte);
  	} while(0);
  
  	printk("
  ");
  }
  
  /*
   * The kernel tried to access some page that wasn't present.
   */
  static void __do_kernel_fault(struct mm_struct *mm, unsigned long addr,
  			      unsigned int esr, struct pt_regs *regs)
  {
  	/*
  	 * Are we prepared to handle this kernel fault?
  	 */
  	if (fixup_exception(regs))
  		return;
  
  	/*
  	 * No handler, we'll have to terminate things with extreme prejudice.
  	 */
  	bust_spinlocks(1);
  	pr_alert("Unable to handle kernel %s at virtual address %08lx
  ",
  		 (addr < PAGE_SIZE) ? "NULL pointer dereference" :
  		 "paging request", addr);
  
  	show_pte(mm, addr);
  	die("Oops", regs, esr);
  	bust_spinlocks(0);
  	do_exit(SIGKILL);
  }
  
  /*
   * Something tried to access memory that isn't in our memory map. User mode
   * accesses just cause a SIGSEGV
   */
  static void __do_user_fault(struct task_struct *tsk, unsigned long addr,
  			    unsigned int esr, unsigned int sig, int code,
  			    struct pt_regs *regs)
  {
  	struct siginfo si;
  
  	if (show_unhandled_signals && unhandled_signal(tsk, sig) &&
  	    printk_ratelimit()) {
  		pr_info("%s[%d]: unhandled %s (%d) at 0x%08lx, esr 0x%03x
  ",
  			tsk->comm, task_pid_nr(tsk), fault_name(esr), sig,
  			addr, esr);
  		show_pte(tsk->mm, addr);
  		show_regs(regs);
  	}
  
  	tsk->thread.fault_address = addr;
  	si.si_signo = sig;
  	si.si_errno = 0;
  	si.si_code = code;
  	si.si_addr = (void __user *)addr;
  	force_sig_info(sig, &si, tsk);
  }
  
  static void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *regs)
  {
  	struct task_struct *tsk = current;
  	struct mm_struct *mm = tsk->active_mm;
  
  	/*
  	 * If we are in kernel mode at this point, we have no context to
  	 * handle this fault with.
  	 */
  	if (user_mode(regs))
  		__do_user_fault(tsk, addr, esr, SIGSEGV, SEGV_MAPERR, regs);
  	else
  		__do_kernel_fault(mm, addr, esr, regs);
  }
  
  #define VM_FAULT_BADMAP		0x010000
  #define VM_FAULT_BADACCESS	0x020000
  
  #define ESR_WRITE		(1 << 6)
  #define ESR_CM			(1 << 8)
  #define ESR_LNX_EXEC		(1 << 24)
  
  static int __do_page_fault(struct mm_struct *mm, unsigned long addr,
  			   unsigned int mm_flags, unsigned long vm_flags,
  			   struct task_struct *tsk)
  {
  	struct vm_area_struct *vma;
  	int fault;
  
  	vma = find_vma(mm, addr);
  	fault = VM_FAULT_BADMAP;
  	if (unlikely(!vma))
  		goto out;
  	if (unlikely(vma->vm_start > addr))
  		goto check_stack;
  
  	/*
  	 * Ok, we have a good vm_area for this memory access, so we can handle
  	 * it.
  	 */
  good_area:
  	/*
  	 * Check that the permissions on the VMA allow for the fault which
  	 * occurred. If we encountered a write or exec fault, we must have
  	 * appropriate permissions, otherwise we allow any permission.
  	 */
  	if (!(vma->vm_flags & vm_flags)) {
  		fault = VM_FAULT_BADACCESS;
  		goto out;
  	}
  
  	return handle_mm_fault(mm, vma, addr & PAGE_MASK, mm_flags);
  
  check_stack:
  	if (vma->vm_flags & VM_GROWSDOWN && !expand_stack(vma, addr))
  		goto good_area;
  out:
  	return fault;
  }
  
  static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
  				   struct pt_regs *regs)
  {
  	struct task_struct *tsk;
  	struct mm_struct *mm;
  	int fault, sig, code;
  	unsigned long vm_flags = VM_READ | VM_WRITE | VM_EXEC;
  	unsigned int mm_flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
  
  	tsk = current;
  	mm  = tsk->mm;
  
  	/* Enable interrupts if they were enabled in the parent context. */
  	if (interrupts_enabled(regs))
  		local_irq_enable();
  
  	/*
  	 * If we're in an interrupt or have no user context, we must not take
  	 * the fault.
  	 */
  	if (in_atomic() || !mm)
  		goto no_context;
  
  	if (user_mode(regs))
  		mm_flags |= FAULT_FLAG_USER;
  
  	if (esr & ESR_LNX_EXEC) {
  		vm_flags = VM_EXEC;
  	} else if ((esr & ESR_WRITE) && !(esr & ESR_CM)) {
  		vm_flags = VM_WRITE;
  		mm_flags |= FAULT_FLAG_WRITE;
  	}
  
  	/*
  	 * As per x86, we may deadlock here. However, since the kernel only
  	 * validly references user space from well defined areas of the code,
  	 * we can bug out early if this is from code which shouldn't.
  	 */
  	if (!down_read_trylock(&mm->mmap_sem)) {
  		if (!user_mode(regs) && !search_exception_tables(regs->pc))
  			goto no_context;
  retry:
  		down_read(&mm->mmap_sem);
  	} else {
  		/*
  		 * The above down_read_trylock() might have succeeded in which
  		 * case, we'll have missed the might_sleep() from down_read().
  		 */
  		might_sleep();
  #ifdef CONFIG_DEBUG_VM
  		if (!user_mode(regs) && !search_exception_tables(regs->pc))
  			goto no_context;
  #endif
  	}
  
  	fault = __do_page_fault(mm, addr, mm_flags, vm_flags, tsk);
  
  	/*
  	 * If we need to retry but a fatal signal is pending, handle the
  	 * signal first. We do not need to release the mmap_sem because it
  	 * would already be released in __lock_page_or_retry in mm/filemap.c.
  	 */
  	if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
  		return 0;
  
  	/*
  	 * Major/minor page fault accounting is only done on the initial
  	 * attempt. If we go through a retry, it is extremely likely that the
  	 * page will be found in page cache at that point.
  	 */
  
  	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);
  	if (mm_flags & FAULT_FLAG_ALLOW_RETRY) {
  		if (fault & VM_FAULT_MAJOR) {
  			tsk->maj_flt++;
  			perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs,
  				      addr);
  		} else {
  			tsk->min_flt++;
  			perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs,
  				      addr);
  		}
  		if (fault & VM_FAULT_RETRY) {
  			/*
  			 * Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk of
  			 * starvation.
  			 */
  			mm_flags &= ~FAULT_FLAG_ALLOW_RETRY;
  			goto retry;
  		}
  	}
  
  	up_read(&mm->mmap_sem);
  
  	/*
  	 * Handle the "normal" case first - VM_FAULT_MAJOR / VM_FAULT_MINOR
  	 */
  	if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP |
  			      VM_FAULT_BADACCESS))))
  		return 0;
  
  	/*
  	 * If we are in kernel mode at this point, we have no context to
  	 * handle this fault with.
  	 */
  	if (!user_mode(regs))
  		goto no_context;
  
  	if (fault & VM_FAULT_OOM) {
  		/*
  		 * We ran out of memory, call the OOM killer, and return to
  		 * userspace (which will retry the fault, or kill us if we got
  		 * oom-killed).
  		 */
  		pagefault_out_of_memory();
  		return 0;
  	}
  
  	if (fault & VM_FAULT_SIGBUS) {
  		/*
  		 * We had some memory, but were unable to successfully fix up
  		 * this page fault.
  		 */
  		sig = SIGBUS;
  		code = BUS_ADRERR;
  	} else {
  		/*
  		 * Something tried to access memory that isn't in our memory
  		 * map.
  		 */
  		sig = SIGSEGV;
  		code = fault == VM_FAULT_BADACCESS ?
  			SEGV_ACCERR : SEGV_MAPERR;
  	}
  
  	__do_user_fault(tsk, addr, esr, sig, code, regs);
  	return 0;
  
  no_context:
  	__do_kernel_fault(mm, addr, esr, regs);
  	return 0;
  }
  
  /*
   * First Level Translation Fault Handler
   *
   * We enter here because the first level page table doesn't contain a valid
   * entry for the address.
   *
   * If the address is in kernel space (>= TASK_SIZE), then we are probably
   * faulting in the vmalloc() area.
   *
   * If the init_task's first level page tables contains the relevant entry, we
   * copy the it to this task.  If not, we send the process a signal, fixup the
   * exception, or oops the kernel.
   *
   * NOTE! We MUST NOT take any locks for this case. We may be in an interrupt
   * or a critical region, and should only copy the information from the master
   * page table, nothing more.
   */
  static int __kprobes do_translation_fault(unsigned long addr,
  					  unsigned int esr,
  					  struct pt_regs *regs)
  {
  	if (addr < TASK_SIZE)
  		return do_page_fault(addr, esr, regs);
  
  	do_bad_area(addr, esr, regs);
  	return 0;
  }
  
  /*
   * This abort handler always returns "fault".
   */
  static int do_bad(unsigned long addr, unsigned int esr, struct pt_regs *regs)
  {
  	return 1;
  }
  
  static struct fault_info {
  	int	(*fn)(unsigned long addr, unsigned int esr, struct pt_regs *regs);
  	int	sig;
  	int	code;
  	const char *name;
  } fault_info[] = {
  	{ do_bad,		SIGBUS,  0,		"ttbr address size fault"	},
  	{ do_bad,		SIGBUS,  0,		"level 1 address size fault"	},
  	{ do_bad,		SIGBUS,  0,		"level 2 address size fault"	},
  	{ do_bad,		SIGBUS,  0,		"level 3 address size fault"	},
  	{ do_translation_fault,	SIGSEGV, SEGV_MAPERR,	"input address range fault"	},
  	{ do_translation_fault,	SIGSEGV, SEGV_MAPERR,	"level 1 translation fault"	},
  	{ do_translation_fault,	SIGSEGV, SEGV_MAPERR,	"level 2 translation fault"	},
  	{ do_page_fault,	SIGSEGV, SEGV_MAPERR,	"level 3 translation fault"	},
  	{ do_bad,		SIGBUS,  0,		"reserved access flag fault"	},
  	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 1 access flag fault"	},
  	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 2 access flag fault"	},
  	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 3 access flag fault"	},
  	{ do_bad,		SIGBUS,  0,		"reserved permission fault"	},
  	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 1 permission fault"	},
  	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 2 permission fault"	},
  	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 3 permission fault"	},
  	{ do_bad,		SIGBUS,  0,		"synchronous external abort"	},
  	{ do_bad,		SIGBUS,  0,		"asynchronous external abort"	},
  	{ do_bad,		SIGBUS,  0,		"unknown 18"			},
  	{ do_bad,		SIGBUS,  0,		"unknown 19"			},
  	{ do_bad,		SIGBUS,  0,		"synchronous abort (translation table walk)" },
  	{ do_bad,		SIGBUS,  0,		"synchronous abort (translation table walk)" },
  	{ do_bad,		SIGBUS,  0,		"synchronous abort (translation table walk)" },
  	{ do_bad,		SIGBUS,  0,		"synchronous abort (translation table walk)" },
  	{ do_bad,		SIGBUS,  0,		"synchronous parity error"	},
  	{ do_bad,		SIGBUS,  0,		"asynchronous parity error"	},
  	{ do_bad,		SIGBUS,  0,		"unknown 26"			},
  	{ do_bad,		SIGBUS,  0,		"unknown 27"			},
  	{ do_bad,		SIGBUS,  0,		"synchronous parity error (translation table walk" },
  	{ do_bad,		SIGBUS,  0,		"synchronous parity error (translation table walk" },
  	{ do_bad,		SIGBUS,  0,		"synchronous parity error (translation table walk" },
  	{ do_bad,		SIGBUS,  0,		"synchronous parity error (translation table walk" },
  	{ do_bad,		SIGBUS,  0,		"unknown 32"			},
  	{ do_bad,		SIGBUS,  BUS_ADRALN,	"alignment fault"		},
  	{ do_bad,		SIGBUS,  0,		"debug event"			},
  	{ do_bad,		SIGBUS,  0,		"unknown 35"			},
  	{ do_bad,		SIGBUS,  0,		"unknown 36"			},
  	{ do_bad,		SIGBUS,  0,		"unknown 37"			},
  	{ do_bad,		SIGBUS,  0,		"unknown 38"			},
  	{ do_bad,		SIGBUS,  0,		"unknown 39"			},
  	{ do_bad,		SIGBUS,  0,		"unknown 40"			},
  	{ do_bad,		SIGBUS,  0,		"unknown 41"			},
  	{ do_bad,		SIGBUS,  0,		"unknown 42"			},
  	{ do_bad,		SIGBUS,  0,		"unknown 43"			},
  	{ do_bad,		SIGBUS,  0,		"unknown 44"			},
  	{ do_bad,		SIGBUS,  0,		"unknown 45"			},
  	{ do_bad,		SIGBUS,  0,		"unknown 46"			},
  	{ do_bad,		SIGBUS,  0,		"unknown 47"			},
  	{ do_bad,		SIGBUS,  0,		"unknown 48"			},
  	{ do_bad,		SIGBUS,  0,		"unknown 49"			},
  	{ do_bad,		SIGBUS,  0,		"unknown 50"			},
  	{ do_bad,		SIGBUS,  0,		"unknown 51"			},
  	{ do_bad,		SIGBUS,  0,		"implementation fault (lockdown abort)" },
  	{ do_bad,		SIGBUS,  0,		"unknown 53"			},
  	{ do_bad,		SIGBUS,  0,		"unknown 54"			},
  	{ do_bad,		SIGBUS,  0,		"unknown 55"			},
  	{ do_bad,		SIGBUS,  0,		"unknown 56"			},
  	{ do_bad,		SIGBUS,  0,		"unknown 57"			},
  	{ do_bad,		SIGBUS,  0,		"implementation fault (coprocessor abort)" },
  	{ do_bad,		SIGBUS,  0,		"unknown 59"			},
  	{ do_bad,		SIGBUS,  0,		"unknown 60"			},
  	{ do_bad,		SIGBUS,  0,		"unknown 61"			},
  	{ do_bad,		SIGBUS,  0,		"unknown 62"			},
  	{ do_bad,		SIGBUS,  0,		"unknown 63"			},
  };
  
  static const char *fault_name(unsigned int esr)
  {
  	const struct fault_info *inf = fault_info + (esr & 63);
  	return inf->name;
  }
  
  /*
   * Dispatch a data abort to the relevant handler.
   */
  asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr,
  					 struct pt_regs *regs)
  {
  	const struct fault_info *inf = fault_info + (esr & 63);
  	struct siginfo info;
  
  	if (!inf->fn(addr, esr, regs))
  		return;
  
  	pr_alert("Unhandled fault: %s (0x%08x) at 0x%016lx
  ",
  		 inf->name, esr, addr);
  
  	info.si_signo = inf->sig;
  	info.si_errno = 0;
  	info.si_code  = inf->code;
  	info.si_addr  = (void __user *)addr;
  	arm64_notify_die("", regs, &info, esr);
  }
  
  /*
   * Handle stack alignment exceptions.
   */
  asmlinkage void __exception do_sp_pc_abort(unsigned long addr,
  					   unsigned int esr,
  					   struct pt_regs *regs)
  {
  	struct siginfo info;
  
  	info.si_signo = SIGBUS;
  	info.si_errno = 0;
  	info.si_code  = BUS_ADRALN;
  	info.si_addr  = (void __user *)addr;
  	arm64_notify_die("", regs, &info, esr);
  }
  
  static struct fault_info debug_fault_info[] = {
  	{ do_bad,	SIGTRAP,	TRAP_HWBKPT,	"hardware breakpoint"	},
  	{ do_bad,	SIGTRAP,	TRAP_HWBKPT,	"hardware single-step"	},
  	{ do_bad,	SIGTRAP,	TRAP_HWBKPT,	"hardware watchpoint"	},
  	{ do_bad,	SIGBUS,		0,		"unknown 3"		},
  	{ do_bad,	SIGTRAP,	TRAP_BRKPT,	"aarch32 BKPT"		},
  	{ do_bad,	SIGTRAP,	0,		"aarch32 vector catch"	},
  	{ do_bad,	SIGTRAP,	TRAP_BRKPT,	"aarch64 BRK"		},
  	{ do_bad,	SIGBUS,		0,		"unknown 7"		},
  };
  
  void __init hook_debug_fault_code(int nr,
  				  int (*fn)(unsigned long, unsigned int, struct pt_regs *),
  				  int sig, int code, const char *name)
  {
  	BUG_ON(nr < 0 || nr >= ARRAY_SIZE(debug_fault_info));
  
  	debug_fault_info[nr].fn		= fn;
  	debug_fault_info[nr].sig	= sig;
  	debug_fault_info[nr].code	= code;
  	debug_fault_info[nr].name	= name;
  }
  
  asmlinkage int __exception do_debug_exception(unsigned long addr,
  					      unsigned int esr,
  					      struct pt_regs *regs)
  {
  	const struct fault_info *inf = debug_fault_info + DBG_ESR_EVT(esr);
  	struct siginfo info;
  
  	if (!inf->fn(addr, esr, regs))
  		return 1;
  
  	pr_alert("Unhandled debug exception: %s (0x%08x) at 0x%016lx
  ",
  		 inf->name, esr, addr);
  
  	info.si_signo = inf->sig;
  	info.si_errno = 0;
  	info.si_code  = inf->code;
  	info.si_addr  = (void __user *)addr;
  	arm64_notify_die("", regs, &info, esr);
  
  	return 0;
  }